repo-sync-2024-08-23T11:30:55+0800 (#177)

* repo-sync-2024-08-23T11:30:55+0800 * repo-sync-2024-08-27T17:04:28+0800
secretflow · Aug 28, 2024 · 6a339a0 · 6a339a0
1 parent b8d10fb
commit 6a339a0
Show file tree

Hide file tree

Showing 72 changed files with 3,650 additions and 2,915 deletions.
diff --git a/README.md b/README.md
@@ -28,7 +28,7 @@ receiver.config:
 {
     "psi_config": {
         "protocol_config": {
-            "protocol": "PROTOCOL_KKRT",
+            "protocol": "PROTOCOL_RR22",
             "role": "ROLE_RECEIVER",
             "broadcast_result": true
         },
@@ -70,7 +70,7 @@ sender.config:
 {
     "psi_config": {
         "protocol_config": {
-            "protocol": "PROTOCOL_KKRT",
+            "protocol": "PROTOCOL_RR22",
             "role": "ROLE_SENDER",
             "broadcast_result": true
         },
@@ -132,7 +132,7 @@ You could also pass a minified JSON config directly. A minified JSON is a compac
 
 e.g.
 ```
-docker run -it  --rm  --network host --mount type=bind,source=/tmp/sender,target=/root/sender  --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --cap-add=NET_ADMIN --privileged=true secretflow-registry.cn-hangzhou.cr.aliyuncs.com/secretflow/psi-anolis8:latest --json '{"psi_config":{"protocol_config":{"protocol":"PROTOCOL_KKRT","role":"ROLE_RECEIVER","broadcast_result":true},"input_config":{"type":"IO_TYPE_FILE_CSV","path":"/root/receiver/receiver_input.csv"},"output_config":{"type":"IO_TYPE_FILE_CSV","path":"/root/receiver/receiver_output.csv"},"keys":["id0","id1"],"debug_options":{"trace_path":"/root/receiver/receiver.trace"}},"self_link_party":"receiver","link_config":{"parties":[{"id":"receiver","host":"127.0.0.1:5300"},{"id":"sender","host":"127.0.0.1:5400"}]}}'
+docker run -it  --rm  --network host --mount type=bind,source=/tmp/sender,target=/root/sender  --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --cap-add=NET_ADMIN --privileged=true secretflow-registry.cn-hangzhou.cr.aliyuncs.com/secretflow/psi-anolis8:latest --json '{"psi_config":{"protocol_config":{"protocol":"PROTOCOL_RR22","role":"ROLE_RECEIVER","broadcast_result":true},"input_config":{"type":"IO_TYPE_FILE_CSV","path":"/root/receiver/receiver_input.csv"},"output_config":{"type":"IO_TYPE_FILE_CSV","path":"/root/receiver/receiver_output.csv"},"keys":["id0","id1"],"debug_options":{"trace_path":"/root/receiver/receiver.trace"}},"self_link_party":"receiver","link_config":{"parties":[{"id":"receiver","host":"127.0.0.1:5300"},{"id":"sender","host":"127.0.0.1:5400"}]}}'
 ```
 
 ## Building SecretFlow PSI Library
@@ -207,6 +207,4 @@ chmod +x traceconv
 
 
 
-## PSI V2 Benchamrk
 
-Please refer to [PSI V2 Benchmark](docs/user_guide/psi_v2_benchmark.md)
diff --git a/bazel/jsoncpp.BUILD b/bazel/jsoncpp.BUILD
@@ -30,6 +30,9 @@ psi_cmake_external(
         "BUILD_OBJECT_LIBS": "OFF",
         "CMAKE_INSTALL_LIBDIR": "lib",
     },
+    env = {
+        "CCACHE_DISABLE": "1",
+    },
     lib_source = "@com_github_open_source_parsers_jsoncpp//:all",
     out_static_libs = ["libjsoncpp.a"],
 )
diff --git a/bazel/patches/apsi-fourq.patch b/bazel/patches/apsi-fourq.patch
@@ -1,32 +1,19 @@
 diff --git a/CMakeLists.txt b/CMakeLists.txt
-index 55e2d77..80d0afc 100644
+index 78d54a6..166047c 100644
 --- a/CMakeLists.txt
 +++ b/CMakeLists.txt
-@@ -146,7 +146,11 @@ if(NOT Flatbuffers_FOUND)
-     message(FATAL_ERROR "Flatbuffers: not found")
- else()
-     message(STATUS "Flatbuffers: found")
--    get_target_property(FLATBUFFERS_FLATC_PATH flatbuffers::flatc IMPORTED_LOCATION_RELEASE)
-+    if (CMAKE_BUILD_TYPE STREQUAL "Debug")
-+        get_target_property(FLATBUFFERS_FLATC_PATH flatbuffers::flatc IMPORTED_LOCATION_DEBUG)
-+    else()
-+        get_target_property(FLATBUFFERS_FLATC_PATH flatbuffers::flatc IMPORTED_LOCATION_RELEASE)
-+    endif()
-     message(STATUS "flatc path: ${FLATBUFFERS_FLATC_PATH}")
-     include(CompileSchemaCXX)
- endif()
-@@ -273,10 +277,9 @@ if(APSI_USE_ZMQ)
+@@ -273,10 +273,9 @@ if(APSI_USE_ZMQ)
      target_link_libraries(apsi PUBLIC libzmq-static cppzmq-static)
  endif()
- 
+
 -# Configurations for FourQlib: system, arch, SIMD, and assembler
 -target_compile_options(apsi PUBLIC -DHAVE_CONFIG)
 -target_compile_options(apsi PUBLIC -DUSE_SECURE_SEED)
 -target_compile_options(apsi PUBLIC -DUSE_ENDO=true)
 +# Add FourQlib
 +target_include_directories(apsi PUBLIC ${EXT_BUILD_DEPS}/FourQlib/include)
 +target_link_libraries(apsi PUBLIC FourQ)
- 
+
  # Set system
  if(MSVC)
 diff --git a/common/apsi/CMakeLists.txt b/common/apsi/CMakeLists.txt
@@ -36,7 +23,7 @@ index a65bbfe..60e246e 100644
 @@ -28,7 +28,6 @@ install(
          ${APSI_INCLUDES_INSTALL_DIR}/apsi
  )
- 
+
 -add_subdirectory(fourq)
  add_subdirectory(network)
  add_subdirectory(oprf)
@@ -47,7 +34,7 @@ index bcaa013..93e2b4a 100644
 +++ b/common/apsi/network/zmq/zmq_channel.cpp
 @@ -8,7 +8,7 @@
  #include <stdexcept>
- 
+
  // APSI
 -#include "apsi/fourq/random.h"
 +#include "random.h"
@@ -60,7 +47,7 @@ index d12313f..5fd9be1 100644
 +++ b/common/apsi/oprf/ecpoint.cpp
 @@ -10,10 +10,10 @@
  #include "apsi/util/utils.h"
- 
+
  // FourQ
 -#include "apsi/fourq/FourQ.h"
 -#include "apsi/fourq/FourQ_api.h"
@@ -70,7 +57,7 @@ index d12313f..5fd9be1 100644
 +#include "FourQ_api.h"
 +#include "FourQ_internal.h"
 +#include "random.h"
- 
+
  // SEAL
  #include "seal/randomgen.h"
 diff --git a/common/apsi/util/label_encryptor.cpp b/common/apsi/util/label_encryptor.cpp
@@ -79,7 +66,7 @@ index 3e00b5e..57a5b40 100644
 +++ b/common/apsi/util/label_encryptor.cpp
 @@ -9,7 +9,7 @@
  #include <vector>
- 
+
  // APSI
 -#include "apsi/fourq/random.h"
 +#include "random.h"

diff --git a/bazel/patches/apsi.patch b/bazel/patches/apsi.patch
@@ -1,3 +1,20 @@
+diff --git a/CMakeLists.txt b/CMakeLists.txt
+index 55e2d77..7d3007b 100644
+--- a/CMakeLists.txt
++++ b/CMakeLists.txt
+@@ -146,7 +146,11 @@ if(NOT Flatbuffers_FOUND)
+     message(FATAL_ERROR "Flatbuffers: not found")
+ else()
+     message(STATUS "Flatbuffers: found")
+-    get_target_property(FLATBUFFERS_FLATC_PATH flatbuffers::flatc IMPORTED_LOCATION_RELEASE)
++    if (CMAKE_BUILD_TYPE STREQUAL "Release")
++        get_target_property(FLATBUFFERS_FLATC_PATH flatbuffers::flatc IMPORTED_LOCATION_RELEASE)
++    else()
++        get_target_property(FLATBUFFERS_FLATC_PATH flatbuffers::flatc IMPORTED_LOCATION_DEBUG)
++    endif()
+     message(STATUS "flatc path: ${FLATBUFFERS_FLATC_PATH}")
+     include(CompileSchemaCXX)
+ endif()
 diff --git a/receiver/apsi/itt.h b/receiver/apsi/itt.h
 index e683045..067d244 100644
 --- a/receiver/apsi/itt.h

diff --git a/bazel/patches/seal.patch b/bazel/patches/seal.patch
@@ -34,7 +34,7 @@ index 31e07441..6f8e6b2a 100644
              uint64_t coeff_modulus_size64 = static_cast<uint64_t>(coeff_modulus_.size());
              uint8_t scheme = static_cast<uint8_t>(scheme_);
 +            uint8_t use_special_prime_size8 = static_cast<uint8_t>(use_special_prime_);
- 
+
              stream.write(reinterpret_cast<const char *>(&scheme), sizeof(uint8_t));
 +            stream.write(reinterpret_cast<const char *>(&use_special_prime_size8), sizeof(uint8_t));
              stream.write(reinterpret_cast<const char *>(&poly_modulus_degree64), sizeof(uint64_t));
@@ -43,7 +43,7 @@ index 31e07441..6f8e6b2a 100644
 @@ -63,6 +65,9 @@ namespace seal
              // This constructor will throw if scheme is invalid
              EncryptionParameters parms(scheme);
- 
+
 +            uint8_t use_special_prime_size8;
 +            stream.read(reinterpret_cast<char *>(&use_special_prime_size8), sizeof(uint8_t));
 +
@@ -55,7 +55,7 @@ index 31e07441..6f8e6b2a 100644
              parms.set_poly_modulus_degree(safe_cast<size_t>(poly_modulus_degree64));
              parms.set_coeff_modulus(coeff_modulus);
 +            parms.set_use_special_prime(use_special_prime_size8);
- 
+
              // Only BFV and BGV uses plain_modulus; set_plain_modulus checks that for
              // other schemes it is zero
 @@ -128,6 +134,7 @@ namespace seal
@@ -64,12 +64,12 @@ index 31e07441..6f8e6b2a 100644
              size_t(1), // poly_modulus_degree
 +            size_t(1), // use_special_prime
              coeff_modulus_size, plain_modulus_.uint64_count());
- 
+
          auto param_data(allocate_uint(total_uint64_count, pool_));
 @@ -139,6 +146,7 @@ namespace seal
          // Write the poly_modulus_degree. Note that it will always be positive.
          *param_data_ptr++ = static_cast<uint64_t>(poly_modulus_degree_);
- 
+
 +        *param_data_ptr++ = static_cast<uint64_t>(use_special_prime_);
          for (const auto &mod : coeff_modulus_)
          {
@@ -81,7 +81,7 @@ index 9e1fbe48..8530eeeb 100644
 @@ -266,6 +266,11 @@ namespace seal
              random_generator_ = std::move(random_generator);
          }
- 
+
 +        inline void set_use_special_prime(bool flag)
 +        {
 +            use_special_prime_ = flag;
@@ -93,7 +93,7 @@ index 9e1fbe48..8530eeeb 100644
 @@ -274,6 +279,11 @@ namespace seal
              return scheme_;
          }
- 
+
 +        bool use_special_prime() const noexcept
 +        {
 +            return use_special_prime_;
@@ -111,9 +111,9 @@ index 9e1fbe48..8530eeeb 100644
                      sizeof(std::uint64_t), // coeff_modulus_size
                      coeff_modulus_total_size,
 @@ -501,6 +512,8 @@ namespace seal
- 
+
          Modulus plain_modulus_{};
- 
+
 +        bool use_special_prime_ = true;
 +
          parms_id_type parms_id_ = parms_id_zero;
@@ -128,7 +128,7 @@ index dabd3bab..61a96ae9 100644
          // Use key_context_data where permutation tables exist since previous runs.
          auto galois_tool = context_.key_context_data()->galois_tool();
 +        bool is_ntt_form = encrypted.is_ntt_form();
- 
+
          // Size check
          if (!product_fits_in(coeff_count, coeff_modulus_size))
 @@ -2412,7 +2413,7 @@ namespace seal
@@ -139,7 +139,7 @@ index dabd3bab..61a96ae9 100644
 +        if (not is_ntt_form)
          {
              // !!! DO NOT CHANGE EXECUTION ORDER!!!
- 
+
 @@ -2426,7 +2427,7 @@ namespace seal
              // Next transform encrypted.data(1)
              galois_tool->apply_galois(encrypted_iter[1], coeff_modulus_size, galois_elt, coeff_modulus, temp);
@@ -148,7 +148,7 @@ index dabd3bab..61a96ae9 100644
 +        else
          {
              // !!! DO NOT CHANGE EXECUTION ORDER!!!
- 
+
 @@ -2440,10 +2441,6 @@ namespace seal
              // Next transform encrypted.data(1)
              galois_tool->apply_galois_ntt(encrypted_iter[1], coeff_modulus_size, galois_elt, temp);
@@ -157,15 +157,15 @@ index dabd3bab..61a96ae9 100644
 -        {
 -            throw logic_error("scheme not implemented");
 -        }
- 
+
          // Wipe encrypted.data(1)
          set_zero_poly(coeff_count, coeff_modulus_size, encrypted.data(1));
 @@ -2530,6 +2527,7 @@ namespace seal
          auto &key_context_data = *context_.key_context_data();
          auto &key_parms = key_context_data.parms();
          auto scheme = parms.scheme();
 +        bool is_ntt_form = encrypted.is_ntt_form();
- 
+
          // Verify parameters.
          if (!is_metadata_valid_for(encrypted, context_) || !is_buffer_valid(encrypted))
 @@ -2559,14 +2557,6 @@ namespace seal
@@ -185,7 +185,7 @@ index dabd3bab..61a96ae9 100644
              throw invalid_argument("BGV encrypted must be in NTT form");
 @@ -2605,7 +2595,7 @@ namespace seal
          set_uint(target_iter, decomp_modulus_size * coeff_count, t_target);
- 
+
          // In CKKS or BGV, t_target is in NTT form; switch back to normal form
 -        if (scheme == scheme_type::ckks || scheme == scheme_type::bgv)
 +        if (is_ntt_form)
@@ -194,7 +194,7 @@ index dabd3bab..61a96ae9 100644
          }
 @@ -2632,7 +2622,7 @@ namespace seal
                  ConstCoeffIter t_operand;
- 
+
                  // RNS-NTT form exists in input
 -                if ((scheme == scheme_type::ckks || scheme == scheme_type::bgv) && (I == J))
 +                if (is_ntt_form && (I == J))
@@ -203,7 +203,7 @@ index dabd3bab..61a96ae9 100644
                  }
 @@ -2789,7 +2779,7 @@ namespace seal
                      SEAL_ITERATE(t_ntt, coeff_count, [fix](auto &K) { K += fix; });
- 
+
                      uint64_t qi_lazy = qi << 1; // some multiples of qi
 -                    if (scheme == scheme_type::ckks)
 +                    if (is_ntt_form)
@@ -226,15 +226,15 @@ index 9e3dd576..bb598ddf 100644
 @@ -1355,10 +1355,12 @@ namespace seal
              apply_galois_inplace(encrypted, galois_tool->get_elt_from_step(0), galois_keys, std::move(pool));
          }
- 
+
 +    public:
          void switch_key_inplace(
              Ciphertext &encrypted, util::ConstRNSIter target_iter, const KSwitchKeys &kswitch_keys,
              std::size_t key_index, MemoryPoolHandle pool = MemoryManager::GetPool()) const;
- 
+
 +    private:
          void multiply_plain_normal(Ciphertext &encrypted, const Plaintext &plain, MemoryPoolHandle pool) const;
- 
+
          void multiply_plain_ntt(Ciphertext &encrypted_ntt, const Plaintext &plain_ntt) const;
 diff --git a/native/src/seal/serializable.h b/native/src/seal/serializable.h
 index a940190c..e490b302 100644
@@ -243,7 +243,7 @@ index a940190c..e490b302 100644
 @@ -135,6 +135,9 @@ namespace seal
              return obj_.save(out, size, compr_mode);
          }
- 
+
 +        const T& obj() const { return obj_; }
 +
 +        T& obj() { return obj_; }

diff --git a/bazel/psi.bzl b/bazel/psi.bzl
@@ -24,7 +24,7 @@ WARNING_FLAGS = [
     "-Wextra",
     "-Werror",
 ]
-DEBUG_FLAGS = ["-O0", "-g"]
+DEBUG_FLAGS = ["-O0", "-g", "-DSPDLOG_ACTIVE_LEVEL=1"]
 RELEASE_FLAGS = ["-O2"]
 FAST_FLAGS = ["-O1"]
 

diff --git a/docker/build.sh b/docker/build.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-set -e
+set -ex
 
 show_help() {
     echo "Usage: bash build.sh [OPTION]... -v {the_version}"
@@ -69,13 +69,17 @@ LATEST_TAG=${DOCKER_REG}/psi-anolis8:latest
 
 echo -e "Build psi binary ${GREEN}PSI ${PSI_VERSION}${NO_COLOR}..."
 
+SCRIPT_DIR="$(realpath $(dirname $0))"
+
 if [[ SKIP -eq 0 ]]; then
-    docker run -it  --rm   --mount type=bind,source="$(pwd)/../../psi",target=/home/admin/dev/src -w /home/admin/dev  --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --cap-add=NET_ADMIN --privileged=true secretflow/release-ci:1.4 /home/admin/dev/src/docker/entry.sh
+    docker run -it  --rm   --mount type=bind,source="${SCRIPT_DIR}/../",target=/home/admin/dev/src -w /home/admin/dev  --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --cap-add=NET_ADMIN --privileged=true secretflow/release-ci:latest /home/admin/dev/src/docker/entry.sh
     echo -e "Finish building psi binary ${GREEN}${IMAGE_LITE_TAG}${NO_COLOR}"
 fi
 
+cd $SCRIPT_DIR
+
 echo -e "Building docker image ${GREEN}${IMAGE_TAG}${NO_COLOR}..."
-docker build . -f Dockerfile -t ${IMAGE_TAG} --build-arg version=${VERSION} --build-arg config_templates="$(cat config_templates.yml)" --build-arg deploy_templates="$(cat deploy_templates.yml)"
+docker buildx build --platform linux/amd64 -f Dockerfile -t ${IMAGE_TAG} --build-arg version=${VERSION} --build-arg config_templates="$(cat config_templates.yml)" --build-arg deploy_templates="$(cat deploy_templates.yml)" .
 echo -e "Finish building docker image ${GREEN}${IMAGE_LITE_TAG}${NO_COLOR}"
 
 if [[ UPLOAD -eq 1 ]]; then
@@ -92,3 +96,5 @@ if [[ LATEST -eq 1 ]]; then
 fi
 
 echo ${VERSION} > version.txt
+
+cd -
diff --git a/docker/entry.sh b/docker/entry.sh
diff --git a/docs/user_guide/index.rst b/docs/user_guide/index.rst
@@ -9,5 +9,4 @@ PSI v2 is recommended to use. We are still working on PIR code refactoring.
    psi
    psi_v2
    pir
-   faq
-   psi_v2_benchmark
+   faq
-Original file line number
+Diff line change
@@ Expand Up @@
        psi
        psi_v2
        pir
-       faq
-       psi_v2_benchmark
+       faq