From b1d307f8582192e90fc4c531c4aa990ab26e127d Mon Sep 17 00:00:00 2001 From: cyjseagull Date: Tue, 10 Dec 2024 18:02:10 +0800 Subject: [PATCH] fix ecdh psi coredump (#115) * test docker.yml * update vcpkg docker * disable default features * update vcpkg-cache dockerfile * fix ecdh psi coredump --- .github/workflows/cpp_full_node_workflow.yml | 47 ++++++++++++++----- .github/workflows/docker.yml | 9 ++-- cpp/vcpkg.json | 1 - .../src/ecdh-multi-psi/EcdhMultiCache.cpp | 16 +++++-- docker-files/cpp/Dockerfile | 28 +++++++++-- docker-files/cpp/vcpkg/Dockerfile | 3 ++ docker-files/model/model/Dockerfile | 2 +- 7 files changed, 78 insertions(+), 28 deletions(-) diff --git a/.github/workflows/cpp_full_node_workflow.yml b/.github/workflows/cpp_full_node_workflow.yml index 4371131d..9800dd95 100644 --- a/.github/workflows/cpp_full_node_workflow.yml +++ b/.github/workflows/cpp_full_node_workflow.yml @@ -86,8 +86,13 @@ jobs: if: runner.os == 'macOS' run: | bash -x cpp/tools/install_depends.sh -o macos + brew install libiconv m4 automake autoconf libtool + echo "## m4:"`which m4` + export DYLD_LIBRARY_PATH=/usr/local/opt/libiconv/lib:${DYLD_LIBRARY_PATH} + export LDFLAGS="-L/usr/local/opt/libiconv/lib" + export CPPFLAGS="-I/usr/local/opt/libiconv/include" mkdir -p cpp/build && cd cpp/build - export SDKROOT=$(xcrun --sdk macosx --show-sdk-path) && cmake -DBUILD_STATIC=ON -DTESTS=ON -DCOVERAGE=ON -DCMAKE_TOOLCHAIN_FILE=${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake ../ + cmake -DBUILD_STATIC=ON -DTESTS=ON -DCOVERAGE=ON -DCMAKE_TOOLCHAIN_FILE=${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake ../ make -j3 - name: Publish Error if: always() @@ -125,9 +130,14 @@ jobs: uses: actions/upload-artifact@v4 with: name: build-x64-osx-dbg-err.log - path: /Users/runner/work/WeDPR-Component/WeDPR-Component/vcpkg/buildtrees/gsasl/build-x64-osx-dbg-err.log + path: /Users/runner/work/WeDPR-Component/WeDPR-Component/vcpkg/buildtrees/gsasl/build-x64-osx-dbg-err.log + - name: Publish Error macos + if: always() + uses: actions/upload-artifact@v4 + with: + name: autoconf-x64-osx-err.log + path: /Users/runner/work/WeDPR-Component/WeDPR-Component/vcpkg/buildtrees/gsasl/autoconf-x64-osx-err.log - build_centos: name: build_centos full node runs-on: ubuntu-latest @@ -178,7 +188,7 @@ jobs: yum update -y yum install -y epel-release centos-release-scl centos-release-scl-rh yum install -y https://packages.endpointdev.com/rhel/7/os/x86_64/endpoint-repo.x86_64.rpm - yum install -y wget java-11-openjdk-devel git autoconf automake make gcc gcc-c++ glibc-static glibc-devel cmake3 ccache devtoolset-11 libzstd-devel zlib-devel flex bison python-devel python3-devel + yum install -y libtool wget java-11-openjdk-devel git autoconf automake make gcc gcc-c++ glibc-static glibc-devel cmake3 ccache devtoolset-11 libzstd-devel zlib-devel flex bison python-devel python3-devel - uses: actions-rs/toolchain@v1 with: toolchain: nightly-2022-07-28 @@ -200,6 +210,13 @@ jobs: df /tmp du /tmp | sort -n ls -lh /tmp + - name: upgradeDepends + run: | + wget http://ftp.gnu.org/gnu/automake/automake-1.14.1.tar.gz + tar -xf automake-1.14.1.tar.gz + cd automake-1.14.1 + ./configure && make -j4 && make install + cd .. - name: DependenciesBuild run: | bash -x cpp/tools/install_depends.sh -o centos @@ -212,13 +229,13 @@ jobs: - name: FreeDiskSpace run: | df -lh - du -sch /usr/local/share/vcpkg/buildtrees/* - du -sch /usr/local/share/vcpkg/buildtrees/grpc/src/* - rm -rf /usr/local/share/vcpkg/buildtrees/grpc/src - rm -rf /usr/local/share/vcpkg/buildtrees/grpc/*-dbg - rm -rf /usr/local/share/vcpkg/buildtrees/openssl/src - rm -rf /usr/local/share/vcpkg/buildtrees/openssl/*-dbg - rm -rf /usr/local/share/vcpkg/buildtrees/*/*-dbg + du -sch /__w/WeDPR-Component/WeDPR-Component/vcpkg/buildtrees/* + du -sch /__w/WeDPR-Component/WeDPR-Component/vcpkg/buildtrees/grpc/src/* + rm -rf /__w/WeDPR-Component/WeDPR-Component/vcpkg/buildtrees/grpc/src + rm -rf /__w/WeDPR-Component/WeDPR-Component/vcpkg/buildtrees/grpc/*-dbg + rm -rf /__w/WeDPR-Component/WeDPR-Component/vcpkg/buildtrees/openssl/src + rm -rf /__w/WeDPR-Component/WeDPR-Component/vcpkg/buildtrees/openssl/*-dbg + rm -rf /__w/WeDPR-Component/WeDPR-Component/vcpkg/buildtrees/*/*-dbg df -lh - name: Compile run: | @@ -246,4 +263,10 @@ jobs: uses: actions/upload-artifact@v4 with: name: vcpkg-manifest-install.log - path: /__w/WeDPR-Component/WeDPR-Component/cpp/build/vcpkg-manifest-install.log \ No newline at end of file + path: /__w/WeDPR-Component/WeDPR-Component/cpp/build/vcpkg-manifest-install.log + - name: Publish Error macos + if: always() + uses: actions/upload-artifact@v4 + with: + name: autoconf-x64-linux-err.log + path: /__w/WeDPR-Component/WeDPR-Component/vcpkg/buildtrees/gsasl/autoconf-x64-linux-err.log \ No newline at end of file diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index b68cde3c..b15a5aa3 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -7,6 +7,8 @@ on: - "README.md" tags: - 'v3.*.*' + branches: + - feature-milestone2-gateway pull_request: paths-ignore: - "docs/**" @@ -25,12 +27,10 @@ env: ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true ACTIONS_RUNNER_FORCED_INTERNAL_NODE_VERSION: node16 ACTIONS_RUNNER_FORCE_ACTIONS_NODE_VERSION: node16 - # Use docker.io for Docker Hub if empty - REGISTRY: ghcr.io jobs: build-docker: - if: github.event_name == 'pull_request' && github.event.action == 'closed' && github.event.pull_request.merged == true + if: github.event.pull_request.merged == true || github.event_name == 'push' runs-on: ubuntu-latest steps: - name: Checkout @@ -43,7 +43,6 @@ jobs: - name: Login to Docker Hub uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3 with: - registry: ${{ env.REGISTRY }} username: ${{ secrets.DOCKER_FISCOPR_USER }} password: ${{ secrets.DOCKER_FISCOPR_TOKEN }} - name: Get git tag @@ -70,7 +69,7 @@ jobs: GW_DOCKER_TAG="fiscoorg/wedpr-gateway-service:${GW_DOCKER_TAG}" echo "* Begin to build wedpr-gateway-service docker: ${GW_DOCKER_TAG}" cd docker-files/cpp/ - docker build --build-arg SOURCE_BRANCH=${BRANCH_NAME} --target wedpr-gateway-service -t ${GW_DOCKER_TAG} . + docker build --no-cache --build-arg SOURCE_BRANCH=${BRANCH_NAME} --target wedpr-gateway-service -t ${GW_DOCKER_TAG} . echo "* Build wedpr-gateway-service docker: ${GW_DOCKER_TAG} success" docker push ${GW_DOCKER_TAG} echo "* Push wedpr-gateway-service docker: ${GW_DOCKER_TAG} success" diff --git a/cpp/vcpkg.json b/cpp/vcpkg.json index 684f225f..a1f68255 100644 --- a/cpp/vcpkg.json +++ b/cpp/vcpkg.json @@ -131,7 +131,6 @@ }, "tarscpp", "tbb", - "libiconv", "gsasl" ] }, diff --git a/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiCache.cpp b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiCache.cpp index 5e7a7168..52c4e083 100644 --- a/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiCache.cpp +++ b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiCache.cpp @@ -328,15 +328,23 @@ bool CalculatorCache::tryToFinalize() { continue; } + auto data = getPlainDataByIndex(it.second.plainDataIndex); if (it.second.plainDataIndex > 0) { - m_intersectionResult.emplace_back(getPlainDataByIndex(it.second.plainDataIndex)); + m_intersectionResult.emplace_back(data); } - // means the header field, swap with the first element if (it.second.plainDataIndex == 0) { - m_intersectionResult.emplace_back(m_intersectionResult[0]); - m_intersectionResult[0] = getPlainDataByIndex(it.second.plainDataIndex); + // means the header field, swap with the first element + if (m_intersectionResult.size() > 0) + { + m_intersectionResult.emplace_back(m_intersectionResult[0]); + m_intersectionResult[0] = data; + } + else + { + m_intersectionResult.emplace_back(data); + } } } m_cacheState = CacheState::Finalized; diff --git a/docker-files/cpp/Dockerfile b/docker-files/cpp/Dockerfile index 88b161d2..192445cc 100644 --- a/docker-files/cpp/Dockerfile +++ b/docker-files/cpp/Dockerfile @@ -1,4 +1,4 @@ -FROM fiscoorg/wedpr-component-vcpkg-cache:v3.0.0 as builder +FROM ubuntu:22.04 as builder LABEL maintainer=service@webank.com @@ -8,13 +8,31 @@ ARG SOURCE_BRANCH ENV DEBIAN_FRONTEND=noninteractive \ SOURCE=${SOURCE_BRANCH:-main} -RUN git clone https://github.com/WeBankBlockchain/WeDPR-Component --recursive --depth=1 -b ${SOURCE} \ +# Note: depends on python3; default jdk for jni +RUN apt-get -q update && apt-get install -qy --no-install-recommends \ + vim curl lcov git make nasm build-essential cmake wget libtool ca-certificates python3.11 python3-dev \ + libgmp-dev flex bison patch libzstd-dev unzip ninja-build pkg-config zip tar ccache uuid-runtime automake autoconf \ + m4 tcpdump net-tools gcc g++ default-jdk \ + && ln -fs /usr/share/zoneinfo/Asia/Shanghai /etc/localtime \ + && apt-get install -qy --no-install-recommends tzdata \ + && dpkg-reconfigure --frontend noninteractive tzdata \ + && rm -rf /var/lib/apt/lists/* + +# install rust +RUN curl https://sh.rustup.rs -sSf | bash -s -- -y + +ENV PATH="/root/.cargo/bin:${PATH}" + +ENV VCPKG_FORCE_SYSTEM_BINARIES=1 + +RUN git clone https://github.com/WeBankBlockchain/WeDPR-Component --recursive --depth=1 -b ${SOURCE} + # ln the cached vcpkg and compile -RUN cd WeDPR-Component/cpp && rm -rf vcpkg && ln -s /vcpkg \ +RUN cd WeDPR-Component/cpp \ && mkdir -p build && cd build \ - && cmake .. -DBUILD_STATIC=ON \ + && cmake .. -DCMAKE_BUILD_TYPE=Release -DBUILD_STATIC=ON \ && make -j8 - + FROM ubuntu:22.04 as wedpr-gateway-service LABEL maintainer=service@webank.com diff --git a/docker-files/cpp/vcpkg/Dockerfile b/docker-files/cpp/vcpkg/Dockerfile index 8d6a86dc..9e1d75f4 100644 --- a/docker-files/cpp/vcpkg/Dockerfile +++ b/docker-files/cpp/vcpkg/Dockerfile @@ -32,3 +32,6 @@ RUN git clone https://github.com/cyjseagull/WeDPR-Component --recursive --depth= # move the vcpkg dependecies to /vcpkg RUN mv /WeDPR-Component/cpp/vcpkg / +# remove the WeDPR-Component +RUN rm -rf /WeDPR-Component/ && rm -rf /vcpkg/buildtrees/*/src && rm -rf /vcpkg/buildtrees/*/*-dbg + diff --git a/docker-files/model/model/Dockerfile b/docker-files/model/model/Dockerfile index ea69413a..9d6a044b 100644 --- a/docker-files/model/model/Dockerfile +++ b/docker-files/model/model/Dockerfile @@ -1,4 +1,4 @@ -FROM wedpr-model-service-base-image:v3.0.0 as wedpr-model-service +FROM fiscoorg/wedpr-model-service-base-image:v3.0.0 as wedpr-model-service LABEL maintainer=service@webank.com ENV LANG zh_CN.UTF-8