diff --git a/.github/workflows/cpp_full_node_workflow.yml b/.github/workflows/cpp_full_node_workflow.yml index d8f263e0..4371131d 100644 --- a/.github/workflows/cpp_full_node_workflow.yml +++ b/.github/workflows/cpp_full_node_workflow.yml @@ -69,13 +69,25 @@ jobs: export CXX='g++-10' sudo bash -x cpp/tools/install_depends.sh -o ubuntu mkdir -p cpp/build && cd cpp/build && cmake -DBUILD_STATIC=ON -DTESTS=ON -DCOVERAGE=ON -DCMAKE_TOOLCHAIN_FILE=${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake ../ + echo "Disk usage before delete files: "`df -lh` + du -sch ${{ env.VCPKG_ROOT }}/buildtrees/* + du -sch ${{ env.VCPKG_ROOT }}/buildtrees/grpc/src/* + rm -rf ${{ env.VCPKG_ROOT }}/buildtrees/grpc/src + rm -rf ${{ env.VCPKG_ROOT }}/buildtrees/grpc/*-dbg + rm -rf ${{ env.VCPKG_ROOT }}/buildtrees/krb5/src + rm -rf ${{ env.VCPKG_ROOT }}/buildtrees/gsasl/src + rm -rf ${{ env.VCPKG_ROOT }}/buildtrees/protobuf/src + rm -rf ${{ env.VCPKG_ROOT }}/buildtrees/wedprcrypto/src + rm -rf ${{ env.VCPKG_ROOT }}buildtrees/openssl/src + rm -rf ${{ env.VCPKG_ROOT }}/buildtrees/*/*-dbg + echo "Disk usage after delete files: "`df -lh` make -j3 - name: Build for macOS if: runner.os == 'macOS' run: | bash -x cpp/tools/install_depends.sh -o macos - rm -rf /Users/runner/work/WeDPR-Component/WeDPR-Component/vcpkg/buildtrees/gsasl - mkdir -p cpp/build && cd cpp/build && cmake -DBUILD_STATIC=ON -DTESTS=ON -DCOVERAGE=ON -DCMAKE_TOOLCHAIN_FILE=${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake ../ + mkdir -p cpp/build && cd cpp/build + export SDKROOT=$(xcrun --sdk macosx --show-sdk-path) && cmake -DBUILD_STATIC=ON -DTESTS=ON -DCOVERAGE=ON -DCMAKE_TOOLCHAIN_FILE=${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake ../ make -j3 - name: Publish Error if: always() @@ -108,7 +120,13 @@ jobs: with: name: build-x64-linux-dbg-err.log path: /home/runner/work/WeDPR-Component/WeDPR-Component/vcpkg/buildtrees/gsasl/build-x64-linux-dbg-err.log - + - name: Publish Error macos + if: always() + uses: actions/upload-artifact@v4 + with: + name: build-x64-osx-dbg-err.log + path: /Users/runner/work/WeDPR-Component/WeDPR-Component/vcpkg/buildtrees/gsasl/build-x64-osx-dbg-err.log + build_centos: name: build_centos full node @@ -160,7 +178,7 @@ jobs: yum update -y yum install -y epel-release centos-release-scl centos-release-scl-rh yum install -y https://packages.endpointdev.com/rhel/7/os/x86_64/endpoint-repo.x86_64.rpm - yum install -y wget java-11-openjdk-devel git make gcc gcc-c++ glibc-static glibc-devel cmake3 ccache devtoolset-11 libzstd-devel zlib-devel flex bison python-devel python3-devel + yum install -y wget java-11-openjdk-devel git autoconf automake make gcc gcc-c++ glibc-static glibc-devel cmake3 ccache devtoolset-11 libzstd-devel zlib-devel flex bison python-devel python3-devel - uses: actions-rs/toolchain@v1 with: toolchain: nightly-2022-07-28 @@ -190,7 +208,7 @@ jobs: rm -rf python mkdir -p cpp/build cd cpp/build - cmake3 -DCMAKE_TOOLCHAIN_FILE=${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake -DBUILD_STATIC=ON -DCMAKE_BUILD_TYPE=Release -DTESTS=ON -DCMAKE_TOOLCHAIN_FILE=/usr/local/share/vcpkg/scripts/buildsystems/vcpkg.cmake ../ + cmake3 -DCMAKE_TOOLCHAIN_FILE=${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake -DBUILD_STATIC=ON -DCMAKE_BUILD_TYPE=Release -DTESTS=ON ../ - name: FreeDiskSpace run: | df -lh @@ -223,3 +241,9 @@ jobs: with: name: ppc-gateway-service-x64 path: ./cpp/build/bin/ppc-gateway-service + - name: Publish Error macos + if: always() + uses: actions/upload-artifact@v4 + with: + name: vcpkg-manifest-install.log + path: /__w/WeDPR-Component/WeDPR-Component/cpp/build/vcpkg-manifest-install.log \ No newline at end of file diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 00000000..b68cde3c --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,98 @@ +name: WeDPR-Component docker ci +on: + push: + paths-ignore: + - "docs/**" + - "Changelog.md" + - "README.md" + tags: + - 'v3.*.*' + pull_request: + paths-ignore: + - "docs/**" + - "python/**" + - "Changelog.md" + - "README.md" + release: + types: [prereleased] + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +env: + RUST_BACKTRACE: 1 + ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true + ACTIONS_RUNNER_FORCED_INTERNAL_NODE_VERSION: node16 + ACTIONS_RUNNER_FORCE_ACTIONS_NODE_VERSION: node16 + # Use docker.io for Docker Hub if empty + REGISTRY: ghcr.io + +jobs: + build-docker: + if: github.event_name == 'pull_request' && github.event.action == 'closed' && github.event.pull_request.merged == true + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Get branch name + uses: nelonoel/branch-name@v1.0.1 + - name: Fetch tag + run: | + git fetch --tags --force + - name: Login to Docker Hub + uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ secrets.DOCKER_FISCOPR_USER }} + password: ${{ secrets.DOCKER_FISCOPR_TOKEN }} + - name: Get git tag + uses: little-core-labs/get-git-tag@v3.0.1 + id: tag_data + with: + tagRegex: (.*) # Optional. Returns specified group text as tag name. Full tag string is returned if regex is not defined. + tagRegexGroup: 1 # Optional. Default is 1. + # the model image + - name: Build and Push model images + run: | + [[ ${{github.ref}} == */tags/* ]] && MODEL_DOCKER_TAG="${GIT_TAG_NAME}" || MODEL_DOCKER_TAG="${BRANCH_NAME}" + MODEL_DOCKER_TAG="fiscoorg/wedpr-model-service:${MODEL_DOCKER_TAG}" + echo "* Begin to build model docker: ${MODEL_DOCKER_TAG}" + cd docker-files/model/model + docker build --build-arg SOURCE_BRANCH=${BRANCH_NAME} -t ${MODEL_DOCKER_TAG} . + echo "* Build model docker: ${MODEL_DOCKER_TAG} success" + docker push ${MODEL_DOCKER_TAG} + echo "* Push model docker: ${MODEL_DOCKER_TAG} success" + # the wedpr-gateway-service image + - name: Build and Push wedpr-gateway-service images + run: | + [[ ${{github.ref}} == */tags/* ]] && GW_DOCKER_TAG="${GIT_TAG_NAME}" || GW_DOCKER_TAG="${BRANCH_NAME}" + GW_DOCKER_TAG="fiscoorg/wedpr-gateway-service:${GW_DOCKER_TAG}" + echo "* Begin to build wedpr-gateway-service docker: ${GW_DOCKER_TAG}" + cd docker-files/cpp/ + docker build --build-arg SOURCE_BRANCH=${BRANCH_NAME} --target wedpr-gateway-service -t ${GW_DOCKER_TAG} . + echo "* Build wedpr-gateway-service docker: ${GW_DOCKER_TAG} success" + docker push ${GW_DOCKER_TAG} + echo "* Push wedpr-gateway-service docker: ${GW_DOCKER_TAG} success" + # the wedpr-pro-node-service image + - name: Build and Push wedpr-pro-node-service images + run: | + [[ ${{github.ref}} == */tags/* ]] && NODE_DOCKER_TAG="${GIT_TAG_NAME}" || NODE_DOCKER_TAG="${BRANCH_NAME}" + NODE_DOCKER_TAG="fiscoorg/wedpr-pro-node-service:${NODE_DOCKER_TAG}" + echo "* Begin to build wedpr-pro-node-service docker: ${NODE_DOCKER_TAG}" + cd docker-files/cpp/ + docker build --build-arg SOURCE_BRANCH=${BRANCH_NAME} --target wedpr-pro-node-service -t ${NODE_DOCKER_TAG} . + echo "* Build wedpr-pro-node-service docker: ${NODE_DOCKER_TAG} success" + docker push ${NODE_DOCKER_TAG} + echo "* Push wedpr-pro-node-service docker: ${NODE_DOCKER_TAG} success" + # the wedpr-mpc-service image + - name: Build and Push wedpr-mpc-service images + run: | + [[ ${{github.ref}} == */tags/* ]] && MPC_DOCKER_TAG="${GIT_TAG_NAME}" || MPC_DOCKER_TAG="${BRANCH_NAME}" + MPC_DOCKER_TAG="fiscoorg/wedpr-mpc-service:${MPC_DOCKER_TAG}" + echo "* Begin to build wedpr-mpc-service docker: ${MPC_DOCKER_TAG}" + cd docker-files/cpp/ + docker build --build-arg SOURCE_BRANCH=${BRANCH_NAME} --target wedpr-mpc-service -t ${MPC_DOCKER_TAG} . + echo "* Build wedpr-mpc-service docker: ${MPC_DOCKER_TAG} success" + docker push ${MPC_DOCKER_TAG} + echo "* Push wedpr-mpc-service docker: ${MPC_DOCKER_TAG} success" \ No newline at end of file diff --git a/.github/workflows/model_docker_image_base.yml b/.github/workflows/model_docker_image_base.yml new file mode 100644 index 00000000..0af4ab5a --- /dev/null +++ b/.github/workflows/model_docker_image_base.yml @@ -0,0 +1,40 @@ +name: Manual trigger wedpr-model-service-base-image docker image build + +on: + workflow_dispatch: + inputs: + logLevel: + description: 'Log level' + required: true + default: 'warning' + tags: + description: 'the input tag' + +jobs: + manual-build-model-base-docker: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Get branch name + uses: nelonoel/branch-name@v1.0.1 + - name: Fetch tag + run: | + git fetch --tags --force + - name: Login to Docker Hub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_FISCOPR_USER }} + password: ${{ secrets.DOCKER_FISCOPR_TOKEN }} + + # the model image + - name: Build and Push wedpr-model-service-base-image images + run: | + echo "* Manual build wedpr-model-service-base-image image, docker version: ${{ github.event.inputs.tags }}" + DOCKER_TAG="fiscoorg/wedpr-model-service-base-image:${{ github.event.inputs.tags }}" + echo "* Begin to build ${DOCKER_TAG}" + cd docker-files/model/base + docker build -t ${DOCKER_TAG} . + echo "* Build ${DOCKER_TAG} success" + docker push ${DOCKER_TAG} + echo "* Push ${DOCKER_TAG} success" \ No newline at end of file diff --git a/.github/workflows/vcpkg_docker_image.yml b/.github/workflows/vcpkg_docker_image.yml new file mode 100644 index 00000000..33c03299 --- /dev/null +++ b/.github/workflows/vcpkg_docker_image.yml @@ -0,0 +1,46 @@ +name: Manual trigger vcpkg-cache docker image build + +on: + workflow_dispatch: + inputs: + logLevel: + description: 'Log level' + required: true + default: 'warning' + tags: + description: 'the input tag' + +jobs: + manual-build-vcpkg-cache-docker: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Get branch name + uses: nelonoel/branch-name@v1.0.1 + - name: Fetch tag + run: | + git fetch --tags --force + - name: Login to Docker Hub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_FISCOPR_USER }} + password: ${{ secrets.DOCKER_FISCOPR_TOKEN }} + + # the model image + - name: Build and Push wedpr-component-vcpkg-cache images + run: | + echo "* Manual build wedpr-component-vcpkg-cache image, docker version: ${{ github.event.inputs.tags }}" + DOCKER_TAG="fiscoorg/wedpr-component-vcpkg-cache:${{ github.event.inputs.tags }}" + echo "* Begin to build ${DOCKER_TAG}" + cd docker-files/cpp/vcpkg + docker build --build-arg SOURCE_BRANCH=${BRANCH_NAME} -t ${DOCKER_TAG} . + echo "* Build ${DOCKER_TAG} success" + docker push ${DOCKER_TAG} + echo "* Push ${DOCKER_TAG} success" + - name: Publish Error + if: always() + uses: actions/upload-artifact@v4 + with: + name: autoconf-x64-linux-err.log + path: /WeDPR-Component/cpp/vcpkg/buildtrees/gsasl/autoconf-x64-linux-err.log \ No newline at end of file diff --git a/ChangeLog.md b/ChangeLog.md index 8b7b54d4..67c02c6f 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -1,21 +1,37 @@ -### 1.0.0-rc1 -(2024-08-21) +# 3.0.0 +(2024-12-9) -**新增** +### 新增 -#### 隐私计算组件 +**隐私计算组件** -- **PSI**: 实现多种类型隐私求交集算法,包括CM2020, RA2018, ECDH-PSI, ECDH-Multi-PSI等 -- **MPC**: 安全多方计算相关组件 -- **MPCSQL**: 基于安全多方计算协议,支持联合分析查询任务 -- **PIR**: 匿踪查询组件 +- PSI: 实现多种类型隐私求交集算法,包括CM2020, RA2018, ECDH-PSI, ECDH-Multi-PSI等 +- MPC: 安全多方计算 +- PIR: 匿踪查询核心组件 -#### 隐私计算互联互通 +**联合建模组件** -- ECDH PSI算法与隐语互联互通 +- 2+方的多方联合XGB/LR建模组件(训练 + 离线预测) +- 2+方多方特征工程组件(特征分箱,WOE/IV计算等) +- 多方数据预处理组件 -#### 隐私建模组件 +**统一网关** + +- 支持基于最短路径的消息路由转发 +- 支持按节点ID、服务名、机构名进行路由寻址 +- 支持服务注册和服务发现 + +**统一网关SDK** + +- 提供Java/Python网关SDK,支持接入网关与其他节点、服务或者机构进行通信 +- 可向网关注册服务 +- 可从网关拉取服务信息 + +**专家模式SDK** + +- `wedpr-ml-toolkit`: 可通过python sdk向WeDPR隐私计算平台发起各类任务,并获取对应的结果,扩展了WeDPR隐私计算系统功能的可扩展性,可在不侵入式修改系统的前提下,实现数据开发人员的各类数据分析需求 + +**密码学组件** + +- 实现了各种同态算法,包括`Paillier`, `IHC`等 -- 2+方的多方联合XGB组件(训练 + 离线预测) -- 2+方多方特征工程组件(特征分箱,WOE/IV计算等) -- 预处理组件 \ No newline at end of file diff --git a/cpp/cmake/Version.cmake b/cpp/cmake/Version.cmake index 206620b2..8259d91c 100644 --- a/cpp/cmake/Version.cmake +++ b/cpp/cmake/Version.cmake @@ -1,2 +1,2 @@ -set(VERSION "1.0.0") -set(PYTHON_TOOLKIT_VERSION "1.0.0") +set(VERSION "3.0.0") +set(PYTHON_TOOLKIT_VERSION "3.0.0") diff --git a/cpp/tools/install_depends.sh b/cpp/tools/install_depends.sh index 1e2a1e6c..28f3f682 100644 --- a/cpp/tools/install_depends.sh +++ b/cpp/tools/install_depends.sh @@ -125,7 +125,7 @@ install_iconv_depend() install_macos_depends() { LOG_INFO "install depends for macos ..." - brew install autoconf nasm lcov + brew install autoconf nasm lcov automake #install_gsasl_depend "macos" #install_iconv_depend LOG_INFO "install depends for macos success ..." diff --git a/cpp/vcpkg-configuration.json b/cpp/vcpkg-configuration.json index 3c205926..e62c646e 100644 --- a/cpp/vcpkg-configuration.json +++ b/cpp/vcpkg-configuration.json @@ -3,7 +3,7 @@ { "kind": "git", "repository": "https://github.com/FISCO-BCOS/registry", - "baseline": "535c4fc74badd509b94e8a736452fa8a76bff944", + "baseline": "d45a1d99c6b01713252bcb734c03f2c1ac0e44b8", "packages": [ "openssl", "bcos-utilities", diff --git a/cpp/vcpkg.json b/cpp/vcpkg.json index a3a838c8..684f225f 100644 --- a/cpp/vcpkg.json +++ b/cpp/vcpkg.json @@ -51,6 +51,14 @@ { "name": "jsoncpp", "version": "1.9.5" + }, + { + "name":"gsasl", + "version": "2.2.1-no-gssapi#0" + }, + { + "name": "libxml2", + "version": "2.11.5#1" } ], "features": { @@ -116,10 +124,15 @@ "name": "bcos-boostssl", "version>=": "3.2.3" }, + { + "name": "libxml2", + "version>=": "2.11.5#1", + "features": ["lzma", "zlib"] + }, "tarscpp", "tbb", - "libxml2", - "libiconv" + "libiconv", + "gsasl" ] }, "sdk": { diff --git a/docker-files/cpp/Dockerfile b/docker-files/cpp/Dockerfile index d14397eb..88b161d2 100644 --- a/docker-files/cpp/Dockerfile +++ b/docker-files/cpp/Dockerfile @@ -1,42 +1,28 @@ -FROM ubuntu:18.04 as builder +FROM fiscoorg/wedpr-component-vcpkg-cache:v3.0.0 as builder -LABEL maintainer service@webank.com +LABEL maintainer=service@webank.com WORKDIR / ARG SOURCE_BRANCH ENV DEBIAN_FRONTEND=noninteractive \ - SOURCE=${SOURCE_BRANCH:-master} + SOURCE=${SOURCE_BRANCH:-main} -RUN apt-get -q update && apt-get install -qy --no-install-recommends \ - vim curl git make build-essential cmake \ - libgmp-dev flex bison patch libzstd-dev unzip ninja-build pkg-config curl zip tar ccache uuid-runtime automake autoconf \ - m4 tcpdump net-tools libkrb5-dev krb5-user pkg-config default-libmysqlclient-dev gcc g++ \ - && ln -fs /usr/share/zoneinfo/Asia/Shanghai /etc/localtime \ - && apt-get install -qy --no-install-recommends tzdata \ - && dpkg-reconfigure --frontend noninteractive tzdata \ - && rm -rf /var/lib/apt/lists/* +RUN git clone https://github.com/WeBankBlockchain/WeDPR-Component --recursive --depth=1 -b ${SOURCE} \ +# ln the cached vcpkg and compile +RUN cd WeDPR-Component/cpp && rm -rf vcpkg && ln -s /vcpkg \ + && mkdir -p build && cd build \ + && cmake .. -DBUILD_STATIC=ON \ + && make -j8 -# install rust -RUN curl https://sh.rustup.rs -sSf | bash -s -- -y - -ENV PATH="/root/.cargo/bin:${PATH}" - -ENV VCPKG_FORCE_SYSTEM_BINARIES=1 - -RUN git clone https://github.com/WeBankBlockchain/WeDPR-Component.git --recursive --depth=1 -b ${SOURCE} \ - && mkdir -p WeDPR-Component/cpp/build && cd WeDPR-Component/cpp/build \ - && cmake .. -DBUILD_STATIC=ON && make -j8 && cat /WeDPR-Component/cpp/build/*.log - - -FROM ubuntu:18.04 as wedpr-gateway-service -LABEL maintainer service@webank.com +FROM ubuntu:22.04 as wedpr-gateway-service +LABEL maintainer=service@webank.com RUN apt-get -q update && apt-get install -qy --no-install-recommends vim \ && ln -fs /usr/share/zoneinfo/Asia/Shanghai /etc/localtime \ && apt-get install -qy --no-install-recommends tzdata \ && dpkg-reconfigure --frontend noninteractive tzdata \ - && rm -rf /var/lib/apt/lists/* \ + && rm -rf /var/lib/apt/lists/* RUN mkdir -p /data/home/wedpr/wedpr-gateway-service @@ -44,15 +30,14 @@ COPY --from=builder /WeDPR-Component/cpp/build/bin/ppc-gateway-service /data/hom ENTRYPOINT ["/data/home/wedpr/wedpr-gateway-service/ppc-gateway-service", "-c", "config.ini"] - -FROM ubuntu:18.04 as wedpr-pro-node-service -LABEL maintainer service@webank.com +FROM ubuntu:22.04 as wedpr-pro-node-service +LABEL maintainer=service@webank.com RUN apt-get -q update && apt-get install -qy --no-install-recommends vim libkrb5-dev krb5-user \ && ln -fs /usr/share/zoneinfo/Asia/Shanghai /etc/localtime \ && apt-get install -qy --no-install-recommends tzdata \ && dpkg-reconfigure --frontend noninteractive tzdata \ - && rm -rf /var/lib/apt/lists/* \ + && rm -rf /var/lib/apt/lists/* RUN mkdir -p /data/home/wedpr/wedpr-pro-node-service @@ -60,14 +45,14 @@ COPY --from=builder /WeDPR-Component/cpp/build/bin/ppc-pro-node /data/home/wedpr ENTRYPOINT ["/data/home/wedpr/wedpr-pro-node-service/ppc-pro-node", "-c", "config.ini"] -FROM ubuntu:18.04 as wedpr-mpc-service -LABEL maintainer service@webank.com +FROM ubuntu:22.04 as wedpr-mpc-service +LABEL maintainer=service@webank.com RUN apt-get -q update && apt-get install -qy --no-install-recommends vim libkrb5-dev krb5-user \ && ln -fs /usr/share/zoneinfo/Asia/Shanghai /etc/localtime \ && apt-get install -qy --no-install-recommends tzdata \ && dpkg-reconfigure --frontend noninteractive tzdata \ - && rm -rf /var/lib/apt/lists/* \ + && rm -rf /var/lib/apt/lists/* RUN mkdir -p /data/home/wedpr/wedpr-mpc-service diff --git a/docker-files/cpp/build.sh b/docker-files/cpp/build.sh new file mode 100644 index 00000000..0e13dd39 --- /dev/null +++ b/docker-files/cpp/build.sh @@ -0,0 +1,16 @@ +#!/bin/bash +echo "========= BUILD IMAGES for WEDPR-COMPONENT, BRANCH: ${1} =========" + +echo "* build wedpr-gateway-service image" +docker build --target wedpr-gateway-service --build-arg SOURCE_BRANCH=${1} -t wedpr-gateway-service . +echo "* build wedpr-gateway-service image success" + +echo "* build wedpr-pro-node-service image" +docker build --target wedpr-pro-node-service --build-arg SOURCE_BRANCH=${1} -t wedpr-pro-node-service . +echo "* build wedpr-pro-node-service image success" + +echo "* build wedpr-mpc-service image" +docker build --target wedpr-mpc-service --build-arg SOURCE_BRANCH=${1} -t wedpr-mpc-service . +echo "* build wedpr-mpc-service image success" + +echo "========= BUILD IMAGES for WEDPR-COMPONENT, BRANCH: ${1} =========" \ No newline at end of file diff --git a/docker-files/cpp/vcpkg/Dockerfile b/docker-files/cpp/vcpkg/Dockerfile new file mode 100644 index 00000000..8d6a86dc --- /dev/null +++ b/docker-files/cpp/vcpkg/Dockerfile @@ -0,0 +1,34 @@ +FROM ubuntu:22.04 as wedpr-component-vcpkg-cache + +LABEL maintainer=service@webank.com + +WORKDIR / + +ARG SOURCE_BRANCH +ENV DEBIAN_FRONTEND=noninteractive \ + SOURCE=${SOURCE_BRANCH:-main} +# Note: depends on python3; default jdk for jni +RUN apt-get -q update && apt-get install -qy --no-install-recommends \ + vim curl lcov git make nasm build-essential cmake wget libtool ca-certificates python3.11 python3-dev \ + libgmp-dev flex bison patch libzstd-dev unzip ninja-build pkg-config zip tar ccache uuid-runtime automake autoconf \ + m4 tcpdump net-tools gcc g++ default-jdk \ + && ln -fs /usr/share/zoneinfo/Asia/Shanghai /etc/localtime \ + && apt-get install -qy --no-install-recommends tzdata \ + && dpkg-reconfigure --frontend noninteractive tzdata \ + && rm -rf /var/lib/apt/lists/* + +# install rust +RUN curl https://sh.rustup.rs -sSf | bash -s -- -y + +ENV PATH="/root/.cargo/bin:${PATH}" + +ENV VCPKG_FORCE_SYSTEM_BINARIES=1 + +# first download and compile the vcpkg dependecies +RUN git clone https://github.com/cyjseagull/WeDPR-Component --recursive --depth=1 -b ${SOURCE} \ + && mkdir -p WeDPR-Component/cpp/build && cd WeDPR-Component/cpp/build \ + && cmake .. -DBUILD_STATIC=ON + +# move the vcpkg dependecies to /vcpkg +RUN mv /WeDPR-Component/cpp/vcpkg / + diff --git a/docker-files/cpp/vcpkg/build.sh b/docker-files/cpp/vcpkg/build.sh new file mode 100644 index 00000000..56ed1efa --- /dev/null +++ b/docker-files/cpp/vcpkg/build.sh @@ -0,0 +1,4 @@ +#!/bin/bash +echo "* build image: wedpr-component-vcpkg-cache, branch: ${1}" +docker build --build-arg SOURCE_BRANCH=${1} -t wedpr-component-vcpkg-cache . +echo "* build image: wedpr-component-vcpkg-cache success, branch: ${1}" diff --git a/docker-files/model/Dockerfile b/docker-files/model/Dockerfile deleted file mode 100644 index 7c57b93b..00000000 --- a/docker-files/model/Dockerfile +++ /dev/null @@ -1,33 +0,0 @@ -FROM wedpr-image:latest as wedpr-model-service - -LABEL maintainer service@webank.com - -ENV LANG zh_CN.UTF-8 -ENV LANGUAGE zh_CN.UTF-8 -ENV LC_ALL zh_CN.UTF-8 - -WORKDIR /data/home/wedpr -ARG SOURCE_BRANCH -ENV DEBIAN_FRONTEND=noninteractive \ - SOURCE=${SOURCE_BRANCH:-master} - -RUN mkdir -p /data/home/wedpr -# copy requirements -COPY depends/requirements.txt /data/home/wedpr/requirements.txt - -# install the requirements -RUN pip install --no-cache-dir -i https://pypi.mirrors.ustc.edu.cn/simple/ -r /data/home/wedpr/requirements.txt - -# obtain the source -RUN git clone https://github.com/WeBankBlockchain/WeDPR-Component.git --recursive --depth=1 -b ${SOURCE} -# move the files to the /data/app path -RUN mkdir -p /data/home/wedpr/wedpr-model/ \ - && mv /data/home/wedpr/WeDPR-Component/python/ppc_common /data/home/wedpr/wedpr-model/ppc-common \ - && mv /data/home/wedpr/WeDPR-Component/python/ppc_model /data/home/wedpr/wedpr-model/ppc-model \ - && mv /data/home/wedpr/WeDPR-Component/python/aes_key.bin /data/home/wedpr/wedpr-model/ppc-model \ - && cp /data/home/wedpr/model/ppc-model/tools/*.sh /data/home/wedpr/wedpr-model/ppc-model - -# clear the WeDPR-Component -RUN rm -rf /data/home/wedpr/WeDPR-Component - -ENTRYPOINT ["/bin/bash", "/data/home/wedpr/wedpr-model/start.sh", "true"] diff --git a/docker-files/model/base/Dockerfile b/docker-files/model/base/Dockerfile new file mode 100644 index 00000000..05c5a433 --- /dev/null +++ b/docker-files/model/base/Dockerfile @@ -0,0 +1,11 @@ +FROM fiscoorg/wedpr-image:v3.0.0 as wedpr-model-service-base-image + +LABEL maintainer=service@webank.com + +RUN apt-get install -qy --no-install-recommends pkg-config default-libmysqlclient-dev +RUN mkdir -p /data/home/wedpr +# copy requirements +COPY depends/requirements.txt /data/home/wedpr/requirements.txt + +# install the requirements +RUN pip install -i https://pypi.mirrors.ustc.edu.cn/simple/ -r /data/home/wedpr/requirements.txt diff --git a/docker-files/model/base/build.sh b/docker-files/model/base/build.sh new file mode 100644 index 00000000..55788924 --- /dev/null +++ b/docker-files/model/base/build.sh @@ -0,0 +1,4 @@ +#!/bin/bash +echo "* build image: wedpr-model-service-base-image" +docker build -t wedpr-model-service-base-image . +echo "* build image: wedpr-model-service-base-image success" \ No newline at end of file diff --git a/docker-files/model/depends/requirements.txt b/docker-files/model/base/depends/requirements.txt similarity index 100% rename from docker-files/model/depends/requirements.txt rename to docker-files/model/base/depends/requirements.txt diff --git a/docker-files/model/model/Dockerfile b/docker-files/model/model/Dockerfile new file mode 100644 index 00000000..ea69413a --- /dev/null +++ b/docker-files/model/model/Dockerfile @@ -0,0 +1,29 @@ +FROM wedpr-model-service-base-image:v3.0.0 as wedpr-model-service +LABEL maintainer=service@webank.com + +ENV LANG zh_CN.UTF-8 +ENV LANGUAGE zh_CN.UTF-8 +ENV LC_ALL zh_CN.UTF-8 + +WORKDIR /data/home/wedpr +ARG SOURCE_BRANCH +ENV DEBIAN_FRONTEND=noninteractive \ + SOURCE=${SOURCE_BRANCH:-main} + +# install the gateway sdk +# TODO: replace with pypip +RUN pip install -i https://test.pypi.org/simple/ wedpr-python-gateway-sdk + +# obtain the source, no need to fetch the submodule(vcpkg) here +RUN git clone https://github.com/WeBankBlockchain/WeDPR-Component.git -b ${SOURCE} +# move the files to the /data/app path +RUN mkdir -p /data/home/wedpr/wedpr-model/ \ + && mv /data/home/wedpr/WeDPR-Component/python/ppc_common /data/home/wedpr/wedpr-model/ppc_common \ + && mv /data/home/wedpr/WeDPR-Component/python/ppc_model /data/home/wedpr/wedpr-model/ppc_model \ + && mv /data/home/wedpr/WeDPR-Component/python/aes_key.bin /data/home/wedpr/wedpr-model/ppc_model \ + && cp /data/home/wedpr/wedpr-model/ppc_model/tools/*.sh /data/home/wedpr/wedpr-model/ppc_model + +# clear the WeDPR-Component +RUN rm -rf /data/home/wedpr/WeDPR-Component + +ENTRYPOINT ["/bin/bash", "/data/home/wedpr/wedpr-model/start.sh", "true"] diff --git a/docker-files/model/model/build.sh b/docker-files/model/model/build.sh new file mode 100644 index 00000000..51ffda7a --- /dev/null +++ b/docker-files/model/model/build.sh @@ -0,0 +1,4 @@ +#!/bin/bash +echo "* build image: wedpr-model-service, branch: ${1}" +docker build --build-arg SOURCE_BRANCH=${1} -t wedpr-model-service . +echo "* build image: wedpr-model-service success, branch: ${1}" \ No newline at end of file