repo-sync-2024-08-28T17:18:31+0800

secretflow · Aug 28, 2024 · b1b58c8 · b1b58c8
1 parent 42cdb3c
commit b1b58c8
Show file tree

Hide file tree

Showing 356 changed files with 20,302 additions and 2,958 deletions.
diff --git a/.gitignore b/.gitignore
@@ -64,4 +64,10 @@ docs/_build
 /secretpad-common/root.crt
 /secretpad-common/root.key
 /secretpad-common/root.srl
+/config/ownerId
+/config/ehcache-data/
+/config/kuscia/
+/secretpad-service/config/
+/secretpad-persistence/config/
+/secretpad-common/output.orc
 /config/ehcache-data/
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -14,6 +14,23 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 `Fixed` for any bug fixes.
 `Security` in case of vulnerabilities.
 
+## [0.10.0b0] - 2024-08-15
+### Added
+- [Feature]  Supports multi-node management in P2P deploy mode
+- [Feature]  Added a data source management module with ODPS(MaxCompute) data source
+- [Feature]  Supports self-define NUll value at data table registration
+- [Feature]  Supports model pacakge and deployment by OSS or ODPS(MaxCompute) data source
+- [Feature]  Supports XGB model serialization and deserialization
+- [Feature]  Supports output received by single side receiver in psi component
+- [Feature]  Added a express condition filter component
+- [Feature]  Added a NULL filter parameter in condition filter component
+- [Feature]  Added a new resident service named dataproxy to access different data sources
+- [Documentation] Supplemental documentation for Secretpad incompatible change (incompatible_statement_cn.md)
+
+### Changed
+- [Incompatible] Refactor VoteRequestDO  [incompatible statement](./docs/development/incompatible_statement_cn.md)
+
+
 ## [0.9.0b0] - 2024-07-15
 
 ### Added

diff --git a/README.md b/README.md
@@ -87,6 +87,11 @@ sh install.sh lite -n domainId -t token -p 30002 -m master endpoint -s 8089 -k 4
 sh install.sh p2p -n domainId  -s 8099 -g 8092 -k 8091 -p 8090 -q 33801
 ```
 
+```sh
+# deploy autonomy-node for multi-node in p2p mode
+sh install.sh autonomy-node -n domainId -g 8096 -k 8087 -p 8088 -P notls -q 13805 -m 'http://secretpad:port' -t "token for node register on pad"  -x 13086
+```
+
 For more details, see [Development](./docs/deployment_experience/v0.6.0b0/deploy_secretpad.md)
 
 ## Versions
@@ -100,15 +105,16 @@ For more details, see [Development](./docs/deployment_experience/v0.6.0b0/deploy
 > > secretflow-registry.cn-hangzhou.cr.aliyuncs.com/secretflow/secretflow-lite-anolis8<br>
 > > secretflow-registry.cn-hangzhou.cr.aliyuncs.com/secretflow/serving-anolis8<br>
 
-| secretpad Version | Kuscia Version | secretflow Version | trustedflow Version | serving Version |
-|-------------------|----------------|--------------------|---------------------|-----------------|
-| 0.9.0b0           | 0.10.0b0       | 1.8.0b0            | 0.1.1b0             | 0.5.0b0         |
-| 0.8.0b0/0.8.1b0   | 0.9.0b0        | 1.7.0b0            | 0.1.1b0             | 0.4.0b0         |
-| 0.7.1b0/0.7.2b0   | 0.8.0b0        | 1.6.1b0            | 0.1.1b0             | 0.3.1b0         |
-| 0.6.0b0           | 0.7.0b0        | 1.5.0b0            | 0.1.1b0             | 0.2.1b0         |
-| 0.5.0b0           | 0.6.0b0        | 1.4.0b0            | 0.1.1b0             | 0.2.0b0         |
-| 0.4.1b0           | 0.5.0b0        | 1.4.0.dev24011601  | 0.1.1b0             |                 |
-| 0.3.0b0           | 0.4.0b0        | 1.3.0.dev20231109  | 0.1.1b0             |                 |
+| secretpad Version | Kuscia Version | secretflow Version | trustedflow Version | serving Version | dataproxy Version |
+|-------------------|----------------|--------------------|---------------------|-----------------|-------------------|
+| 0.10.0b0          | 0.11.0b0       | 1.9.0b0            | 0.1.1b0             | 0.6.0b0         | 0.1.0b1           |
+| 0.9.0b0           | 0.10.0b0       | 1.8.0b0            | 0.1.1b0             | 0.5.0b0         |                   |
+| 0.8.0b0/0.8.1b0   | 0.9.0b0        | 1.7.0b0            | 0.1.1b0             | 0.4.0b0         |                   |
+| 0.7.1b0/0.7.2b0   | 0.8.0b0        | 1.6.1b0            | 0.1.1b0             | 0.3.1b0         |                   |
+| 0.6.0b0           | 0.7.0b0        | 1.5.0b0            | 0.1.1b0             | 0.2.1b0         |                   |
+| 0.5.0b0           | 0.6.0b0        | 1.4.0b0            | 0.1.1b0             | 0.2.0b0         |                   |
+| 0.4.1b0           | 0.5.0b0        | 1.4.0.dev24011601  | 0.1.1b0             |                 |                   |
+| 0.3.0b0           | 0.4.0b0        | 1.3.0.dev20231109  | 0.1.1b0             |                 |                   |
 
 ## Contact
 

diff --git a/README.zh-CN.md b/README.zh-CN.md
@@ -77,6 +77,11 @@ sh install.sh lite -n domainId -t token -p 30002 -m master endpoint -s 8089 -k 4
 sh install.sh p2p -n domainId  -s 8099 -g 8092 -k 8091 -p 8090 -q 33801
 ```
 
+```sh
+# 注册一个P2P节点到secretpad平台
+sh install.sh autonomy-node -n domainId -g 8096 -k 8087 -p 8088 -P notls -q 13805 -m 'http://secretpad:port' -t "token for node register on pad"  -x 13086
+```
+
 详细内容, 请查看 [Development](./docs/deployment_experience/v0.6.0b0/deploy_secretpad.md)
 
 ## 组件版本
@@ -90,15 +95,17 @@ sh install.sh p2p -n domainId  -s 8099 -g 8092 -k 8091 -p 8090 -q 33801
 > > secretflow-registry.cn-hangzhou.cr.aliyuncs.com/secretflow/secretflow-lite-anolis8<br>
 > > secretflow-registry.cn-hangzhou.cr.aliyuncs.com/secretflow/serving-anolis8<br>
 
-| secretpad Version | Kuscia Version | secretflow Version | trustedflow Version | serving Version |
-|-------------------|----------------|--------------------|---------------------|-----------------|
-| 0.9.0b0           | 0.10.0b0       | 1.8.0b0            | 0.1.1b0             | 0.5.0b0         |
-| 0.8.0b0/0.8.1b0   | 0.9.0b0        | 1.7.0b0            | 0.1.1b0             | 0.4.0b0         |
-| 0.7.1b0/0.7.2b0   | 0.8.0b0        | 1.6.1b0            | 0.1.1b0             | 0.3.1b0         |
-| 0.6.0b0           | 0.7.0b0        | 1.5.0b0            | 0.1.1b0             | 0.2.1b0         |
-| 0.5.0b0           | 0.6.0b0        | 1.4.0b0            | 0.1.1b0             | 0.2.0b0         |
-| 0.4.1b0           | 0.5.0b0        | 1.4.0.dev24011601  | 0.1.1b0             |                 |
-| 0.3.0b0           | 0.4.0b0        | 1.3.0.dev20231109  | 0.1.1b0             |                 |
+| secretpad Version | Kuscia Version | secretflow Version | trustedflow Version | serving Version | dataproxy Version |
+|-------------------|----------------|--------------------|---------------------|-----------------|-------------------|
+| 0.10.0b0          | 0.11.0b0       | 1.9.0b0            | 0.1.1b0             | 0.6.0b0         | 0.1.0b1           |
+| 0.9.0b0           | 0.10.0b0       | 1.8.0b0            | 0.1.1b0             | 0.5.0b0         |                   |
+| 0.8.0b0/0.8.1b0   | 0.9.0b0        | 1.7.0b0            | 0.1.1b0             | 0.4.0b0         |                   |
+| 0.7.1b0/0.7.2b0   | 0.8.0b0        | 1.6.1b0            | 0.1.1b0             | 0.3.1b0         |                   |
+| 0.6.0b0           | 0.7.0b0        | 1.5.0b0            | 0.1.1b0             | 0.2.1b0         |                   |
+| 0.5.0b0           | 0.6.0b0        | 1.4.0b0            | 0.1.1b0             | 0.2.0b0         |                   |
+| 0.4.1b0           | 0.5.0b0        | 1.4.0.dev24011601  | 0.1.1b0             |                 |                   |
+| 0.3.0b0           | 0.4.0b0        | 1.3.0.dev20231109  | 0.1.1b0             |                 |                   |
+
 
 ## Contact
 

diff --git a/build/Dockerfiles/anolis.Dockerfile b/build/Dockerfiles/anolis.Dockerfile
@@ -13,8 +13,8 @@ COPY config /app/config
 COPY scripts /app/scripts
 COPY demo/data /app/data
 COPY target/secretpad.jar secretpad.jar
-ENV JAVA_OPTS="-server -Xmx3100m -Xms3100m -XX:+UseZGC" SPRING_PROFILES_ACTIVE="default"
+ENV JAVA_OPTS="-server -Xms2048m -Xmx2300m -XX:MetaspaceSize=256m -XX:MaxMetaspaceSize=512m" SPRING_PROFILES_ACTIVE="default"
 EXPOSE 80
 EXPOSE 8080
 EXPOSE 9001
-ENTRYPOINT java ${JAVA_OPTS} -Dsun.net.http.allowRestrictedHeaders=true -jar -Dspring.profiles.active=${SPRING_PROFILES_ACTIVE} /app/secretpad.jar
+ENTRYPOINT java ${JAVA_OPTS} -Dsun.net.http.allowRestrictedHeaders=true  -jar -Dspring.profiles.active=${SPRING_PROFILES_ACTIVE} /app/secretpad.jar
diff --git a/config/application-p2p.yaml b/config/application-p2p.yaml
@@ -10,11 +10,15 @@ secretpad:
   node-id: ${NODE_ID}
   center-platform-service: secretpad.master.svc
   p2p-platform-service: secretpad
+  inst-name: ${INST_NAME} #  input at install.sh
   auth:
     enabled: true
   data:
     host-path: ${HOST_PATH}
     dir-path: /app/data/
+  certs:
+    dir-path: /app/config/certs/
+
   datasync:
     center: false
     p2p: true
@@ -49,6 +53,7 @@ data:
     - org.secretflow.secretpad.persistence.entity.ProjectModelPackDO
     - org.secretflow.secretpad.persistence.entity.ProjectFeatureTableDO
     - org.secretflow.secretpad.persistence.entity.ProjectGraphDomainDatasourceDO
+    - org.secretflow.secretpad.persistence.entity.ProjectInstDO
 
 # path which do not use in this mode ,config this mvc return 404 to user
 # @see DynamicBeanRegisterInit
@@ -94,3 +99,10 @@ white-list:
     - /api/v1alpha1/graph/detail
     - /api/v1alpha1/graph/node/output
     - /api/v1alpha1/graph/node/logs
+    - /api/v1alpha1/inst/get
+    - /api/v1alpha1/inst/node/list
+    - /api/v1alpha1/inst/node/add
+    - /api/v1alpha1/inst/node/token
+    - /api/v1alpha1/inst/node/newToken
+    - /api/v1alpha1/inst/node/delete
+    - /api/v1alpha1/inst/node/register
diff --git a/config/application-test.yaml b/config/application-test.yaml
@@ -18,6 +18,8 @@ secretpad:
     max-request-size: -1 # -1 means not limit, e.g.  200MB, 1GB
   data:
     dir-path: ../tmp/data/
+  certs:
+    dir-path: ../tmp/certs/
   datasync:
     center: true
     p2p: false

diff --git a/config/application.yaml b/config/application.yaml
@@ -4,6 +4,10 @@ server:
       enabled: true
       directory: /var/log/secretpad
   servlet:
+    encoding:
+      charset: UTF-8
+      enabled: true
+      force: true
     session:
       timeout: 30m
   http-port: 8080
@@ -18,10 +22,9 @@ server:
     key-store-type: JKS
   compression:
     enabled: true
-    mime-types:
-      - application/javascript
-      - text/css
+    mime-types: text/html,text/xml,text/plain,text/css,application/javascript,application/json
     min-response-size: 1024
+
 spring:
   task:
     scheduling:
@@ -88,11 +91,21 @@ springdoc:
   api-docs:
     enabled: false
 management:
+  endpoint:
+    shutdown:
+      enabled: false
+    health:
+      show-details: always
   endpoints:
     web:
       exposure:
-        include: health,info,readiness,prometheus
-    enabled-by-default: false
+        include:
+          - prometheus
+          - health
+  metrics:
+    tags:
+      application: ${spring.application.name}
+
 kusciaapi:
   protocol: ${KUSCIA_PROTOCOL:tls}
 
@@ -160,6 +173,7 @@ secretpad:
     tee-app-image: ${TEE_APP_IMAGE:0.1.0b0}
     tee-dm-image: ${TEE_DM_IMAGE:0.1.0b0}
     capsule-manager-sim-image: ${CAPSULE_MANAGER_SIM_IMAGE:0.1.2b0}
+    data-proxy-image: ${DATA_PROXY_IMAGE:0.1.0b0}
 
   component:
     hide:
@@ -169,6 +183,8 @@ secretpad:
       - secretflow/model/model_export:0.0.1
       - secretflow/ml.train/slnn_train:0.0.1
       - secretflow/ml.predict/slnn_predict:0.0.2
+  data-proxy:
+    enabled: ${DATAPROXY_ENABLE:true}
 
 sfclusterDesc:
   deviceConfig: