repo-sync-2024-10-22T17:03:47+0800

.gitignore

@@ -71,3 +71,10 @@ docs/_build
 /secretpad-persistence/config/
 /secretpad-common/output.orc
 /config/ehcache-data/
+/config/kuscia/
+/secretpad-service/config/
+/secretpad-persistence/config/
+/secretpad-common/output.orc
+/config/ehcache-data/
+/db/secretpadQuartz.mv.db.mv.db
+/.cloudide/

CHANGELOG.md

@@ -14,8 +14,25 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 `Fixed` for any bug fixes.
 `Security` in case of vulnerabilities.
 
+## [0.11.0b0] - 2024-10-15
+
+### Added
+
+- [Feature]  Data Source Support for MySQL Database (Structured data tables only).
+- [Feature]  io_read_data/io_write_data Support for sgb, glm, New Read Model, and Model Backtesting Component.
+- [Feature]  Support for Reading ODPS Data Table Partitions.
+- [Feature]  Support for Scheduled Periodic Tasks: Including configuration, task list, and result viewing.
+- [Feature]  New Prometheus Monitoring Metrics.
+- [Feature]  Specify Field Types When Granting Sample Tables to Projects.
+
+### Changed
+
+- [Compatible] Partial Code Refactoring Optimization.
+
 ## [0.10.0b0] - 2024-08-15
+
 ### Added
+
 - [Feature]  Supports multi-node management in P2P deploy mode
 - [Feature]  Added a data source management module with ODPS(MaxCompute) data source
 - [Feature]  Supports self-define NUll value at data table registration
@@ -28,8 +45,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - [Documentation] Supplemental documentation for Secretpad incompatible change (incompatible_statement_cn.md)
 
 ### Changed
-- [Incompatible] Refactor VoteRequestDO  [incompatible statement](./docs/development/incompatible_statement_cn.md)
 
+- [Incompatible] Refactor VoteRequestDO  [incompatible statement](./docs/development/incompatible_statement_cn.md)
 
 ## [0.9.0b0] - 2024-07-15
 

README.md

@@ -107,6 +107,7 @@ For more details, see [Development](./docs/deployment_experience/v0.6.0b0/deploy
 
 | secretpad Version | Kuscia Version | secretflow Version | trustedflow Version | serving Version | dataproxy Version |
 |-------------------|----------------|--------------------|---------------------|-----------------|-------------------|
+| 0.11.0b0          | 0.12.0b0       | 1.10.0b0           | 0.1.1b0             | 0.7.0b0         | 0.2.0b0           |
 | 0.10.0b0/0.10.1b0 | 0.11.0b0       | 1.9.0b0/1.9.0b2    | 0.1.1b0             | 0.6.0b0         | 0.1.0b1           |
 | 0.9.0b0           | 0.10.0b0       | 1.8.0b0            | 0.1.1b0             | 0.5.0b0         |                   |
 | 0.8.0b0/0.8.1b0   | 0.9.0b0        | 1.7.0b0            | 0.1.1b0             | 0.4.0b0         |                   |

README.zh-CN.md

@@ -97,6 +97,7 @@ sh install.sh autonomy-node -n domainId -g 8096 -k 8087 -p 8088 -P notls -q 1380
 
 | secretpad Version | Kuscia Version | secretflow Version | trustedflow Version | serving Version | dataproxy Version |
 |-------------------|----------------|--------------------|---------------------|-----------------|-------------------|
+| 0.11.0b0          | 0.12.0b0       | 1.10.0b0           | 0.1.1b0             | 0.7.0b0         | 0.2.0b0           |
 | 0.10.0b0/0.10.1b0 | 0.11.0b0       | 1.9.0b0/1.9.0b2    | 0.1.1b0             | 0.6.0b0         | 0.1.0b1           |
 | 0.9.0b0           | 0.10.0b0       | 1.8.0b0            | 0.1.1b0             | 0.5.0b0         |                   |
 | 0.8.0b0/0.8.1b0   | 0.9.0b0        | 1.7.0b0            | 0.1.1b0             | 0.4.0b0         |                   |
@@ -106,7 +107,6 @@ sh install.sh autonomy-node -n domainId -g 8096 -k 8087 -p 8088 -P notls -q 1380
 | 0.4.1b0           | 0.5.0b0        | 1.4.0.dev24011601  | 0.1.1b0             |                 |                   |
 | 0.3.0b0           | 0.4.0b0        | 1.3.0.dev20231109  | 0.1.1b0             |                 |                   |
 
-
 ## Contact
 
 * [B站](https://space.bilibili.com/2073575923): secretflow B站账号

config/application-dev.yaml

@@ -1,11 +1,6 @@
 spring:
   jpa:
     show-sql: false
-    properties:
-      hibernate:
-        format_sql: false
-  datasource:
-    url: jdbc:sqlite:./db/secretpad.sqlite
 
 springdoc:
   api-docs:
@@ -24,25 +19,25 @@ kuscia:
     - domainId: ${NODE_ID:kuscia-system}
       mode: master
       host: ${KUSCIA_API_ADDRESS:root-kuscia-master}
-      port: ${KUSCIA_API_PORT:8083}
+      port: ${KUSCIA_API_PORT:18083}
       protocol: ${KUSCIA_PROTOCOL:tls}
       cert-file: config/certs/client.crt
       key-file: config/certs/client.pem
       token: config/certs/token
 
     - domainId: alice
       mode: lite
-      host: ${KUSCIA_API_LITE_ALICE_ADDRESS:root-kuscia-lite-alice}
-      port: ${KUSCIA_API_PORT:8083}
+      host: ${KUSCIA_API_ADDRESS:root-kuscia-lite-alice}
+      port: ${KUSCIA_API_PORT:28083}
       protocol: ${KUSCIA_PROTOCOL:tls}
       cert-file: config/certs/alice/client.crt
       key-file: config/certs/alice/client.pem
       token: config/certs/alice/token
 
     - domainId: bob
       mode: lite
-      host: ${KUSCIA_API_LITE_BOB_ADDRESS:root-kuscia-lite-bob}
-      port: ${KUSCIA_API_PORT:8083}
+      host: ${KUSCIA_API_ADDRESS:root-kuscia-lite-bob}
+      port: ${KUSCIA_API_PORT:38083}
       protocol: ${KUSCIA_PROTOCOL:tls}
       cert-file: config/certs/bob/client.crt
       key-file: config/certs/bob/client.pem

config/application-edge.yaml

@@ -1,9 +1,7 @@
-spring:
-  flyway:
-    baseline-on-migrate: true
+flyway:
+  default:
     locations:
       - filesystem:./config/schema/edge
-
 secretpad:
   platform-type: EDGE
   node-id: ${NODE_ID}
@@ -44,6 +42,9 @@ data:
     - org.secretflow.secretpad.persistence.entity.FeatureTableDO
     - org.secretflow.secretpad.persistence.entity.ProjectFeatureTableDO
     - org.secretflow.secretpad.persistence.entity.ProjectGraphDomainDatasourceDO
+    - org.secretflow.secretpad.persistence.entity.ProjectScheduleDO
+    - org.secretflow.secretpad.persistence.entity.ProjectScheduleJobDO
+    - org.secretflow.secretpad.persistence.entity.ProjectScheduleTaskDO
 
 
 edge:

config/application-p2p.yaml

@@ -1,6 +1,5 @@
-spring:
-  flyway:
-    baseline-on-migrate: true
+flyway:
+  default:
     locations:
       - filesystem:./config/schema/p2p
 
@@ -54,6 +53,10 @@ data:
     - org.secretflow.secretpad.persistence.entity.ProjectFeatureTableDO
     - org.secretflow.secretpad.persistence.entity.ProjectGraphDomainDatasourceDO
     - org.secretflow.secretpad.persistence.entity.ProjectInstDO
+    - org.secretflow.secretpad.persistence.entity.ProjectInstDO
+    - org.secretflow.secretpad.persistence.entity.ProjectScheduleDO
+    - org.secretflow.secretpad.persistence.entity.ProjectScheduleJobDO
+    - org.secretflow.secretpad.persistence.entity.ProjectScheduleTaskDO
 
 # path which do not use in this mode ,config this mvc return 404 to user
 # @see DynamicBeanRegisterInit

config/application-test.yaml

@@ -41,4 +41,7 @@ data:
     - org.secretflow.secretpad.persistence.entity.ProjectModelServingDO
     - org.secretflow.secretpad.persistence.entity.ProjectModelPackDO
     - org.secretflow.secretpad.persistence.entity.ProjectFeatureTableDO
-    - org.secretflow.secretpad.persistence.entity.ProjectGraphDomainDatasourceDO
+    - org.secretflow.secretpad.persistence.entity.ProjectGraphDomainDatasourceDO
+    - org.secretflow.secretpad.persistence.entity.ProjectScheduleDO
+    - org.secretflow.secretpad.persistence.entity.ProjectScheduleJobDO
+    - org.secretflow.secretpad.persistence.entity.ProjectScheduleTaskDO

config/application.yaml

@@ -26,6 +26,13 @@ server:
     min-response-size: 1024
 
 spring:
+  quartz:
+    job-store-type: jdbc
+    jdbc:
+      initialize-schema: never
+    properties:
+      org.quartz.scheduler.instanceName: secretpadScheduler
+      org.quartz.jobStore.isClustered: false
   task:
     scheduling:
       pool:
@@ -40,17 +47,14 @@ spring:
         format_sql: false
     open-in-view: false
   datasource:
-    driver-class-name: org.sqlite.JDBC
-    url: jdbc:sqlite:./db/secretpad.sqlite
-    hikari:
-      idle-timeout: 60000
-      maximum-pool-size: 1
-      connection-timeout: 6000
-  flyway:
-    baseline-on-migrate: true
-    locations:
-      - filesystem:./config/schema/center
-
+    default:
+      driver-class-name: org.sqlite.JDBC
+      jdbc-url: jdbc:sqlite:./db/secretpad.sqlite
+    quartz:
+      driver-class-name: org.h2.Driver
+      jdbc-url: jdbc:h2:./db/secretpadQuartz.mv.db;DB_CLOSE_ON_EXIT=FALSE
+      username: sa
+      password: password
   #datasource used for mysql
   #spring:
   #  task:
@@ -87,6 +91,20 @@ spring:
     jcache:
       config:
         classpath:ehcache.xml
+
+flyway:
+  default:
+    locations:
+      - filesystem:./config/schema/center
+    baseline-on-migrate: true
+    encoding: UTF-8
+  quartz:
+    baseline-on-migrate: true
+    encoding: UTF-8
+    locations:
+      - filesystem:./config/schema/quartz
+
+
 springdoc:
   api-docs:
     enabled: false
@@ -100,7 +118,8 @@ management:
     web:
       exposure:
         include:
-          - prometheus
+          # prometheus monitoring indicator interface does not have security protection. If you need to use it, please inject security protection. It is best to use an intranet environment.
+          #          - prometheus
           - health
   metrics:
     tags:
@@ -177,10 +196,10 @@ secretpad:
 
   component:
     hide:
-      - secretflow/io/read_data:0.0.1
-      - secretflow/io/write_data:0.0.1
-      - secretflow/io/identity:0.0.1
-      - secretflow/model/model_export:0.0.1
+      - secretflow/io/read_data:1.0.0
+      - secretflow/io/write_data:1.0.0
+      - secretflow/io/identity:1.0.0
+      - secretflow/model/model_export:1.0.0
       - secretflow/ml.train/slnn_train:0.0.1
       - secretflow/ml.predict/slnn_predict:0.0.2
   data-proxy:
@@ -216,6 +235,10 @@ data:
     - org.secretflow.secretpad.persistence.entity.FeatureTableDO
     - org.secretflow.secretpad.persistence.entity.ProjectFeatureTableDO
     - org.secretflow.secretpad.persistence.entity.ProjectGraphDomainDatasourceDO
+    - org.secretflow.secretpad.persistence.entity.ProjectScheduleDO
+    - org.secretflow.secretpad.persistence.entity.ProjectScheduleJobDO
+    - org.secretflow.secretpad.persistence.entity.ProjectScheduleTaskDO
+
 
 inner-port:
   path: