add site_init scripts

wedpr-builder/db/wedpr_dml.sql

@@ -1,10 +1,6 @@
 -- the Wizard algorithm template
 insert into `wedpr_config_table`(`config_key`, `config_value`) values("wedpr_algorithm_templates", '{"version":"1.0","templates":[{"name":"PSI","title":"数据对齐","supportable":true,"enable":true,"participateNumber":"2+","detail":"","version":"1.0"},{"name":"PIR","title":"匿踪查询","supportable":true,"enable":true,"participateNumber":"1","detail":"","version":"1.0"},{"name":"SQL","title":"联表分析","supportable":true,"enable":true,"participateNumber":"1+","detail":"","version":"1.0"},{"name":"MPC","title":"自定义计算","supportable":true,"enable":true,"participateNumber":"1+","detail":"","version":"1.0"},{"name":"XGB_TRAINING","title":"SecureLGBM训练","supportable":true,"enable":true,"participateNumber":"1+","needTagsProvider":true,"detail":"","version":"1.0"},{"name":"XGB_PREDICTING","title":"SecureLGBM预测","supportable":true,"enable":true,"participateNumber":"1+","needTagsProvider":true,"detail":"","version":"1.0"},{"name":"LR_TRAINING","title":"SecureLR建模","supportable":true,"enable":true,"participateNumber":"1+","needTagsProvider":true,"detail":"","version":"1.0"},{"name":"LR_PREDICTING","title":"SecureLR预测","supportable":true,"enable":true,"participateNumber":"1+","needTagsProvider":true,"detail":"","version":"1.0"},{"name":"PREPROCESSING","title":"数据预处理","supportable":true,"detail":"","version":"1.0"},{"name":"FEATURE_ENGINEERING","title":"特征工程","supportable":true,"detail":"","version":"1.0"}]}');
 
-
--- the jupyter related host settings
--- insert into `wedpr_config_table`(`config_key`, `config_value`) values("jupyter_entrypoints", '{"hostSettings":[{"entryPoint":"192.168.0.238:14000","maxJupyterCount":5,"jupyterStartPort":14000}]}');
-
 -- the  jupyter related code template
 -- command to create user
 insert into `wedpr_config_table`(`config_key`, `config_value`) values("wedpr_create_user", 'useradd -m ${user_name}');
@@ -41,4 +37,7 @@ insert into wedpr_group_detail (group_id, username, status) values('100000000000
 insert into wedpr_user (username, password, status) values('admin', '{bcrypt}$2a$10$9ZhDOBp.sRKat4l14ygu/.LscxrMUcDAfeVOEPiYwbcRkoB09gCmi', 0);
 insert into wedpr_user_role(username, role_id) values ('admin', '1');
 insert into wedpr_role_permission (role_id, role_name, permission_id) values ('1', 'admin_user', '1');
-insert into wedpr_role_permission (role_id, role_name, permission_id) values ('2', 'original_user', '2');
+insert into wedpr_role_permission (role_id, role_name, permission_id) values ('2', 'original_user', '2');
+
+-- the jupyter related host settings
+-- insert into `wedpr_config_table`(`config_key`, `config_value`) values("jupyter_entrypoints", '{"hostSettings":[{"entryPoint":"192.168.0.238:14000","maxJupyterCount":5,"jupyterStartPort":14000}]}');

wedpr-builder/wedpr_builder/common/constant.py

@@ -91,6 +91,15 @@ class ConfigInfo:
     nginx_tpl_path = get_abs_path("nginx/")
     nginx_config_file_list = ["nginx.conf"]
 
+    # the init db file path
+    pwd_path = os.getcwd()
+    db_dir_name = "db"
+    db_file_path = os.path.join(pwd_path, db_dir_name)
+    dml_sql_file = "wedpr_dml.sql"
+    # the site init conf
+    init_tpl_path = get_abs_path("init/")
+    init_file_path_list = ["site_init.sh"]
+
     @staticmethod
     def get_docker_path(file_path: str):
         return os.path.join(ConfigInfo.default_docker_work_dir, file_path)
@@ -113,6 +122,10 @@ class ConfigProperities:
     SQLALCHEMY_URL = "SQLALCHEMY_URL"
     MYSQL_USER = "MYSQL_USER"
     MYSQL_PASSWORD = "MYSQL_PASSWORD"
+    # the mysql var used for init script
+    MYSQL_HOST = "MYSQL_HOST"
+    MYSQL_PORT = "MYSQL_PORT"
+    MYSQL_DATABASE = "MYSQL_DATABASE"
     # the blockchain configuration
     BLOCKCHAIN_GROUP_ID = "BLOCKCHAIN_GROUP_ID"
     BLOCKCHAIN_PEERS = "BLOCKCHAIN_PEERS"

wedpr-builder/wedpr_builder/config/wedpr_deploy_config.py

@@ -5,6 +5,7 @@
 from wedpr_builder.common import utilities
 from wedpr_builder.common import constant
 from wedpr_builder.config.wedpr_user_config import UserJWTConfig
+import json
 
 
 class ComponentSwitch:
@@ -300,9 +301,54 @@ def to_properties(self) -> {}:
         properties.update({constant.ConfigProperities.MYSQL_USER: self.user})
         properties.update(
             {constant.ConfigProperities.MYSQL_PASSWORD: self.password})
+        # mysql host
+        properties.update({constant.ConfigProperities.MYSQL_HOST: self.host})
+        properties.update({constant.ConfigProperities.MYSQL_PORT: self.port})
+        properties.update(
+            {constant.ConfigProperities.MYSQL_DATABASE: self.database})
         return properties
 
 
+class JupyterInfos:
+    def __init__(self):
+        self.jupyters = []
+
+    def to_string(self) -> str:
+        """
+        convert to json string
+        :return:
+        """
+        dict_result = {}
+        jupyter_dict_array = []
+        for jupyter in self.jupyters:
+            jupyter_dict_array.append(jupyter.as_dict())
+        dict_result.update({"hostSettings": jupyter_dict_array})
+        return json.dumps(dict_result)
+
+
+class SingleJupyterInfo:
+    def __init__(
+            self,
+            entry_point: str,
+            start_port: int,
+            max_jupyter_count: int = 10):
+        self.entryPoint = entry_point
+        self.jupyterStartPort = start_port
+        self.maxJupyterCount = max_jupyter_count
+
+    def __repr__(self):
+        return f"entryPoint: {self.entryPoint}, " \
+               f"jupyterStartPort: {self.jupyterStartPort}," \
+               f" maxJupyterCount: {self.maxJupyterCount}"
+
+    def as_dict(self):
+        result = {}
+        result.update({"\"entryPoint\"": f"\"{self.entryPoint}\""})
+        result.update({"\"jupyterStartPort\"": f"{self.jupyterStartPort}"})
+        result.update({"\"maxJupyterCount\"": f"{self.maxJupyterCount}"})
+        return result
+
+
 class ServiceConfig:
     def __init__(self, config, env_config: EnvConfig,
                  gateway_targets: str,
@@ -325,10 +371,19 @@ def __init__(self, config, env_config: EnvConfig,
             self.config, "server_start_port",
             0, must_exist, config_section))
         self.server_backend_list = []
+        self.jupyter_infos = JupyterInfos()
+
+    def to_jupyter_sql(self):
+        if self.service_type != constant.ServiceInfo.wedpr_jupyter_worker_service:
+            return ""
+        jupyter_setting = "'%s'" % self.jupyter_infos.to_string()
+        utilities.log_info(f"* jupyter_setting: {jupyter_setting}")
+        sql = 'insert into \`wedpr_config_table\`(\`config_key\`, \`config_valule\`) values(\\\"jupyter_entrypoints\\\", %s);' % jupyter_setting
+        return sql
 
     def __repr__(self):
         return f"**ServiceConfig: deploy_ip: {self.deploy_ip_list}, " \
-               f"agency: {self.agency}" \
+               f"agency: {self.agency}, " \
                f"server_start_port: {self.server_start_port}," \
                f"service_type: {self.service_type} \n**"
 
@@ -343,8 +398,8 @@ def to_nginx_properties(self):
 
     def to_properties(self, deploy_ip, node_index: int) -> {}:
         props = {}
-        start_port = self.server_start_port + 2 * node_index
-        self.server_backend_list.append(f"{deploy_ip}:{start_port}")
+        server_start_port = self.server_start_port + 2 * node_index
+        self.server_backend_list.append(f"{deploy_ip}:{server_start_port}")
         # nodeid
         node_id = f"{self.service_type}-{self.env_config.zone}-node{node_index}"
         props.update({constant.ConfigProperities.WEDPR_NODE_ID: node_id})
@@ -360,26 +415,31 @@ def to_properties(self, deploy_ip, node_index: int) -> {}:
             {constant.ConfigProperities.WEDPR_TRANSPORT_HOST_IP: host_ip})
         # the server listen port
         props.update(
-            {constant.ConfigProperities.WEDPR_SERVER_LISTEN_PORT: start_port})
+            {constant.ConfigProperities.WEDPR_SERVER_LISTEN_PORT: server_start_port})
         # transport listen_port
         props.update(
-            {constant.ConfigProperities.WEDPR_TRANSPORT_LISTEN_PORT: start_port + 1})
+            {constant.ConfigProperities.WEDPR_TRANSPORT_LISTEN_PORT: server_start_port + 1})
         # set the image desc for docker mode
         image_desc = self.env_config.get_image_desc_by_service_name(
             self.service_type)
         if image_desc is not None:
             props.update(
                 {constant.ConfigProperities.WEDPR_IMAGE_DESC: image_desc})
         # set the exposed port
-        exposed_port_list = f"-p {start_port}:{start_port} -p {start_port + 1}:{start_port + 1}"
+        exposed_port_list = f"-p {server_start_port}:{server_start_port} -p {server_start_port + 1}:{server_start_port + 1}"
         # default expose 20 ports for jupyter use
         # reserver 100 ports for jupyter use
-        jupyter_start_port = start_port + 100
+        jupyter_start_port = server_start_port + 100
         default_jupyter_max_num = 20
         if self.service_type == constant.ServiceInfo.wedpr_jupyter_worker_service:
-            start_port = jupyter_start_port + default_jupyter_max_num * node_index
-            end_port = start_port + default_jupyter_max_num
-            exposed_port_list = f"{exposed_port_list} -p {start_port}-{end_port}:{start_port}-{end_port}"
+            begin_port = jupyter_start_port + default_jupyter_max_num * node_index
+            end_port = begin_port + default_jupyter_max_num
+            exposed_port_list = f"{exposed_port_list} -p {begin_port}-{end_port}:{begin_port}-{end_port}"
+            entry_point = f"http://{deploy_ip}:{server_start_port}"
+            # add the SingleJupyterInfo
+            self.jupyter_infos.jupyters.append(SingleJupyterInfo(
+                entry_point=entry_point,
+                start_port=begin_port,))
         props.update(
             {constant.ConfigProperities.WEDPR_DOCKER_EXPORSE_PORT_LIST: exposed_port_list})
         # set the docker name
@@ -906,6 +966,36 @@ def get_pir_properties(self, deploy_ip: str, node_index: int):
             constant.ConfigInfo.wedpr_pir_docker_dir))
         return props
 
+    def __update_dml__(self, sql, dml_file_path, use_double_quote=False):
+        command = "echo '%s' >> %s" % (sql, dml_file_path)
+        if use_double_quote:
+            command = "echo \"%s\" >> %s" % (sql, dml_file_path)
+        (ret, output) = utilities.execute_command_and_getoutput(command)
+        if ret is False:
+            raise Exception(
+                f"update dml file {dml_file_path} failed for {output}")
+
+    def update_dml(self, agency_list, init_path):
+        """
+        update the dml information
+        :param agency_list:
+        :param init_path:
+        :return:
+        """
+        sql = "\n"
+        # set the agency information
+        for agency in agency_list:
+            sql = f'{sql}insert into `wedpr_agency_table`(`name`, `cnName`, `desc`, `meta`) ' \
+                  f'values("{agency}", "{agency}", "{agency}", "{agency}");\n'
+        dml_file_path = os.path.join(
+            init_path, constant.ConfigInfo.db_dir_name, constant.ConfigInfo.dml_sql_file)
+        self.__update_dml__(sql, dml_file_path)
+        # set the jupyter setting
+        if self.jupyter_worker_config is None:
+            raise Exception("Must set the jupyter worker configuration!")
+        sql = self.jupyter_worker_config.to_jupyter_sql()
+        self.__update_dml__(sql, dml_file_path, True)
+
     def __repr__(self):
         return f"agency: {self.agency_name}, gateway_config: {self.gateway_config}, " \
                f"node_config: {self.node_list}, hdfs_config: {self.hdfs_storage_config}"

wedpr-builder/wedpr_builder/controller/commandline_helper.py

@@ -24,7 +24,6 @@ def parse_command():
                 "* generate wedpr-site config:\t python3 build_wedpr.py -t wedpr-site\n " \
                 "* generate wedpr-pir config:\t python3 build_wedpr.py -t wedpr-pir\n" \
                 "* generate wedpr-model service config:\t python3 build_wedpr.py -t wedpr-model\n " \
-                "* generate wedpr-jupyter-worker config:\t python3 build_wedpr.py -t wedpr-jupyter-worker\n " \
                 "* generate gateway config:\t python3 build_wedpr.py -o genconfig -c config.toml -t wedpr-gateway -d wedpr-generated\n " \
                 "* generate node config:\t python3 build_wedpr.py -o genconfig -c config.toml -t wedpr-node -d wedpr-generated"
     parser = argparse.ArgumentParser(
@@ -85,8 +84,14 @@ def generate_node_config(args, toml_config):
             sys.exit(-1)
     # the site config generator
     if service_type == constant.ServiceInfo.wedpr_site_service:
-        component_switch = ComponentSwitch(site_must_exists=True)
+        # default generate the jupyter config
+        component_switch = ComponentSwitch(
+            site_must_exists=True, jupyter_must_exists=True)
         config = WeDPRDeployConfig(toml_config, component_switch)
+        jupyter_generator = WedprJupyterWorkerServiceGenerator(
+            config, args.output)
+        jupyter_generator.generate_config()
+        # Note: generate site config after jupyter config generate success
         site_generator = WedprSiteServiceGenerator(config, args.output)
         site_generator.generate_config()
     # the pir service generator
@@ -102,13 +107,16 @@ def generate_node_config(args, toml_config):
         model_service_generator = WedprModelServiceGenerator(
             config, args.output)
         model_service_generator.generate_config()
+    # Note: the jupyter config is generated with the site config
+    """
     # the jupyter worker config generator
     if service_type == constant.ServiceInfo.wedpr_jupyter_worker_service:
         component_switch = ComponentSwitch(jupyter_must_exists=True)
         config = WeDPRDeployConfig(toml_config, component_switch)
         jupyter_generator = WedprJupyterWorkerServiceGenerator(
             config, args.output)
         jupyter_generator.generate_config()
+    """
 
 
 def execute_command(args):