From cddb45e902d981013a91ec6816d999afb03e36c5 Mon Sep 17 00:00:00 2001 From: cyjseagull Date: Thu, 12 Dec 2024 17:23:09 +0800 Subject: [PATCH] update depends version (#197) * fix create_docker template * update depends version * fix jupyter configuration --- .github/workflows/docker.yml | 1 - build.gradle | 2 +- docker-files/jupyter/Dockerfile | 4 ++-- wedpr-builder/conf/config-example.toml | 2 ++ wedpr-builder/db/scripts/drop/db_drop.sh | 2 ++ wedpr-builder/db/scripts/init/db_init.sh | 2 ++ .../config/wedpr_deploy_config.py | 15 +++++++++++++-- .../wedpr_builder/tpl/docker/create_docker.sh | 2 +- .../jupyter/core/JupyterHostSetting.java | 18 ++++++++++++++++-- .../jupyter/core/JupyterSetting.java | 1 + .../integration/jupyter/dao/JupyterInfoDO.java | 5 ++++- 11 files changed, 44 insertions(+), 10 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index a66e2097..864a516b 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -8,7 +8,6 @@ on: tags: - 'v3.*.*' branches: - - feature-milestone2 - main release: types: [prereleased] diff --git a/build.gradle b/build.gradle index 41cc6e88..315809ad 100644 --- a/build.gradle +++ b/build.gradle @@ -64,7 +64,7 @@ ext { guavaVersion = "32.0.1-jre" bcprovVersion = "1.78.1" googleAutoServiceVersion = "1.1.1" - wedprGatewaySDKVersion = "1.0.0-SNAPSHOT" + wedprGatewaySDKVersion = "3.0.0-SNAPSHOT" gsonVersion = "2.10.1" servletApiVersion="4.0.1" javaeeApiVersion="8.0.1" diff --git a/docker-files/jupyter/Dockerfile b/docker-files/jupyter/Dockerfile index 3312c471..64b7f34a 100644 --- a/docker-files/jupyter/Dockerfile +++ b/docker-files/jupyter/Dockerfile @@ -15,5 +15,5 @@ RUN chmod -R 777 /home/share/ RUN pip install --no-cache-dir -i https://pypi.mirrors.ustc.edu.cn/simple/ -r root/requirements.txt # TODO: use the non-test pip after uploaded -RUN pip install --no-cache-dir -i https://test.pypi.org/simple/ wedpr-authenticator==1.0.0.dev-20241124 -RUN pip install --no-cache-dir -i https://test.pypi.org/simple/ wedpr-ml-toolkit==1.0.0.dev-20241201 \ No newline at end of file +RUN pip install --no-cache-dir -i https://test.pypi.org/simple/ wedpr-authenticator +RUN pip install --no-cache-dir -i https://test.pypi.org/simple/ wedpr-ml-toolkit diff --git a/wedpr-builder/conf/config-example.toml b/wedpr-builder/conf/config-example.toml index 2f2b4a5e..f9217065 100644 --- a/wedpr-builder/conf/config-example.toml +++ b/wedpr-builder/conf/config-example.toml @@ -133,6 +133,7 @@ wedpr_api_token = "" deploy_ip = ["127.0.0.1:1"] # the server start port server_start_port = "19000" + jupyter_external_ip = "" # configuration for mpc [agency.mpc] @@ -269,6 +270,7 @@ wedpr_api_token = "" deploy_ip = ["127.0.0.1:1"] # the server start port server_start_port = "29000" + jupyter_external_ip = "" # configuration for mpc [agency.mpc] diff --git a/wedpr-builder/db/scripts/drop/db_drop.sh b/wedpr-builder/db/scripts/drop/db_drop.sh index bd4f9b46..726b0dfd 100644 --- a/wedpr-builder/db/scripts/drop/db_drop.sh +++ b/wedpr-builder/db/scripts/drop/db_drop.sh @@ -1,4 +1,6 @@ #!/bin/sh +SHELL_FOLDER=$(cd $(dirname $0);pwd) +cd ${SHELL_FOLDER} if [[ $# -lt 5 ]] ; then echo "Usage: bash ${0} DB_IP DB_PORT DB_USER DB_PASSWORD DB_NAME" diff --git a/wedpr-builder/db/scripts/init/db_init.sh b/wedpr-builder/db/scripts/init/db_init.sh index 7bda6a5f..2a784088 100644 --- a/wedpr-builder/db/scripts/init/db_init.sh +++ b/wedpr-builder/db/scripts/init/db_init.sh @@ -1,4 +1,6 @@ #!/bin/sh +SHELL_FOLDER=$(cd $(dirname $0);pwd) +cd ${SHELL_FOLDER} if [[ $# -lt 5 ]] ; then echo "Usage: bash ${0} DB_IP DB_PORT DB_USER DB_PASSWORD DB_NAME" diff --git a/wedpr-builder/wedpr_builder/config/wedpr_deploy_config.py b/wedpr-builder/wedpr_builder/config/wedpr_deploy_config.py index fc3bc235..f30f2f58 100644 --- a/wedpr-builder/wedpr_builder/config/wedpr_deploy_config.py +++ b/wedpr-builder/wedpr_builder/config/wedpr_deploy_config.py @@ -343,9 +343,11 @@ def __init__( self, entry_point: str, start_port: int, + jupyter_external_ip: str, max_jupyter_count: int = 10): self.entryPoint = entry_point self.jupyterStartPort = start_port + self.jupyterExternalIp = jupyter_external_ip self.maxJupyterCount = max_jupyter_count def __repr__(self): @@ -356,6 +358,8 @@ def __repr__(self): def as_dict(self): result = {} result.update({"\"entryPoint\"": f"\"{self.entryPoint}\""}) + result.update( + {"\"jupyterExternalIp\"": f"\"{self.jupyterExternalIp}\""}) result.update({"\"jupyterStartPort\"": f"{self.jupyterStartPort}"}) result.update({"\"maxJupyterCount\"": f"{self.maxJupyterCount}"}) return result @@ -377,6 +381,9 @@ def __init__(self, config, env_config: EnvConfig, self.agency = agency self.tpl_config_file_path = tpl_config_file_path self.config_file_list = config_file_list + # the jupyter external ip + self.jupyter_external_ip = utilities.get_item_value( + self.config, "jupyter_external_ip", None, False, config_section) self.deploy_ip_list = utilities.get_item_value( self.config, "deploy_ip", [], must_exist, config_section) self.server_start_port = int(utilities.get_item_value( @@ -390,7 +397,7 @@ def to_jupyter_sql(self): return "" jupyter_setting = "'%s'" % self.jupyter_infos.to_string() utilities.log_info(f"* jupyter_setting: {jupyter_setting}") - sql = 'insert into \`wedpr_config_table\`(\`config_key\`, \`config_valule\`) values(\\\"jupyter_entrypoints\\\", %s);' % jupyter_setting + sql = 'insert into \`wedpr_config_table\`(\`config_key\`, \`config_value\`) values(\\\"jupyter_entrypoints\\\", %s);' % jupyter_setting return sql def __repr__(self): @@ -454,14 +461,18 @@ def to_properties(self, deploy_ip, node_index: int) -> {}: # reserver 100 ports for jupyter use jupyter_start_port = server_start_port + 100 default_jupyter_max_num = 20 + jupyter_external_ip = self.jupyter_external_ip + if jupyter_external_ip is None or len(jupyter_external_ip) == 0: + jupyter_external_ip = deploy_ip if self.service_type == constant.ServiceInfo.wedpr_jupyter_worker_service: begin_port = jupyter_start_port + default_jupyter_max_num * node_index end_port = begin_port + default_jupyter_max_num exposed_port_list = f"{exposed_port_list} -p {begin_port}-{end_port}:{begin_port}-{end_port}" - entry_point = f"http://{deploy_ip}:{server_start_port}" + entry_point = f"{deploy_ip}:{server_start_port}" # add the SingleJupyterInfo self.jupyter_infos.jupyters.append(SingleJupyterInfo( entry_point=entry_point, + jupyter_external_ip=jupyter_external_ip, start_port=begin_port)) if self.service_type == constant.ServiceInfo.wedpr_mpc_service: spdz_listen_port = server_start_port + 2 diff --git a/wedpr-builder/wedpr_builder/tpl/docker/create_docker.sh b/wedpr-builder/wedpr_builder/tpl/docker/create_docker.sh index 6c700b5d..c74183e3 100644 --- a/wedpr-builder/wedpr_builder/tpl/docker/create_docker.sh +++ b/wedpr-builder/wedpr_builder/tpl/docker/create_docker.sh @@ -15,7 +15,7 @@ if [[ "${confirm}" == "Y" || "${confirm}" == "y" ]]; then LOG_INFO "* Pull image ${WEDPR_IMAGE_DESC}" docker pull ${WEDPR_IMAGE_DESC} LOG_INFO "* Pull image ${WEDPR_IMAGE_DESC} success, begin to create docker" - docker run -d --net host -v ${SHELL_FOLDER}/${WEDPR_CONFIG_DIR}:${DOCKER_CONF_PATH} -v ${SHELL_FOLDER}/${WEDPR_LOG_DIR}:${DOCKER_LOG_PATH} ${EXTENDED_MOUNT_CONF} --name ${WEDPR_DOCKER_NAME} ${WEDPR_IMAGE_DESC} ${WEDPR_DOCKER_EXPORSE_PORT_LIST} + docker run -d ${WEDPR_DOCKER_EXPORSE_PORT_LIST} -v ${SHELL_FOLDER}/${WEDPR_CONFIG_DIR}:${DOCKER_CONF_PATH} -v ${SHELL_FOLDER}/${WEDPR_LOG_DIR}:${DOCKER_LOG_PATH} ${EXTENDED_MOUNT_CONF} --name ${WEDPR_DOCKER_NAME} ${WEDPR_IMAGE_DESC} LOG_INFO "Create docker: ${WEDPR_DOCKER_NAME} success" else LOG_INFO "Exit without create docker ${WEDPR_DOCKER_NAME}" diff --git a/wedpr-components/env-integration/jupyter/src/main/java/com/webank/wedpr/components/integration/jupyter/core/JupyterHostSetting.java b/wedpr-components/env-integration/jupyter/src/main/java/com/webank/wedpr/components/integration/jupyter/core/JupyterHostSetting.java index e39804d9..fb8a6675 100644 --- a/wedpr-components/env-integration/jupyter/src/main/java/com/webank/wedpr/components/integration/jupyter/core/JupyterHostSetting.java +++ b/wedpr-components/env-integration/jupyter/src/main/java/com/webank/wedpr/components/integration/jupyter/core/JupyterHostSetting.java @@ -23,9 +23,9 @@ import java.util.ArrayList; import java.util.List; import lombok.Data; +import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.util.StringUtils; @Data public class JupyterHostSetting { @@ -33,12 +33,24 @@ public class JupyterHostSetting { @Data public static class SingleHostSetting { + // the client entryPoint + private String jupyterExternalIp; // the entry point of the host private String entryPoint; // the limitation private Integer maxJupyterCount = JupyterConfig.getMaxJupyterPerHost(); // the startPort private Integer jupyterStartPort = JupyterConfig.getDefaultJupyterStartPort(); + + public String getJupyterExternalIp() { + if (StringUtils.isBlank(jupyterExternalIp)) { + if (StringUtils.isBlank(entryPoint)) { + return entryPoint; + } + return entryPoint.split(":")[0]; + } + return jupyterExternalIp; + } } private List hostSettings = new ArrayList<>(); @@ -75,6 +87,8 @@ public JupyterInfoDO allocateJupyter(String userName, String agency) throws Exce allocatedHost.toString()); JupyterSetting jupyterSetting = new JupyterSetting(userName, listenPort); + // set the host ip + jupyterSetting.setHostIp(allocatedHost.getJupyterExternalIp()); // insert the information JupyterInfoDO allocatedJupyter = new JupyterInfoDO(); allocatedJupyter.setAgency(agency); @@ -98,7 +112,7 @@ public String serialize() throws Exception { } public static JupyterHostSetting deserialize(String data) throws Exception { - if (StringUtils.isEmpty(data)) { + if (StringUtils.isBlank(data)) { return null; } return ObjectMapperFactory.getObjectMapper().readValue(data, JupyterHostSetting.class); diff --git a/wedpr-components/env-integration/jupyter/src/main/java/com/webank/wedpr/components/integration/jupyter/core/JupyterSetting.java b/wedpr-components/env-integration/jupyter/src/main/java/com/webank/wedpr/components/integration/jupyter/core/JupyterSetting.java index f78cabcf..97e931c8 100644 --- a/wedpr-components/env-integration/jupyter/src/main/java/com/webank/wedpr/components/integration/jupyter/core/JupyterSetting.java +++ b/wedpr-components/env-integration/jupyter/src/main/java/com/webank/wedpr/components/integration/jupyter/core/JupyterSetting.java @@ -28,6 +28,7 @@ public class JupyterSetting { private String listenIp = JupyterConfig.getDefaultJupyterListenIp(); private Integer listenPort; private String noteBookPath; + private String hostIp; public JupyterSetting() {} diff --git a/wedpr-components/env-integration/jupyter/src/main/java/com/webank/wedpr/components/integration/jupyter/dao/JupyterInfoDO.java b/wedpr-components/env-integration/jupyter/src/main/java/com/webank/wedpr/components/integration/jupyter/dao/JupyterInfoDO.java index 04eedefd..e30b3545 100644 --- a/wedpr-components/env-integration/jupyter/src/main/java/com/webank/wedpr/components/integration/jupyter/dao/JupyterInfoDO.java +++ b/wedpr-components/env-integration/jupyter/src/main/java/com/webank/wedpr/components/integration/jupyter/dao/JupyterInfoDO.java @@ -101,7 +101,10 @@ protected void generateJupyterAccessUrl() { if (this.jupyterSetting == null) { return; } - String accessHost = (this.getAccessEntry().split(":")[0]); + String accessHost = (this.jupyterSetting.getHostIp()); + if (StringUtils.isBlank(accessHost)) { + accessHost = this.accessEntry.split(":")[0]; + } this.jupyterAccessUrl = Common.getUrl( String.format(