Skip to content

Commit

Permalink
mount log && fix spdz builder
Browse files Browse the repository at this point in the history
  • Loading branch information
@cyjseagull
cyjseagull committed Dec 13, 2024
1 parent c94a6c5 commit 1e12fd0
Show file tree
Hide file tree
Showing 5 changed files with 58 additions and 27 deletions.
8 changes: 5 additions & 3 deletions wedpr-builder/conf/config-example.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@ wedpr_api_token = ""
home = "/user/wedpr/agency0"
name_node = "127.0.0.1"
name_node_port = 9000
webfs_port = 50700
webfs_port = 50070
token = ""
# enable auth or not, default is false
enable_krb5_auth = false
Expand Down Expand Up @@ -140,6 +140,7 @@ wedpr_api_token = ""
deploy_ip = ["127.0.0.1:1"]
# the server start port
server_start_port = "20000"
external_ip = ""

# configuration for the wedpr nodes
[[agency.node]]
Expand Down Expand Up @@ -223,7 +224,7 @@ wedpr_api_token = ""
home = "/user/wedpr/agency1"
name_node = "127.0.0.1"
name_node_port = 9000
webfs_port = 50700
webfs_port = 50070
token = ""
# enable auth or not, default is false
enable_krb5_auth = false
Expand Down Expand Up @@ -277,6 +278,7 @@ wedpr_api_token = ""
deploy_ip = ["127.0.0.1:1"]
# the server start port
server_start_port = "30000"
external_ip = ""

# configuration for the wedpr nodes
[[agency.node]]
Expand Down Expand Up @@ -325,4 +327,4 @@ wedpr_api_token = ""
port = 33060
user = "wedpr"
password = ""
database = ""
database = ""
4 changes: 3 additions & 1 deletion wedpr-builder/wedpr_builder/common/constant.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ class ConfigInfo:
wedpr_pir_docker_dir = "wedpr-pir"
wedpr_site_docker_dir = "wedpr-site"
wedpr_mpc_docker_dir = "wedpr-mpc-service"
wedpr_model_docker_dir = "wedpr-model"
wedpr_model_docker_dir = "wedpr-model/ppc_model"

wedpr_gateway_service_dir = "wedpr-gateway-service"
wedpr_node_service_dir = "wedpr-pro-node-service"
Expand Down Expand Up @@ -111,6 +111,7 @@ class ConfigInfo:
scripts_file_list = ["start.sh", "stop.sh"]
# the default path for docker
default_spdz_home = "/data/app/wedpr/scripts/wedpr-mpc-no-gateway"
default_mpc_server_start_port = 37000
cpp_component_cmd = " -c config.ini "

@staticmethod
Expand Down Expand Up @@ -155,6 +156,7 @@ class ConfigProperities:
WEDPR_SERVER_LISTEN_PORT = "WEDPR_SERVER_LISTEN_PORT"
# the mpc spdz listen port
WEDPR_MPC_SPDZ_LISTEN_PORT = "WEDPR_MPC_SPDZ_LISTEN_PORT"
WEDPR_MPC_SPDZ_EXTERNAL_IP = "WEDPR_MPC_SPDZ_EXTERNAL_IP"
# the hdfs configuration
HDFS_USER = "HDFS_USER"
HDFS_HOME = "HDFS_HOME"
Expand Down
41 changes: 30 additions & 11 deletions wedpr-builder/wedpr_builder/config/wedpr_deploy_config.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -389,6 +389,9 @@ def __init__(self, config, env_config: EnvConfig,
self.server_start_port = int(utilities.get_item_value(
self.config, "server_start_port",
0, must_exist, config_section))
# the external ip, mpc service need this config when deploy to different agencies
self.external_ip = utilities.get_item_value(
self.config, "external_ip", "", False, config_section)
self.server_backend_list = []
self.jupyter_infos = JupyterInfos()

Expand Down Expand Up @@ -420,7 +423,7 @@ def to_nginx_properties(self, nginx_listen_port):
{constant.ConfigProperities.NGINX_PORT: nginx_listen_port})
return props

def to_properties(self, deploy_ip, node_index: int) -> {}:
def to_properties(self, deploy_ip, node_index: int, agency_index: int = 0) -> {}:
props = {}
server_start_port = self.server_start_port + 3 * node_index
self.server_backend_list.append(
Expand Down Expand Up @@ -474,11 +477,18 @@ def to_properties(self, deploy_ip, node_index: int) -> {}:
entry_point=entry_point,
jupyter_external_ip=jupyter_external_ip,
start_port=begin_port))
# use host for mpc service
if self.service_type == constant.ServiceInfo.wedpr_mpc_service:
spdz_listen_port = server_start_port + 2
spdz_start_port = self.server_start_port + 2
props.update(
{constant.ConfigProperities.WEDPR_MPC_SPDZ_LISTEN_PORT: spdz_listen_port})
exposed_port_list = f"{exposed_port_list} -p {spdz_listen_port}:{spdz_listen_port}"
{constant.ConfigProperities.WEDPR_MPC_SPDZ_LISTEN_PORT: spdz_start_port})
exposed_port_list = "--net host "
# set the external ip
external_ip = self.external_ip
if self.external_ip is None or len(self.external_ip) == 0:
external_ip = deploy_ip
props.update(
{constant.ConfigProperities.WEDPR_MPC_SPDZ_EXTERNAL_IP: external_ip})
props.update(
{constant.ConfigProperities.WEDPR_DOCKER_EXPORSE_PORT_LIST: exposed_port_list})

Expand Down Expand Up @@ -945,7 +955,7 @@ def generate_cpp_component_docker_properties(
{constant.ConfigProperities.WEDPR_DOCKER_NAME: docker_name})
return props

def __generate_java_service_docker_properties__(self, prefix_path) -> {}:
def __generate_java_service_docker_properties__(self, prefix_path, mount_log: bool = False) -> {}:
props = {}
# the config mount info
props.update({constant.ConfigProperities.WEDPR_CONFIG_DIR: "conf"})
Expand All @@ -956,6 +966,13 @@ def __generate_java_service_docker_properties__(self, prefix_path) -> {}:
props.update({constant.ConfigProperities.WEDPR_LOG_DIR: "logs"})
props.update({constant.ConfigProperities.DOCKER_LOG_PATH:
constant.ConfigInfo.get_docker_path(f"{prefix_path}/logs")})
if mount_log:
local_log_path = "${SHELL_FOLDER}/log"
docker_log_path = constant.ConfigInfo.get_docker_path(
f"{prefix_path}/log")
extra_mount_info = f"-v {local_log_path}:{docker_log_path}"
props.update(
{constant.ConfigProperities.EXTENDED_MOUNT_CONF: extra_mount_info})
return props

def get_wedpr_site_properties(self, deploy_ip: str, node_index: int) -> {}:
Expand All @@ -978,11 +995,12 @@ def get_wedpr_site_properties(self, deploy_ip: str, node_index: int) -> {}:
# the hdfs config
props.update(self.hdfs_storage_config.to_properties())
props.update(self.__generate_java_service_docker_properties__(
constant.ConfigInfo.wedpr_site_docker_dir))
constant.ConfigInfo.wedpr_site_docker_dir, True))
# add nginx configuration mount
local_mount_path = '${SHELL_FOLDER}/conf/nginx.conf'
remote_mount_path = "/etc/nginx/nginx.conf"
extended_mount_conf = f" -v {local_mount_path}:{remote_mount_path}"
extended_mount_conf = f" -v {local_mount_path}:{remote_mount_path} " \
f"{props.get(constant.ConfigProperities.EXTENDED_MOUNT_CONF)}"
props.update(
{constant.ConfigProperities.EXTENDED_MOUNT_CONF: extended_mount_conf})
return props
Expand All @@ -1000,7 +1018,7 @@ def get_jupyter_worker_properties(self, deploy_ip: str, node_index: int) -> {}:
props.update(self.jupyter_worker_config.to_properties(
deploy_ip, node_index))
props.update(self.__generate_java_service_docker_properties__(
constant.ConfigInfo.wedpr_worker_docker_dir))
constant.ConfigInfo.wedpr_worker_docker_dir, False))
return props

def get_pir_properties(self, deploy_ip: str, node_index: int):
Expand All @@ -1019,15 +1037,16 @@ def get_pir_properties(self, deploy_ip: str, node_index: int):
# the hdfs config
props.update(self.hdfs_storage_config.to_properties())
props.update(self.__generate_java_service_docker_properties__(
constant.ConfigInfo.wedpr_pir_docker_dir))
constant.ConfigInfo.wedpr_pir_docker_dir, True))
return props

def get_mpc_properties(self, deploy_ip: str, node_index: int):
def get_mpc_properties(self, deploy_ip: str, node_index: int, agency_index: int):
props = self.to_properties()
# the zone config
props.update(self.env_config.to_properties())
# the service config
props.update(self.mpc_config.to_properties(deploy_ip, node_index))
props.update(self.mpc_config.to_properties(
deploy_ip, node_index, agency_index))
props.update(self.hdfs_storage_config.to_properties())
# the config mount info
docker_prefix_path = constant.ConfigInfo.wedpr_mpc_docker_dir
Expand Down
31 changes: 19 additions & 12 deletions wedpr-builder/wedpr_builder/generator/wedpr_service_generator.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,8 @@ def __init__(self,
def get_properties(
self, deploy_ip: str,
agency_config: AgencyConfig,
node_index: int) -> {}:
node_index: int,
agency_index: int = 0) -> {}:
pass

@abstractmethod
Expand All @@ -31,11 +32,13 @@ def get_service_config(self, agency_config: AgencyConfig) -> ServiceConfig:

def generate_config(self):
agency_list = self.config.agency_list.keys()
agency_index = 0
for agency in self.config.agency_list.keys():
agency_config = self.config.agency_list.get(agency)
service_config = self.get_service_config(agency_config)
self.__generate_service_config__(
agency_config, agency_list, service_config)
agency_config, agency_list, service_config, agency_index)
agency_index += 1

@abstractmethod
def generate_nginx_config(self, node_path: str, server_config: ServiceConfig):
Expand All @@ -53,7 +56,7 @@ def generate_init_scripts(self, init_dir, agency_list, agency_config: AgencyConf
def __generate_service_config__(
self, agency_config: AgencyConfig,
agency_list,
service_config: ServiceConfig):
service_config: ServiceConfig, agency_index: int):
utilities.print_badge(f"* generate {service_config.service_type} config, "
f"agency: {agency_config.agency_name}, deploy_dir: "
f"{self.config.env_config.deploy_dir}, "
Expand All @@ -71,7 +74,8 @@ def __generate_service_config__(
agency_config=agency_config,
service_config=service_config,
agency_name=service_config.agency,
deploy_ip=ip, node_index=i)
deploy_ip=ip, node_index=i,
agency_index=agency_index)
node_path_list.append(node_path)
nginx_listen_port.append(
service_config.get_nginx_listen_port(i))
Expand Down Expand Up @@ -102,13 +106,15 @@ def __generate_single_node_config__(
service_config: ServiceConfig,
agency_name: str,
deploy_ip: str,
node_index: int):
node_index: int,
agency_index: int):
node_name = f"{service_config.service_type}-node{node_index}"
node_path = self.__get_deploy_path__(
agency_name, deploy_ip, node_name, service_config.service_type)
utilities.print_badge(f"* generate {service_config.service_type} config, "
f"deploy_ip: {deploy_ip}, "
f"node_index: {node_index}, "
f"agency_index: {agency_index},"
f"node_path: {node_path}")
if os.path.exists(node_path) is True:
raise Exception(f"The path {node_path} already exists!"
Expand All @@ -135,7 +141,7 @@ def __generate_single_node_config__(
self.__copy_binary__(dist_path, node_path)
# substitute the configuration
config_properties = self.get_properties(
deploy_ip, agency_config, node_index)
deploy_ip, agency_config, node_index, agency_index)

# the docker mode case
self.__generate_docker_scripts__(
Expand Down Expand Up @@ -301,7 +307,7 @@ def generate_nginx_config(self, node_path: str, server_config: ServiceConfig, ng
def get_properties(
self, deploy_ip: str,
agency_config: AgencyConfig,
node_index: int) -> {}:
node_index: int, agency_index: int = 0) -> {}:
return agency_config.get_wedpr_site_properties(deploy_ip, node_index)

def get_service_config(self, agency_config: AgencyConfig) -> ServiceConfig:
Expand Down Expand Up @@ -351,7 +357,7 @@ def __generate_shell_scripts__(self, dist_path, dst_path):
def get_properties(
self, deploy_ip: str,
agency_config: AgencyConfig,
node_index: int) -> {}:
node_index: int, agency_index: int = 0) -> {}:
return agency_config.get_wedpr_model_properties(deploy_ip, node_index)

def get_service_config(self, agency_config: AgencyConfig) -> ServiceConfig:
Expand Down Expand Up @@ -379,7 +385,7 @@ def __generate_shell_scripts__(self, dist_path, dst_path):
def get_properties(
self, deploy_ip: str,
agency_config: AgencyConfig,
node_index: int) -> {}:
node_index: int, agency_index: int = 0) -> {}:
return agency_config.get_pir_properties(deploy_ip, node_index)

def get_service_config(self, agency_config: AgencyConfig) -> ServiceConfig:
Expand Down Expand Up @@ -432,8 +438,9 @@ def __generate_shell_scripts__(self, dist_path, node_path):
def get_properties(
self, deploy_ip: str,
agency_config: AgencyConfig,
node_index: int) -> {}:
return agency_config.get_mpc_properties(deploy_ip, node_index)
node_index: int, agency_index: int = 0) -> {}:
return agency_config.get_mpc_properties(
deploy_ip, node_index, agency_index)

def get_service_config(self, agency_config: AgencyConfig) -> ServiceConfig:
return agency_config.mpc_config
Expand Down Expand Up @@ -462,7 +469,7 @@ def __copy_binary__(self, dist_path, dst_path):
def get_properties(
self, deploy_ip: str,
agency_config: AgencyConfig,
node_index: int) -> {}:
node_index: int, agency_index: int = 0) -> {}:
return agency_config.get_jupyter_worker_properties(deploy_ip, node_index)

def get_service_config(self, agency_config: AgencyConfig) -> ServiceConfig:
Expand Down
1 change: 1 addition & 0 deletions wedpr-builder/wedpr_builder/tpl/mpc/config.ini
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@
nodeid=${WEDPR_NODE_ID}

[spdz]
connected_external_ip=${WEDPR_MPC_SPDZ_EXTERNAL_IP}
connected_port=${WEDPR_MPC_SPDZ_LISTEN_PORT}


Expand Down

0 comments on commit 1e12fd0

Please sign in to comment.