diff --git a/cpp/tools/build_ppc.sh b/cpp/tools/build_ppc.sh index 730b518c..509a0bd0 100644 --- a/cpp/tools/build_ppc.sh +++ b/cpp/tools/build_ppc.sh @@ -28,6 +28,8 @@ sm2_params="sm_sm2.param" sm_mode="false" days=36500 rsa_key_length=2048 +p2p_connected_conf_name="nodes.json" +file_dir="./" default_version="v1.1.0" compatibility_version=${default_version} @@ -338,7 +340,7 @@ Usage: -l [Required] "ip1:nodeNum1,ip2:nodeNum2" e.g:"192.168.0.1:2,192.168.0.2:3" -e [Optional] ppc-air-node binary exec -o [Optional] output directory, default ./nodes - -p [Optional] Default 40300,10200 means p2p_port start from 40300, rpc_port from 10200 + -p [Optional] Default 40300,10200,18000 means p2p_port start from 40300, rpc_port from 10200, grpc_port from 18000 -s [Optional] SM SSL connection or not, default is false -d [Optional] Disable ra2018 psi or not, default is false -h Help @@ -350,8 +352,39 @@ EOF exit 0 } + +generate_p2p_connected_conf() { + local output="${1}" + local ip_params="${2}" + local template="${3}" + + local p2p_host_list="" + if [[ "${template}" == "true" ]]; then + p2p_host_list="${ip_params}" + else + local ip_array=(${ip_params//,/ }) + local ip_length=${#ip_array[@]} + + local i=0 + for (( ; i < ip_length; i++)); do + local ip=${ip_array[i]} + local delim="" + if [[ $i == $((ip_length - 1)) ]]; then + delim="" + else + delim="," + fi + p2p_host_list="${p2p_host_list}\"${ip}\"${delim}" + done + fi + + cat <"${output}" +{"nodes":[${p2p_host_list}]} +EOF +} + # generate the config.ini -generate_config_ini() { +generate_node_config_ini() { local output="${1}" local gateway_listen_ip="${2}" local gateway_listen_port="${3}" @@ -364,6 +397,7 @@ generate_config_ini() { local agency_id="${8}" local index="${9}" + local nodeid="${10}" cat <"${output}" [agency] @@ -389,6 +423,7 @@ generate_config_ini() { service.gateway_target = ; the components service.components = + nodeid=${nodeid} [crypto] sm_crypto = ${sm_mode} @@ -396,6 +431,10 @@ generate_config_ini() { [gateway] listen_ip=${gateway_listen_ip} listen_port=${gateway_listen_port} + ;the dir that contains the connected endpoint information, e.g.nodes.json + ;nodes_path=${file_dir} + ; the file that configure the connected endpoint information + ; nodes_file=${p2p_connected_conf_name} ; thread_count = 4 ; ssl or sm ssl sm_ssl=${sm_mode} @@ -408,10 +447,6 @@ generate_config_ini() { ;reconnect_time = 10000 ; the unreachable distance ;unreachable_distance=10 - ;the dir that contains the connected endpoint information, e.g.nodes.json - ;nodes_path=./ - ; the file that configure the connected endpoint information - ; nodes_path=nodes.json [rpc] listen_ip=${rpc_listen_ip} @@ -786,6 +821,7 @@ generate_private_key() { fi ${OPENSSL_CMD} genpkey -paramfile ${sm2_params} -out ${output_path}/node.pem 2>/dev/null $OPENSSL_CMD ec -in "$output_path/node.pem" -text -noout 2> /dev/null | sed -n '3,5p' | sed 's/://g' | tr "\n" " " | sed 's/ //g' | cat > "$output_path/node.privateKey" + ${OPENSSL_CMD} ec -text -noout -in "${output_path}/node.pem" 2>/dev/null | sed -n '7,11p' | tr -d ": \n" | awk '{print substr($0,3);}' | cat >"$output_path"/node.nodeid private_key=$(cat $output_path/node.privateKey) echo ${private_key} } @@ -819,6 +855,7 @@ deploy_nodes() # generate the ca-cert ca_dir="${output_dir}"/ca generate_ca_cert "${sm_mode}" "${ca_dir}" + connected_nodes="" for line in ${ip_array[*]}; do ip=${line%:*} num=${line#*:} @@ -849,7 +886,7 @@ deploy_nodes() # generate the node-script generate_node_scripts "${node_dir}" local port=$((gateway_listen_port + node_count)) - " + connected_nodes=${connected_nodes}"${ip}:${port}, " ((agency_index += 1)) set_value ${ip//./}_count $(($(get_value ${ip//./}_count) + 1)) ((++count)) @@ -875,7 +912,9 @@ deploy_nodes() local grpc_port=$((grpc_listen_port + node_count)) local agency_id="agency${count}" private_key=$(generate_private_key "${node_dir}/conf") - generate_config_ini "${node_dir}/config.ini" "${listen_ip}" "${gateway_port}" "${listen_ip}" "${rpc_port}" "${listen_ip}" "${grpc_port}" ${agency_id} "${count}" + node_id=$(cat "${node_dir}/conf/node.nodeid") + generate_node_config_ini "${node_dir}/config.ini" "${listen_ip}" "${gateway_port}" "${listen_ip}" "${rpc_port}" "${listen_ip}" "${grpc_port}" ${agency_id} "${count}" "${node_id}" + generate_p2p_connected_conf "${node_dir}/${p2p_connected_conf_name}" "${connected_nodes}" "false" set_value ${ip//./}_count $(($(get_value ${ip//./}_count) + 1)) ((++count)) done diff --git a/cpp/tools/ppc-builder/build_ppc.py b/cpp/tools/ppc-builder/build_ppc.py index e609a1c5..713d1801 100644 --- a/cpp/tools/ppc-builder/build_ppc.py +++ b/cpp/tools/ppc-builder/build_ppc.py @@ -1,11 +1,11 @@ #!/usr/bin/python # -*- coding: UTF-8 -*- # Note: here can't be refactored by autopep -import sys -sys.path.append("src/") - from controller import commandline_helper from common import utilities +import traceback +import sys +sys.path.append("src/") def main(): @@ -14,6 +14,7 @@ def main(): commandline_helper.execute_command(args) except Exception as error: utilities.log_error("%s" % error) + traceback.print_exc() if __name__ == "__main__": diff --git a/cpp/tools/ppc-builder/conf/config-example.toml b/cpp/tools/ppc-builder/conf/config-example.toml index 167afe85..7225f7e7 100644 --- a/cpp/tools/ppc-builder/conf/config-example.toml +++ b/cpp/tools/ppc-builder/conf/config-example.toml @@ -1,6 +1,6 @@ -[tars] -tars_pkg_dir = "bin/" -app_name = "ppc1" +[env] +binary_path = "bin/" +deploy_dir = "ppc1" [crypto] # the gateway-service enable ssl or not, default enable ssl @@ -27,25 +27,11 @@ holding_msg_minutes = 30 listen_port=40300 # the thread count thread_count = 4 - # the cache config for the gateway - cache_type = 0 - cache_proxy = "" - cache_obServer = "" - cache_cluster = "" - cache_host = "127.0.0.1" - cache_port = 6379 - cache_password = "" - cache_database = 1 - cache_pool_size = 16 - # the redis connection timeout, in ms, default is 500ms - cache_connection_timeout = 500 - # the redis socket timeout, in ms, default is 500ms - cache_socket_timeout = 500 - # the tars config - # gateway tars server listen ip - tars_listen_ip="0.0.0.0" - # gateway tars server listen start port - tars_listen_port=40600 + # the grpc config + # gateway grpc server listen ip + grpc_listen_ip="0.0.0.0" + # gateway grpc server listen start port + grpc_listen_port=40600 # gateway connected peers, should be all of the gateway peers info [[agency.gateway.peers]] agency = "agency0" @@ -61,10 +47,12 @@ holding_msg_minutes = 30 deploy_ip=["127.0.0.1:2"] # node name, Notice: node_name in the same agency and group must be unique node_name = "node0" - # node tars server listen ip - tars_listen_ip="0.0.0.0" - # node tars server listen port - tars_listen_port=40402 + # node grpc server listen ip + grpc_listen_ip="0.0.0.0" + # node grpc server listen port + grpc_listen_port=40402 + # the components + components = "node" # the rpc config for the node [agency.node.rpc] listen_ip = "0.0.0.0" @@ -108,7 +96,7 @@ holding_msg_minutes = 30 token = "" # the gateway config [agency.node.gateway] - tars_endpoints = ["127.0.0.1:40600", "127.0.0.1:40601"] + gateway_grpc_target = ["127.0.0.1:40600", "127.0.0.1:40601"] [[agency]] @@ -124,25 +112,11 @@ holding_msg_minutes = 30 listen_port=40320 # the thread count thread_count = 4 - # the cache config for the gateway - cache_type = 0 - cache_proxy = "" - cache_obServer = "" - cache_cluster = "" - cache_host = "127.0.0.1" - cache_port = 6379 - cache_password = "" - cache_database = 2 - cache_pool_size = 16 - # the redis connection timeout, in ms, default is 500ms - cache_connection_timeout = 500 - # the redis socket timeout, in ms, default is 500ms - cache_socket_timeout = 500 - # the tars config - # gateway tars server listen ip - tars_listen_ip="0.0.0.0" - # gateway tars server listen start port - tars_listen_port=40620 + # the grpc config + # gateway grpc server listen ip + grpc_listen_ip="0.0.0.0" + # gateway grpc server listen start port + grpc_listen_port=40620 # gateway connected peers, should be all of the gateway peers info [[agency.gateway.peers]] agency = "agency0" @@ -158,10 +132,12 @@ holding_msg_minutes = 30 deploy_ip=["127.0.0.1:2"] # node name, Notice: node_name in the same agency and group must be unique node_name = "node0" - # node tars server listen ip - tars_listen_ip="0.0.0.0" - # node tars server listen port - tars_listen_port=40422 + # node grpc server listen ip + grpc_listen_ip="0.0.0.0" + # node grpc server listen port + grpc_listen_port=40422 + # the componet + components = "node" # the rpc config for the node [agency.node.rpc] listen_ip = "0.0.0.0" @@ -205,4 +181,4 @@ holding_msg_minutes = 30 token = "" # the gateway config [agency.node.gateway] - tars_endpoints = ["127.0.0.1:40620", "127.0.0.1:40621"] + gateway_grpc_target = ["127.0.0.1:40620", "127.0.0.1:40621"] diff --git a/cpp/tools/ppc-builder/src/common/utilities.py b/cpp/tools/ppc-builder/src/common/utilities.py index 34679544..5e4b3c32 100644 --- a/cpp/tools/ppc-builder/src/common/utilities.py +++ b/cpp/tools/ppc-builder/src/common/utilities.py @@ -15,14 +15,6 @@ class ServiceInfo: ssl_file_list = ["ca.crt", "ssl.key", "ssl.crt"] sm_ssl_file_list = ["sm_ca.crt", "sm_ssl.key", "sm_ssl.crt", "sm_enssl.key", "sm_enssl.crt"] - - node_service_postfix = "NodeService" - gateway_servant = "GatewayService" - gateway_servant_obj = ["GatewayServiceObj"] - - node_servant = ["FrontService"] - node_servant_object = ["FrontServiceObj"] - cert_generation_script_path = "src/scripts/gen_cert.sh" node_service_type = "node" gateway_service_type = "gateway" @@ -31,29 +23,26 @@ class ServiceInfo: class ConfigInfo: config_ini_file = "config.ini" - tars_config_file = "tars.conf" - tpl_abs_path = "src/tpl/" pwd_path = os.getcwd() node_config_tpl_path = os.path.join( pwd_path, tpl_abs_path, "config.ini.node") gateway_config_tpl_path = os.path.join( pwd_path, tpl_abs_path, "config.ini.gateway") - tars_config_tpl_path = os.path.join(pwd_path, tpl_abs_path, "tars.conf") - - tars_start_tpl_path = os.path.join( - pwd_path, tpl_abs_path, "tars_start.sh") - tars_stop_tpl_path = os.path.join( - pwd_path, tpl_abs_path, "tars_stop.sh") - - tars_start_all_tpl_path = os.path.join( - pwd_path, tpl_abs_path, "tars_start_all.sh") - tars_stop_all_tpl_path = os.path.join( - pwd_path, tpl_abs_path, "tars_stop_all.sh") ppc_gateway_binary_name = "ppc-gateway-service" ppc_node_binary_name = "ppc-pro-node" + start_tpl_path = os.path.join( + pwd_path, tpl_abs_path, "start.sh") + stop_tpl_path = os.path.join( + pwd_path, tpl_abs_path, "stop.sh") + + start_all_tpl_path = os.path.join( + pwd_path, tpl_abs_path, "start_all.sh") + stop_all_tpl_path = os.path.join( + pwd_path, tpl_abs_path, "stop_all.sh") + class CommandInfo: generate_config = "genconfig" @@ -123,10 +112,6 @@ def mkfiledir(filepath): mkdir(parent_dir) -def generate_service_name(prefix, service_name): - return prefix + service_name - - def convert_bool_to_str(value): if value is True: return "true" @@ -162,7 +147,12 @@ def store_config(config_content, config_type, config_path, desc): mkdir(os.path.dirname(config_path)) with open(config_path, 'w') as configFile: - config_content.write(configFile) + if isinstance(config_content, str): + configFile.write(config_content) + elif isinstance(config_content, bytes): + configFile.write(config_content) + else: + config_content.write(configFile) log_info("* store %s config for %s success" % (config_type, desc)) return True @@ -176,14 +166,3 @@ def load_config(file_path): config_content.optionxform = str config_content.read(file_path) return config_content - - -def check_service_name(tag, service_name): - """ - Note: tars service name can only contain letters and numbers - """ - service_name_len = len(service_name) - ret = re.search(r'^[A-Za-z0-9]+', service_name).span() - if ret is None or (ret[0] != 0 or ret[1] != service_name_len): - raise Exception( - "the %s must be letters|numbers, invalid value: %s" % (tag, service_name)) diff --git a/cpp/tools/ppc-builder/src/config/cert_generator.py b/cpp/tools/ppc-builder/src/config/cert_generator.py index 77ae0fb9..8521ee8b 100644 --- a/cpp/tools/ppc-builder/src/config/cert_generator.py +++ b/cpp/tools/ppc-builder/src/config/cert_generator.py @@ -53,5 +53,5 @@ def generate_private_key(output_path): "* generate private key error, output_path: %s" % output_path) return (False, output) utilities.log_info( - "* generate private_key success, path: %s, private_key: %s" % (output_path, output)) + "* generate private_key success, path: %s, public_key: %s" % (output_path, output)) return (True, output) diff --git a/cpp/tools/ppc-builder/src/config/ppc_deploy_config.py b/cpp/tools/ppc-builder/src/config/ppc_deploy_config.py index 285baf34..bdaffb85 100644 --- a/cpp/tools/ppc-builder/src/config/ppc_deploy_config.py +++ b/cpp/tools/ppc-builder/src/config/ppc_deploy_config.py @@ -4,40 +4,32 @@ from common import utilities -class TarsConfig: - """ - the tars config - """ - - def __init__(self, config): - self.config = config - self.app_name = utilities.get_value( - self.config, "tars", "app_name", None, True) - self.binary_path = utilities.get_value( - self.config, "tars", "tars_pkg_dir", None, True) - - class PeerInfo: def __init__(self, agency, endpoints): self.agency = agency self.endpoints = endpoints +class EnvConfig: + def __init__(self, config, section_name): + self.config = config + self.section_name = section_name + self.binary_path = utilities.get_value( + self.config, self.section_name, "binary_path", None, True) + self.deploy_dir = utilities.get_value( + self.config, self.section_name, "deploy_dir", None, True) + + class GatewayConfig: """ the gateway config """ - def __init__(self, app_name, agency_name, holding_msg_minutes, config, config_section, must_exist): + def __init__(self, agency_name, holding_msg_minutes, config, config_section, must_exist): self.config = config self.config_section = config_section - self.app_name = app_name self.holding_msg_minutes = holding_msg_minutes self.agency_name = agency_name - self.service_name = "%s%s" % ( - self.agency_name, utilities.ServiceInfo.gateway_servant) - self.servant_list = [utilities.ServiceInfo.gateway_servant] - self.servant_object_list = utilities.ServiceInfo.gateway_servant_obj # the deploy_ip self.deploy_ip = utilities.get_item_value( @@ -62,34 +54,12 @@ def __init__(self, app_name, agency_name, holding_msg_minutes, config, config_se peer, "endpoints", None, must_exist, "[[peers]]") self.peers.append(PeerInfo(agency, endpoints)) - self.cache_type = utilities.get_item_value( - self.config, "cache_type", 0, must_exist, config_section) - self.cache_proxy = utilities.get_item_value( - self.config, "cache_proxy", None, must_exist, config_section) - self.cache_obServer = utilities.get_item_value( - self.config, "cache_obServer", None, must_exist, config_section) - self.cache_cluster = utilities.get_item_value( - self.config, "cache_cluster", None, must_exist, config_section) - self.cache_host = utilities.get_item_value( - self.config, "cache_host", None, must_exist, config_section) - self.cache_port = utilities.get_item_value( - self.config, "cache_port", None, must_exist, config_section) - self.cache_password = utilities.get_item_value( - self.config, "cache_password", "", must_exist, config_section) - self.cache_database = utilities.get_item_value( - self.config, "cache_database", None, must_exist, config_section) - self.cache_pool_size = utilities.get_item_value( - self.config, "cache_pool_size", 16, False, config_section) - self.cache_connection_timeout = utilities.get_item_value( - self.config, "cache_connection_timeout", 500, False, config_section) - self.cache_socket_timeout = utilities.get_item_value( - self.config, "cache_socket_timeout", 500, False, config_section) - # the tars_listen_ip - self.tars_listen_ip = utilities.get_item_value( - self.config, "tars_listen_ip", "0.0.0.0", False, config_section) - # the tars_listen_port - self.tars_listen_port = utilities.get_item_value( - self.config, "tars_listen_port", None, must_exist, config_section) + # the grpc_listen_ip + self.grpc_listen_ip = utilities.get_item_value( + self.config, "grpc_listen_ip", "0.0.0.0", False, config_section) + # the grpc_listen_port + self.grpc_listen_port = utilities.get_item_value( + self.config, "grpc_listen_port", None, must_exist, config_section) class RpcConfig: @@ -183,16 +153,15 @@ class NodeGatewayConfig: the gateway config for the node """ - def __init__(self, app_name, agency_name, config, node_must_exists): + def __init__(self, agency_name, config, node_must_exists): self.config = config - self.app_name = app_name self.agency_name = agency_name self.desc = "[agency.node]" - self.endpoints = utilities.get_item_value( - self.config, "tars_endpoints", None, node_must_exists, self.desc) - # obtain the gateway name - self.service_name = "%s.%s%s" % ( - self.app_name, self.agency_name, utilities.ServiceInfo.gateway_servant) + self.gateway_grpc_target_array = utilities.get_item_value( + self.config, "gateway_grpc_target", None, node_must_exists, self.desc) + self.gateway_grpc_target = "ipv4:" + self.gateway_grpc_target += ','.join( + map(str, self.gateway_grpc_target_array)) class NodeConfig: @@ -200,28 +169,30 @@ class NodeConfig: the ppc-node config """ - def __init__(self, app_name, agency_name, holding_msg_minutes, config, must_exist): + def __init__(self, agency_name, holding_msg_minutes, config, must_exist): self.config = config self.section_name = "[[agency.node]]." self.holding_msg_minutes = holding_msg_minutes - self.app_name = app_name # set the agency_name self.agency_name = agency_name # disable ra2018 or not, default enable the ra2018 self.disable_ra2018 = utilities.get_item_value( self.config, "disable_ra2018", False, False, self.section_name) + # the components + self.components = utilities.get_item_value( + self.config, "components", None, False, self.section_name) # the deploy_ip self.deploy_ip = utilities.get_item_value( self.config, "deploy_ip", None, must_exist, self.section_name) # the node_name self.node_name = utilities.get_item_value( self.config, "node_name", None, must_exist, self.section_name) - # the tars_listen_ip - self.tars_listen_ip = utilities.get_item_value( - self.config, "tars_listen_ip", "0.0.0.0", False, self.section_name) - # the tars_listen_port - self.tars_listen_port = utilities.get_item_value( - self.config, "tars_listen_port", None, must_exist, self.section_name) + # the grpc_listen_ip + self.grpc_listen_ip = utilities.get_item_value( + self.config, "grpc_listen_ip", "0.0.0.0", False, self.section_name) + # the grpc_listen_port + self.grpc_listen_port = utilities.get_item_value( + self.config, "grpc_listen_port", None, must_exist, self.section_name) utilities.log_debug("load the node config success") # parse the rpc config @@ -276,24 +247,16 @@ def __init__(self, app_name, agency_name, holding_msg_minutes, config, must_exis self.gateway_config = None if gateway_config_object is not None: self.gateway_config = NodeGatewayConfig( - self.app_name, self.agency_name, gateway_config_object, must_exist) + self.agency_name, gateway_config_object, must_exist) utilities.log_debug("load the gateway success") - # set the server name - self.service_name = "%s%s%s" % ( - self.agency_name, self.node_name, utilities.ServiceInfo.node_service_postfix) - # set the servant name - self.servant_list = utilities.ServiceInfo.node_servant - self.servant_object_list = utilities.ServiceInfo.node_servant_object - class AgencyConfig: """ the agency config """ - def __init__(self, app_name, config, gateway_must_exists, node_must_exists): - self.app_name = app_name + def __init__(self, config, gateway_must_exists, node_must_exists): self.config = config self.section_name = "[[agency]]" # the agency-name @@ -309,9 +272,8 @@ def __init__(self, app_name, config, gateway_must_exists, node_must_exists): self.config, "gateway", None, gateway_must_exists, gateway_config_section_name) self.gateway_config = None if gateway_config_object is not None: - self.gateway_config = GatewayConfig( - self.app_name, self.agency_name, self.holding_msg_minutes, gateway_config_object, - gateway_config_section_name, gateway_must_exists) + self.gateway_config = GatewayConfig(self.agency_name, self.holding_msg_minutes, gateway_config_object, + gateway_config_section_name, gateway_must_exists) utilities.log_debug("load the gateway config success") # parse the node config @@ -323,7 +285,7 @@ def __init__(self, app_name, config, gateway_must_exists, node_must_exists): # TODO: check the node-name for node_object in node_config_list: node_config = NodeConfig( - self.app_name, self.agency_name, self.holding_msg_minutes, node_object, node_must_exists) + self.agency_name, self.holding_msg_minutes, node_object, node_must_exists) self.node_list[node_config.node_name] = node_config utilities.log_debug( "load node config for %s success" % node_config.node_name) @@ -353,8 +315,7 @@ def __init__(self, config, gateway_must_exists, node_must_exists): self.sm_crypto = utilities.get_value( self.config, crypto_section, "sm_crypto", False, False) utilities.log_debug("load the crypto config success") - # load the tars config - self.tars_config = TarsConfig(self.config) + self.env_config = EnvConfig(self.config, "env") # load the agency config # TODO: check duplicated case utilities.log_debug("load the agency config") @@ -363,7 +324,7 @@ def __init__(self, config, gateway_must_exists, node_must_exists): self.config, "agency", None, False, "[[agency]]") for agency_object in agency_list_object: agency_config = AgencyConfig( - self.tars_config.app_name, agency_object, gateway_must_exists, node_must_exists) + agency_object, gateway_must_exists, node_must_exists) self.agency_list[agency_config.agency_name] = agency_config utilities.log_debug( "load the agency config for %s success" % agency_config.agency_name) diff --git a/cpp/tools/ppc-builder/src/config/ppc_gateway_config_generator.py b/cpp/tools/ppc-builder/src/config/ppc_gateway_config_generator.py index b75f7987..a92eea28 100644 --- a/cpp/tools/ppc-builder/src/config/ppc_gateway_config_generator.py +++ b/cpp/tools/ppc-builder/src/config/ppc_gateway_config_generator.py @@ -1,10 +1,11 @@ #!/usr/bin/python # -*- coding: UTF-8 -*- import os +import json from common import utilities from config.binary_generator import BinaryGenerator from config.cert_generator import CertGenerator -from config.tars_config_generator import TarsConfigGenerator +from config.shell_script_generator import ShellScriptGenerator class PPCGatewayConfigGenerator: @@ -15,138 +16,123 @@ class PPCGatewayConfigGenerator: def __init__(self, config, output_dir): self.config = config self.output_dir = output_dir - self.with_tars = False self.binary_name = utilities.ConfigInfo.ppc_gateway_binary_name self.service_type = utilities.ServiceInfo.gateway_service_type def generate_gateway_config(self): - utilities.print_badge("* generate gateway config, app: %s" % - self.config.tars_config.app_name) + utilities.print_badge("* generate gateway config, deploy_dir: %s" % + self.config.env_config.deploy_dir) # generate the ca ret = CertGenerator.generate_ca_cert( self.config.gateway_sm_ssl, self.__generate_ca_cert_path__()) if ret is False: utilities.log_error( - "* generate ca-cert config for %s failed" % self.config.tars_config.app_name) + "* generate ca-cert config for %s failed" % self.config.env_config.deploy_dir) return False + connection_config = {} + connection_config["nodes"] = [] + connection_config_path_list = [] for agency_config in self.config.agency_list.values(): gateway_config = agency_config.gateway_config - ret = self.__generate_single_gateway_config__(gateway_config) + ret = self.__generate_single_gateway_config__( + gateway_config, connection_config, connection_config_path_list) if ret is False: return False + self.__generate_connection_info__( + connection_config_path_list, connection_config) utilities.print_badge( - "* generate gateway config success, app: %s" % self.config.tars_config.app_name) + "* generate gateway config success, deploy_dir: %s" % self.config.env_config.deploy_dir) return True - def __generate_single_gateway_config__(self, gateway_config): + def __generate_single_gateway_config__(self, gateway_config, connection_config, connection_config_path_list): # load the config from tpl_config_path - utilities.log_info("* generate config for ppc-gateway %s.%s" % - (gateway_config.app_name, gateway_config.service_name)) - # store the - # store the config.ini + utilities.log_info("* generate config for ppc-gateway") for ip_str in gateway_config.deploy_ip: ip_array = ip_str.split(":") ip = ip_array[0] + # generate the shell scripts for the given ip + ret = ShellScriptGenerator.generate_ip_shell_scripts( + self.__generate_ip_shell_scripts_output_path__(gateway_config.agency_name, ip), "start_all.sh", "stop_all.sh") + if ret is False: + return False node_count = 1 if len(ip_array) >= 2: node_count = int(ip_array[1]) for node_index in range(node_count): node_name = "node" + str(node_index) - utilities.print_badge("* generate config for ppc-gateway %s.%s.%s" % - (gateway_config.app_name, gateway_config.service_name, node_name)) + connection_config_path_list.append(os.path.join(self.__generate_node_path__( + gateway_config.agency_name, ip, node_name), "nodes.json")) + utilities.print_badge( + "* generate config for ppc-gateway %s.%s, deploy_ip: %s" % (gateway_config.agency_name, node_name, ip)) config_content = utilities.load_config( utilities.ConfigInfo.gateway_config_tpl_path) # load the common config self.__generate_common_config__(gateway_config, config_content) # load the gateway config listen_port = gateway_config.listen_port + node_index + connection_config["nodes"].append(ip + ":" + str(listen_port)) + grpc_listen_port = gateway_config.grpc_listen_port + node_index self.__generate_gateway_config_content__( - gateway_config, config_content, listen_port) - # load the cache config - self.__generate_cache_config__(gateway_config, config_content) - # generate the shell scripts for the given ip - ret = TarsConfigGenerator.generate_ip_shell_scripts( - self.__generate_ip_shell_scripts_output_path__(ip), "start_all.sh", "stop_all.sh") - if ret is False: - return False - # generate the start.sh/stop.sh for given service - ret = TarsConfigGenerator.generate_ip_shell_scripts( - self.__generate_ip_shell_scripts_for_given_service__(ip, gateway_config.service_name), "start.sh", - "stop.sh") - if ret is False: - return False + gateway_config, config_content, listen_port, grpc_listen_port) # generate the binary binary_path = os.path.join( - self.config.tars_config.binary_path, self.binary_name) + self.config.env_config.binary_path, self.binary_name) dst_binary_path = os.path.join( - self.__generate_ip_shell_scripts_for_given_service__(ip, gateway_config.service_name), + self.__generate_ip_shell_scripts_output_path__( + gateway_config.agency_name, ip), self.binary_name) ret = BinaryGenerator.generate_binary( binary_path, dst_binary_path) if ret is False: return False - # generate the shell scripts for the node - service_name = gateway_config.service_name - if self.with_tars is False: - service_name = utilities.ConfigInfo.ppc_gateway_binary_name - ret = TarsConfigGenerator.generate_node_shell_scripts( - self.__generate_node_path__(gateway_config, ip, node_name), service_name) - if ret is False: - return False # generate the ini config - config_output_path = self.__generate_conf_output_path__( - gateway_config, ip, node_name) ini_config_output_path = os.path.join( - config_output_path, utilities.ConfigInfo.config_ini_file) + self.__generate_node_path__(gateway_config.agency_name, ip, node_name), utilities.ConfigInfo.config_ini_file) ret = utilities.store_config( - config_content, "ini", ini_config_output_path, gateway_config.service_name) - if ret is False: - utilities.log_error("* generate config for ppc-gateway %s.%s, ip: %s failed" % - (gateway_config.app_name, gateway_config.service_name, ip)) - return False - # generate tars config - tars_config_path = os.path.join( - config_output_path, utilities.ConfigInfo.tars_config_file) - tars_listen_port = gateway_config.tars_listen_port + node_index - ret = TarsConfigGenerator.generate_and_store_tars_conf(utilities.ConfigInfo.tars_config_tpl_path, - tars_config_path, - gateway_config.app_name, - gateway_config.service_name, - gateway_config.servant_object_list, ip, - tars_listen_port) + config_content, "ini", ini_config_output_path, "config.ini") if ret is False: + utilities.log_error( + "* generate config for ppc-gateway, ip: %s failed" % (ip)) return False # generate the node config ret = CertGenerator.generate_node_cert(self.config.gateway_sm_ssl, self.__generate_ca_cert_path__( - ), self.__generate_conf_output_path__(gateway_config, ip, node_name)) + ), self.__generate_conf_output_path__(gateway_config.agency_name, ip, node_name)) if ret is False: utilities.log_error( - "* generate config for ppc-gateway %s.%s failed for generate the node config failed" % - (gateway_config.app_name, gateway_config.service_name)) + "* generate config for ppc-gateway failed for generate the node config failed") return False - utilities.print_badge("* generate config for ppc-gateway %s.%s.%s success" % - (gateway_config.app_name, gateway_config.service_name, node_name)) - utilities.log_info("* generate config for ppc-gateway %s.%s success" % - (gateway_config.app_name, gateway_config.service_name)) + ret = ShellScriptGenerator.generate_node_shell_scripts(self.__generate_node_path__( + gateway_config.agency_name, ip, node_name), self.binary_name) + if ret is False: + return False + utilities.print_badge("* generate config for ppc-gateway%s success" % + (node_name)) + utilities.log_info("* generate config for ppc-gateway success") + return True + + def __generate_connection_info__(self, connection_config_path_list, connection_config): + connection_str = json.dumps(connection_config) + for path in connection_config_path_list: + ret = utilities.store_config( + connection_str, "json", path, "nodes.json") + if ret is False: + utilities.log_error( + "* generate nodes.json failed, path: %s" % path) + return False return True def __generate_ca_cert_path__(self): - return os.path.join(self.output_dir, self.config.tars_config.app_name, "ca", self.service_type) + return os.path.join(self.output_dir, self.service_type, "ca") - def __generate_node_path__(self, config, ip, node_name): - return os.path.join(self.output_dir, self.config.tars_config.app_name, ip, self.service_type, - config.service_name, node_name) + def __generate_node_path__(self, agency_name, ip, node_name): + return os.path.join(self.output_dir, agency_name, ip, self.service_type, node_name) - def __generate_conf_output_path__(self, config, ip, node_name): - node_path = self.__generate_node_path__(config, ip, node_name) + def __generate_conf_output_path__(self, agency_name, ip, node_name): + node_path = self.__generate_node_path__(agency_name, ip, node_name) return os.path.join(node_path, "conf") - def __generate_ip_shell_scripts_output_path__(self, ip): - return os.path.join(self.output_dir, self.config.tars_config.app_name, ip, self.service_type) - - def __generate_ip_shell_scripts_for_given_service__(self, ip, service_name): - ip_path = self.__generate_ip_shell_scripts_output_path__(ip) - return os.path.join(ip_path, service_name) + def __generate_ip_shell_scripts_output_path__(self, agency_name, ip): + return os.path.join(self.output_dir, agency_name, ip, self.service_type) def __generate_common_config__(self, config, config_content): """ @@ -157,12 +143,8 @@ def __generate_common_config__(self, config, config_content): config_content[section]["id"] = config.agency_name config_content["gateway"]["holding_msg_minutes"] = str( config.holding_msg_minutes) - # the peers info - for peer in config.peers: - key = "agency." + peer.agency - config_content[section][key] = str(','.join(peer.endpoints)) - def __generate_gateway_config_content__(self, config, config_content, listen_port): + def __generate_gateway_config_content__(self, config, config_content, listen_port, grpc_listen_port): """ generate the gateway config """ @@ -179,29 +161,7 @@ def __generate_gateway_config_content__(self, config, config_content, listen_por # disable ssl config_content[section]["disable_ssl"] = utilities.convert_bool_to_str( self.config.gateway_disable_ssl) - - def __generate_cache_config__(self, config, config_content): - """ - generate the cache config - """ - section = "cache" - config_content[section]["type"] = config.cache_type - config_content[section]["proxy"] = str(config.cache_proxy) - config_content[section]["obServer"] = str(config.cache_obServer) - config_content[section]["cluster"] = str(config.cache_cluster) - # the host - config_content[section]["host"] = config.cache_host - # the port - config_content[section]["port"] = str(config.cache_port) - # the password - config_content[section]["password"] = str(config.cache_password) - # the database - config_content[section]["database"] = str(config.cache_database) - # the pool_size - config_content[section]["pool_size"] = str(config.cache_pool_size) - # the connection_timeout - config_content[section]["connection_timeout"] = str( - config.cache_connection_timeout) - # the socket_timeout - config_content[section]["socket_timeout"] = str( - config.cache_socket_timeout) + section = "transport" + config_content[section]["listen_ip"] = config.grpc_listen_ip + # the listen port + config_content[section]["listen_port"] = str(config.grpc_listen_port) diff --git a/cpp/tools/ppc-builder/src/config/ppc_node_config_generator.py b/cpp/tools/ppc-builder/src/config/ppc_node_config_generator.py index b3962b4c..3e9d8c2c 100644 --- a/cpp/tools/ppc-builder/src/config/ppc_node_config_generator.py +++ b/cpp/tools/ppc-builder/src/config/ppc_node_config_generator.py @@ -1,10 +1,11 @@ #!/usr/bin/python # -*- coding: UTF-8 -*- import os +import sys from common import utilities from config.binary_generator import BinaryGenerator from config.cert_generator import CertGenerator -from config.tars_config_generator import TarsConfigGenerator +from config.shell_script_generator import ShellScriptGenerator class PPCNodeConfigGenerator: @@ -15,7 +16,6 @@ class PPCNodeConfigGenerator: def __init__(self, config, output_dir): self.config = config self.output_dir = output_dir - self.with_tars = False self.binary_name = utilities.ConfigInfo.ppc_node_binary_name self.service_type = utilities.ServiceInfo.node_service_type @@ -26,83 +26,74 @@ def generate_node_config(self): self.config.rpc_sm_ssl, self.__generate_ca_cert_path__()) if ret is False: utilities.log_error( - "* generate_node_config failed for generate ca error, app: %s" % (self.config.tars_config.app_name)) + "* generate_node_config failed for generate ca error") return False for agency_config in self.config.agency_list.values(): for node_config in agency_config.node_list.values(): for ip_str in node_config.deploy_ip: + if ret is False: + return False ip_array = ip_str.split(":") ip = ip_array[0] + # generate the shell scripts for the given ip + ret = ShellScriptGenerator.generate_ip_shell_scripts( + self.__generate_ip_output_path__(node_config.agency_name, ip), "start_all.sh", "stop_all.sh") node_count = 1 if len(ip_array) >= 2: node_count = int(ip_array[1]) for node_index in range(node_count): node_name = "node" + str(node_index) - if self.__generate_single_node_config__(node_config, ip, node_name, node_index) is False: + if self.__generate_single_node_config__(node_config, ip, node_name, node_config.agency_name, node_index) is False: return False utilities.print_badge("* generate_node_config success") return True - def __generate_single_node_config__(self, node_config, ip, node_name, node_index): - utilities.print_badge("* generate node config for %s.%s.%s, ip: %s" % - (node_config.app_name, node_config.service_name, node_name, ip)) - # generate the shell scripts for the given ip - ret = TarsConfigGenerator.generate_ip_shell_scripts( - self.__generate_ip_output_path__(ip), "start_all.sh", "stop_all.sh") - if ret is False: - return False - # generate the start.sh/stop.sh for given service - ret = TarsConfigGenerator.generate_ip_shell_scripts( - self.__generate_ip_shell_scripts_for_given_service__(ip, node_config.service_name), "start.sh", "stop.sh") - if ret is False: - return False + def __generate_single_node_config__(self, node_config, ip, node_name, agency_name, node_index): + utilities.print_badge("* generate node config for %s, ip: %s, agency: %s" % + (node_name, ip, agency_name)) # copy the binary binary_path = os.path.join( - self.config.tars_config.binary_path, self.binary_name) + self.config.env_config.binary_path, self.binary_name) dst_binary_path = os.path.join( - self.__generate_ip_shell_scripts_for_given_service__(ip, node_config.service_name), self.binary_name) + self.__generate_ip_output_path__(agency_name, ip), self.binary_name) ret = BinaryGenerator.generate_binary(binary_path, dst_binary_path) if ret is False: return False # generate the node config - node_path = self.__generate_node_conf_path__( - node_config, ip, node_name) + node_path = self.__generate_node_path__( + agency_name, ip, node_name) + private_key_path = self.__generate_node_conf_path__( - node_config, ip, node_name) - tars_output_path = private_key_path + agency_name, ip, node_name) if self.__generate_single_node_inner_config__(utilities.ConfigInfo.node_config_tpl_path, - utilities.ConfigInfo.tars_config_tpl_path, node_path, - private_key_path, tars_output_path, node_config, ip, + node_path, + private_key_path, node_config, ip, node_index) is False: - utilities.log_error("* generate node config for %s.%s, ip: %s failed" % - (node_config.app_name, node_config.service_name, ip)) + utilities.log_error("* generate node config, ip: %s failed" % + (ip)) return False - # generate the node shell_scipts - service_name = node_config.service_name - if self.with_tars is False: - service_name = utilities.ConfigInfo.ppc_node_binary_name - ret = TarsConfigGenerator.generate_node_shell_scripts( - self.__generate_node_path__(node_config, ip, node_name), service_name) - if ret is False: - return False # generate the node cert(for rpc) ret = CertGenerator.generate_node_cert(self.config.rpc_sm_ssl, self.__generate_ca_cert_path__( - ), self.__generate_node_conf_path__(node_config, ip, node_name)) + ), self.__generate_node_conf_path__(agency_name, ip, node_name)) + if ret is False: + utilities.log_error("* generate node config, ip: %s failed for generate rpc cert failed" % + (ip)) + return False + ret = ShellScriptGenerator.generate_node_shell_scripts( + self.__generate_node_path__(agency_name, ip, node_name), self.binary_name) if ret is False: - utilities.log_error("* generate node config for %s.%s, ip: %s failed for generate rpc cert failed" % - (node_config.app_name, node_config.service_name, ip)) return False - utilities.print_badge("* generate node config for %s.%s.%s, ip: %s success" % - (node_config.app_name, node_config.service_name, node_name, ip)) + utilities.print_badge("* generate node config %s, ip: %s.%s success" % + (node_name, agency_name, ip)) return True - def __generate_single_node_inner_config__(self, tpl_config_path, tars_config_tpl_path, node_path, private_key_path, - tars_output_path, node_config, ip, node_index): + def __generate_single_node_inner_config__(self, tpl_config_path, node_path, private_key_path, + node_config, ip, node_index): config_content = utilities.load_config(tpl_config_path) utilities.log_debug( - "__generate_single_node_config__, load config.ini from %s" % tpl_config_path) + "__generate_single_node_inner_config__, load config.ini from %s" % tpl_config_path) # generate the private key - (ret, private_key) = CertGenerator.generate_private_key(private_key_path) + (ret, node_id) = CertGenerator.generate_private_key(private_key_path) if ret is False: return False # load the common config @@ -111,7 +102,9 @@ def __generate_single_node_inner_config__(self, tpl_config_path, tars_config_tpl # load the rpc config self.__generate_rpc_config__( config_content, node_config.rpc_config, node_index) - # TODO: check the configuration + # load the transport config + self.__generate_transport_config__(config_content, + node_config, node_id, ip) # load the storage config self.__generate_storage_config__( config_content, node_config.storage_config) @@ -121,38 +114,28 @@ def __generate_single_node_inner_config__(self, tpl_config_path, tars_config_tpl # load the ra2018psi config self.__generate_ra2018psi_config__( config_content, node_config.ra2018psi_config) - # load the gateway tars config - self.__generate_tars_gateway_config__( - config_content, node_config.gateway_config) # store the config ini_config_output_path = os.path.join( node_path, utilities.ConfigInfo.config_ini_file) ret = utilities.store_config( - config_content, "ini", ini_config_output_path, node_config.service_name) + config_content, "ini", ini_config_output_path, "config.ini") if ret is False: return False - # load the tars config - self.__generate_tars_config__( - tars_config_tpl_path, tars_output_path, node_config, ip, node_index) return True def __generate_ca_cert_path__(self): - return os.path.join(self.output_dir, self.config.tars_config.app_name, "ca", self.service_type) + return os.path.join(self.output_dir, self.service_type, "ca") - def __generate_node_path__(self, node_config, ip, node_name): - return os.path.join(self.output_dir, self.config.tars_config.app_name, ip, self.service_type, - node_config.service_name, node_name) + def __generate_node_path__(self, agency_name, ip, node_name): + return os.path.join(self.output_dir, agency_name, ip, self.service_type, + node_name) - def __generate_node_conf_path__(self, node_config, ip, node_name): - node_path = self.__generate_node_path__(node_config, ip, node_name) + def __generate_node_conf_path__(self, agency_name, ip, node_name): + node_path = self.__generate_node_path__(agency_name, ip, node_name) return os.path.join(node_path, "conf") - def __generate_ip_output_path__(self, ip): - return os.path.join(self.output_dir, self.config.tars_config.app_name, ip, self.service_type) - - def __generate_ip_shell_scripts_for_given_service__(self, ip, service_name): - ip_path = self.__generate_ip_output_path__(ip) - return os.path.join(ip_path, service_name) + def __generate_ip_output_path__(self, agency_name, ip): + return os.path.join(self.output_dir, agency_name, ip, self.service_type) def __generate_common_config__(self, config_content, node_config): """ @@ -160,9 +143,6 @@ def __generate_common_config__(self, config_content, node_config): """ # the agency config config_content["agency"]["id"] = node_config.agency_name - # the holding_msg_minutes - config_content["tars_gateway"]["holding_msg_minutes"] = str( - node_config.holding_msg_minutes) # disable ra2018 or not config_content["agency"]["disable_ra2018"] = utilities.convert_bool_to_str( node_config.disable_ra2018) @@ -212,6 +192,31 @@ def __generate_hdfs_storage_config__(self, config_content, hdfs_storage_config): hdfs_storage_config.name_node_port) config_content[section_name]["token"] = hdfs_storage_config.token + def __generate_transport_config__(self, config_content, node_config, node_id, deploy_ip): + """_summary_ + + Args: + ; the endpoint information + listen_ip = 0.0.0.0 + listen_port = 18000 + host_ip = + ; the threadPoolSize + thread_count = 4 + ; the gatewayService endpoint information + service.gateway_target = + ; the components + service.components = + nodeid= + """ + section = "transport" + config_content[section]["listen_ip"] = node_config.grpc_listen_ip + config_content[section]["listen_port"] = str( + node_config.grpc_listen_port) + config_content[section]["host_ip"] = deploy_ip + config_content[section]["service.gateway_target"] = node_config.gateway_config.gateway_grpc_target + config_content[section]["service.components"] = node_config.components + config_content[section]["nodeid"] = node_id + def __generate_ra2018psi_config__(self, config_content, ra2018psi_config): """ generate the ra2018psi config @@ -236,24 +241,3 @@ def __generate_ra2018psi_config__(self, config_content, ra2018psi_config): ra2018psi_config.data_batch_size) config_content[section_name]["use_hdfs"] = utilities.convert_bool_to_str( ra2018psi_config.use_hdfs) - - def __generate_tars_gateway_config__(self, config_content, tars_gateway_config): - section_name = "tars_gateway" - config_content[section_name]["name"] = tars_gateway_config.service_name - i = 0 - for endpoint in tars_gateway_config.endpoints: - key = "proxy.%d" % i - config_content[section_name][key] = endpoint - i = i + 1 - - def __generate_tars_config__(self, tars_config_tpl_path, tars_config_dir, node_config, ip, node_index): - """ - generate the tars config - """ - tars_config_path = os.path.join( - tars_config_dir, utilities.ConfigInfo.tars_config_file) - # generate the tars config of the gateway-service - tars_listen_port = node_config.tars_listen_port + node_index - TarsConfigGenerator.generate_and_store_tars_conf(tars_config_tpl_path, tars_config_path, - node_config.app_name, node_config.service_name, - node_config.servant_object_list, ip, tars_listen_port) diff --git a/cpp/tools/ppc-builder/src/config/shell_script_generator.py b/cpp/tools/ppc-builder/src/config/shell_script_generator.py new file mode 100644 index 00000000..557e03ff --- /dev/null +++ b/cpp/tools/ppc-builder/src/config/shell_script_generator.py @@ -0,0 +1,75 @@ +#!/usr/bin/python +# -*- coding: UTF-8 -*- +from common import utilities +import os + + +class ShellScriptGenerator: + """ + generate the shell-scripts + """ + def generate_ip_shell_scripts(script_output_dir, start_shell_script_name, stop_shell_script_name): + start_all_path = os.path.join( + script_output_dir, start_shell_script_name) + utilities.mkdir(script_output_dir) + if os.path.exists(start_all_path) is False: + utilities.log_debug( + "* generate shell script, dst: %s" % start_all_path) + # start_all.sh + command = "cp %s %s" % ( + utilities.ConfigInfo.start_all_tpl_path, start_all_path) + (result, output) = utilities.execute_command_and_getoutput(command) + if result is False: + utilities.log_error( + "* generate %s failed, error: %s" % (start_all_path, output)) + return False + stop_all_path = os.path.join( + script_output_dir, stop_shell_script_name) + if os.path.exists(stop_all_path) is False: + # tars stop_all.sh + command = "cp %s %s" % ( + utilities.ConfigInfo.stop_all_tpl_path, stop_all_path) + (result, output) = utilities.execute_command_and_getoutput(command) + if result is False: + utilities.log_error( + "* generate %s failed, error: %s" % (stop_all_path, output)) + return False + utilities.log_debug( + "* generate_ip_shell_scripts success, output: %s" % script_output_dir) + return True + + def __update_binary__(file_path, output_path, binary_name): + command = "cp %s %s" % (file_path, output_path) + (ret, _) = utilities.execute_command_and_getoutput(command) + if ret is False: + return False + updated_config = "" + with open(output_path, 'r', encoding='UTF-8') as file: + updated_config = file.read() + updated_config = updated_config.replace( + "@BINARY_NAME@", binary_name) + with open(output_path, 'w', encoding='UTF-8') as file: + file.write(updated_config) + return True + + def generate_node_shell_scripts(script_output_dir, binary_name): + utilities.log_debug("* generate shell scripts for %s, dst: %s" % + (binary_name, script_output_dir)) + utilities.mkdir(script_output_dir) + # the start.sh + output_path = os.path.join(script_output_dir, "start.sh") + ret = ShellScriptGenerator.__update_binary__( + utilities.ConfigInfo.start_tpl_path, output_path, binary_name) + if ret is False: + utilities.log_error( + "generate_node_shell_scripts %s error" % output_path) + return False + # the stop.sh + output_path = os.path.join(script_output_dir, "stop.sh") + ret = ShellScriptGenerator.__update_binary__( + utilities.ConfigInfo.stop_tpl_path, output_path, binary_name) + if ret is False: + utilities.log_error( + "generate_node_shell_scripts %s error" % output_path) + return False + return True diff --git a/cpp/tools/ppc-builder/src/config/tars_config_generator.py b/cpp/tools/ppc-builder/src/config/tars_config_generator.py deleted file mode 100644 index 1540f486..00000000 --- a/cpp/tools/ppc-builder/src/config/tars_config_generator.py +++ /dev/null @@ -1,123 +0,0 @@ -#!/usr/bin/python -# -*- coding: UTF-8 -*- -from common import utilities -import os - - -class TarsConfigGenerator: - """ - generate the tars-config - """ - def __generate_adapter_info__(app_name, server_name, servant_object, tars_listen_ip, tars_listen_port): - adapter_name = "%s.%s.%sAdapter" % ( - app_name, server_name, servant_object) - servant_name = "%s.%s.%s" % (app_name, server_name, servant_object) - result = "<%s>\n" % adapter_name - result = "%s\t\tallow\n" % result - result = "%s\t\tendpoint=tcp -h %s -p %d -t 60000\n" % ( - result, tars_listen_ip, tars_listen_port) - result = "%s\t\tmaxconns=100000\n\t\tprotocol=tars\n\t\tqueuecap=50000\n\t\tqueuetimeout=20000\n" % result - result = "%s\t\tservant=%s\n" % (result, servant_name) - result = "%s\t\tthreads=8\n\t\t\n" % (result, adapter_name) - return result - - def generate_ip_shell_scripts(script_output_dir, start_shell_script_name, stop_shell_script_name): - tars_start_all_path = os.path.join( - script_output_dir, start_shell_script_name) - utilities.mkdir(script_output_dir) - if os.path.exists(tars_start_all_path) is False: - utilities.log_debug( - "* generate shell script, dst: %s" % tars_start_all_path) - # tars_start_all.sh - command = "cp %s %s" % ( - utilities.ConfigInfo.tars_start_all_tpl_path, tars_start_all_path) - (result, output) = utilities.execute_command_and_getoutput(command) - if result is False: - utilities.log_error( - "* generate %s failed, error: %s" % (tars_start_all_path, output)) - return False - tars_stop_all_path = os.path.join( - script_output_dir, stop_shell_script_name) - if os.path.exists(tars_stop_all_path) is False: - # tars stop_all.sh - command = "cp %s %s" % ( - utilities.ConfigInfo.tars_stop_all_tpl_path, tars_stop_all_path) - (result, output) = utilities.execute_command_and_getoutput(command) - if result is False: - utilities.log_error( - "* generate %s failed, error: %s" % (tars_stop_all_path, output)) - return False - utilities.log_debug( - "* generate_ip_shell_scripts success, output: %s" % script_output_dir) - return True - - def __update_service_name__(file_path, output_path, service_name): - command = "cp %s %s" % (file_path, output_path) - (ret, output) = utilities.execute_command_and_getoutput(command) - if ret is False: - return False - updated_config = "" - with open(output_path, 'r', encoding='UTF-8') as file: - updated_config = file.read() - updated_config = updated_config.replace( - "@SERVICE_NAME@", "../" + service_name) - with open(output_path, 'w', encoding='UTF-8') as file: - file.write(updated_config) - return True - - def generate_node_shell_scripts(script_output_dir, service_name): - utilities.log_debug("* generate shell scripts for %s, dst: %s" % - (service_name, script_output_dir)) - utilities.mkdir(script_output_dir) - # the start.sh - output_path = os.path.join(script_output_dir, "start.sh") - ret = TarsConfigGenerator.__update_service_name__( - utilities.ConfigInfo.tars_start_tpl_path, output_path, service_name) - if ret is False: - utilities.log_error( - "generate_node_shell_scripts %s error" % output_path) - return False - # the stop.sh - output_path = os.path.join(script_output_dir, "stop.sh") - ret = TarsConfigGenerator.__update_service_name__( - utilities.ConfigInfo.tars_stop_tpl_path, output_path, service_name) - if ret is False: - utilities.log_error( - "generate_node_shell_scripts %s error" % output_path) - return False - return True - - def generate_and_store_tars_conf(conf_tpl_path, output_path, app_name, server_name, servant_object_list, tars_listen_ip, tars_listen_port): - utilities.log_debug("generate_and_store_tars_conf: %s" % conf_tpl_path) - utilities.mkfiledir(output_path) - # copy tpl to output_path - if os.path.exists(output_path): - utilities.log_error("the tars file %s already exists!") - return False - command = "cp %s %s" % (conf_tpl_path, output_path) - (ret, output) = utilities.execute_command_and_getoutput(command) - if ret is False: - utilities.log_error("copy tpl tars config %s to %s error: %s" % ( - conf_tpl_path, output_path, output)) - return False - # config - module_name = "%s.%s" % (app_name, server_name) - - tars_adapters = "" - for servant_obj in servant_object_list: - adapter_info = TarsConfigGenerator.__generate_adapter_info__( - app_name, server_name, servant_obj, tars_listen_ip, tars_listen_port) - tars_adapters = "%s%s" % (tars_adapters, adapter_info) - utilities.log_debug("tars_adapters: %s" % tars_adapters) - updated_config = "" - with open(output_path, 'r', encoding='UTF-8') as file: - updated_config = file.read() - updated_config = updated_config.replace("@TARS_APP@", app_name) - updated_config = updated_config.replace( - "@TARS_SERVER@", server_name) - updated_config = updated_config.replace( - "@MODULE_NAME@", module_name) - updated_config = updated_config.replace( - "@TARS_ADAPTERS@", tars_adapters) - with open(output_path, 'w', encoding='UTF-8') as file: - file.write(updated_config) diff --git a/cpp/tools/ppc-builder/src/scripts/gen_cert.sh b/cpp/tools/ppc-builder/src/scripts/gen_cert.sh index 2ee333b6..187fa66e 100644 --- a/cpp/tools/ppc-builder/src/scripts/gen_cert.sh +++ b/cpp/tools/ppc-builder/src/scripts/gen_cert.sh @@ -435,8 +435,9 @@ generate_private_key_for_psi_server() { fi ${OPENSSL_CMD} genpkey -paramfile ${sm2_params} -out ${output_path}/node.pem 2>/dev/null $OPENSSL_CMD ec -in "$output_path/node.pem" -text -noout 2> /dev/null | sed -n '3,5p' | sed 's/://g' | tr "\n" " " | sed 's/ //g' | cat > "$output_path/node.privateKey" - private_key=$(cat $output_path/node.privateKey) - echo ${private_key} + ${OPENSSL_CMD} ec -text -noout -in "${output_path}/node.pem" 2>/dev/null | sed -n '7,11p' | tr -d ": \n" | awk '{print substr($0,3);}' | cat >"$output_path"/node.nodeid + public_key=$(cat $output_path/node.nodeid) + echo ${public_key} } check_env() { diff --git a/cpp/tools/ppc-builder/src/tpl/config.ini.gateway b/cpp/tools/ppc-builder/src/tpl/config.ini.gateway index a7319da2..8acc4990 100644 --- a/cpp/tools/ppc-builder/src/tpl/config.ini.gateway +++ b/cpp/tools/ppc-builder/src/tpl/config.ini.gateway @@ -1,7 +1,6 @@ [agency] ; the agency-id of self-party id = agency0 - ; the agency info [cert] ; directory the certificates located in @@ -10,30 +9,27 @@ [gateway] listen_ip=0.0.0.0 listen_port=40300 - ;thread_count = 4 + ;the dir that contains the connected endpoint information, e.g.nodes.json + ;nodes_path="./" + ; the file that configure the connected endpoint information + ; nodes_file=nodes.json + ; thread_count = 4 ; ssl or sm ssl sm_ssl=false + ; the max allowed message size in MBytes, default is 100MBytes + max_allow_msg_size = 100 ;ssl connection switch, if disable the ssl connection, default: false - disable_ssl = false + ;disable_ssl = true ;the time of the gateway holding and waiting to dispatcher the unsynced task, in minutes holding_msg_minutes = 30 - ; disable_cache = false -[cache] - ; the cache type, only support redis now - type = 0 - proxy = 127.0.0.1:20002 - obServer = 127.0.0.1:10904 - cluster = REDIS_CLUSTER - user = 1194 - host = 127.0.0.1 - port = 6379 - password = - database = 1 - pool_size = 16 - ; the redis connection timeout, in ms, default is 500ms - connection_timeout = 500 - ; the redis socket timeout, in ms, default is 500ms - socket_timeout = 500 + ;reconnect_time = 10000 + ; the unreachable distance + ;unreachable_distance=10 + +[transport] + ; the endpoint information + listen_ip = 0.0.0.0 + listen_port = 19000 [log] enable=true @@ -52,7 +48,5 @@ ; archive_path=./log/ compress_archive_file=true ; ; 0: no limit, in MB - ; max_archive_files=10 - ; ; 0: no limit, in MB ; max_archive_size=0 ; min_free_space=0 \ No newline at end of file diff --git a/cpp/tools/ppc-builder/src/tpl/config.ini.node b/cpp/tools/ppc-builder/src/tpl/config.ini.node index 4989f1d1..177536ac 100644 --- a/cpp/tools/ppc-builder/src/tpl/config.ini.node +++ b/cpp/tools/ppc-builder/src/tpl/config.ini.node @@ -30,12 +30,18 @@ ; directory the certificates located in cert_path=./conf -[tars_gateway] - ; the gateway service name - name = - ;the time of the gateway holding and waiting to dispatcher the unsynced task, in minutes - holding_msg_minutes = 30 - ; the gateway service endpoints +[transport] + ; the endpoint information + listen_ip = 0.0.0.0 + listen_port = 18000 + host_ip = + ; the threadPoolSize + thread_count = 4 + ; the gatewayService endpoint information + service.gateway_target = + ; the components + service.components = + nodeid= [storage] host = 127.0.0.1 @@ -103,8 +109,6 @@ ; if archive_path is empty, the archive function will be disabled ; archive_path=./log/ compress_archive_file=true - ; ; 0: no limit, in MB + ; 0: no limit, in MB ; max_archive_files=10 - ; ; 0: no limit, in MB - ; max_archive_size=0 ; min_free_space=0 \ No newline at end of file diff --git a/cpp/tools/ppc-builder/src/tpl/start.sh b/cpp/tools/ppc-builder/src/tpl/start.sh new file mode 100755 index 00000000..83921a4c --- /dev/null +++ b/cpp/tools/ppc-builder/src/tpl/start.sh @@ -0,0 +1,39 @@ +#!/bin/bash +SHELL_FOLDER=$(cd $(dirname $0);pwd) + +LOG_ERROR() { + content=${1} + echo -e "\033[31m[ERROR] ${content}\033[0m" +} + +LOG_INFO() { + content=${1} + echo -e "\033[32m[INFO] ${content}\033[0m" +} + +binary=${SHELL_FOLDER}/../@BINARY_NAME@ +cd ${SHELL_FOLDER} +node=$(basename ${SHELL_FOLDER}) +node_pid=$(ps aux|grep ${binary}|grep -v grep|awk '{print $2}') +if [ ! -z ${node_pid} ];then + echo " ${node} is running, pid is $node_pid." + exit 0 +else + nohup ${binary} -c config.ini >>nohup.out 2>&1 & + sleep 1.5 +fi +try_times=4 +i=0 +while [ $i -lt ${try_times} ] +do + node_pid=$(ps aux|grep ${binary}|grep -v grep|awk '{print $2}') + success_flag=$(tail -n20 nohup.out | grep running) + if [[ ! -z ${node_pid} && ! -z "${success_flag}" ]];then + echo -e "\033[32m ${node} start successfully pid=${node_pid}\033[0m" + exit 0 + fi + sleep 0.5 + ((i=i+1)) +done +echo -e "\033[31m Exceed waiting time. Please try again to start ${node} \033[0m" +tail -n20 nohup.out diff --git a/cpp/tools/ppc-builder/src/tpl/tars_start_all.sh b/cpp/tools/ppc-builder/src/tpl/start_all.sh similarity index 62% rename from cpp/tools/ppc-builder/src/tpl/tars_start_all.sh rename to cpp/tools/ppc-builder/src/tpl/start_all.sh index 006f53b4..b0e20f1b 100644 --- a/cpp/tools/ppc-builder/src/tpl/tars_start_all.sh +++ b/cpp/tools/ppc-builder/src/tpl/start_all.sh @@ -5,10 +5,9 @@ cd "${dirpath}" dirs=($(ls -l ${dirpath} | awk '/^d/ {print $NF}')) for dir in ${dirs[*]} do - if [[ -f "${dirpath}/${dir}/start.sh" ]];then - echo "try to start ${dirpath}/${dir}" + if [[ -f "${dirpath}/${dir}/config.ini" && -f "${dirpath}/${dir}/start.sh" ]];then + echo "try to start ${dir}" bash ${dirpath}/${dir}/start.sh & fi done wait - diff --git a/cpp/tools/ppc-builder/src/tpl/stop.sh b/cpp/tools/ppc-builder/src/tpl/stop.sh new file mode 100755 index 00000000..ae40e5ba --- /dev/null +++ b/cpp/tools/ppc-builder/src/tpl/stop.sh @@ -0,0 +1,35 @@ +#!/bin/bash +SHELL_FOLDER=$(cd $(dirname $0);pwd) + +LOG_ERROR() { + content=${1} + echo -e "\033[31m[ERROR] ${content}\033[0m" +} + +LOG_INFO() { + content=${1} + echo -e "\033[32m[INFO] ${content}\033[0m" +} + +binary=${SHELL_FOLDER}/../@BINARY_NAME@ +node=$(basename ${SHELL_FOLDER}) +node_pid=$(ps aux|grep ${binary}|grep -v grep|awk '{print $2}') +try_times=10 +i=0 +if [ -z ${node_pid} ];then + echo " ${node} isn't running." + exit 0 +fi +[ ! -z ${node_pid} ] && kill ${node_pid} > /dev/null +while [ $i -lt ${try_times} ] +do + sleep 1 + node_pid=$(ps aux|grep ${binary}|grep -v grep|awk '{print $2}') + if [ -z ${node_pid} ];then + echo -e "\033[32m stop ${node} success.\033[0m" + exit 0 + fi + ((i=i+1)) +done +echo " Exceed maximum number of retries. Please try again to stop ${node}" +exit 1 diff --git a/cpp/tools/ppc-builder/src/tpl/tars_stop_all.sh b/cpp/tools/ppc-builder/src/tpl/stop_all.sh similarity index 62% rename from cpp/tools/ppc-builder/src/tpl/tars_stop_all.sh rename to cpp/tools/ppc-builder/src/tpl/stop_all.sh index 7a71beca..63cc7f48 100644 --- a/cpp/tools/ppc-builder/src/tpl/tars_stop_all.sh +++ b/cpp/tools/ppc-builder/src/tpl/stop_all.sh @@ -5,10 +5,9 @@ cd "${dirpath}" dirs=($(ls -l ${dirpath} | awk '/^d/ {print $NF}')) for dir in ${dirs[*]} do - if [[ -f "${dirpath}/${dir}/stop.sh" ]];then - echo "try to stop ${dirpath}/${dir}" + if [[ -f "${dirpath}/${dir}/config.ini" && -f "${dirpath}/${dir}/stop.sh" ]];then + echo "try to stop ${dir}" bash ${dirpath}/${dir}/stop.sh fi done wait - diff --git a/cpp/tools/ppc-builder/src/tpl/tars.conf b/cpp/tools/ppc-builder/src/tpl/tars.conf deleted file mode 100644 index f2157319..00000000 --- a/cpp/tools/ppc-builder/src/tpl/tars.conf +++ /dev/null @@ -1,31 +0,0 @@ - - - enableset=n - setdivision=NULL - - app=@TARS_APP@ - server=@TARS_SERVER@ - localip=127.0.0.1 - basepath=./conf/ - datapath=./.data/ - logpath=./log/ - logsize=100M - lognum=5 - logLevel=INFO - deactivating-timeout=3000 - activating-timeout=10000 - opencoroutine=0 - coroutinememsize=1G - coroutinestack=128K - closecout=0 - netthread=4 - @TARS_ADAPTERS@ - - - sync-invoke-timeout=3000000 - async-invoke-timeout=5000000 - asyncthread=8 - modulename=@MODULE_NAME@ - - - diff --git a/cpp/tools/ppc-builder/src/tpl/tars_start.sh b/cpp/tools/ppc-builder/src/tpl/tars_start.sh deleted file mode 100644 index a03b6fb4..00000000 --- a/cpp/tools/ppc-builder/src/tpl/tars_start.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash -dirpath="$(cd "$(dirname "$0")" && pwd)" -cd ${dirpath} - -service_name='@SERVICE_NAME@' -service=${dirpath}/${service_name} - -pid=$(ps aux|grep ${service}|grep -v grep|awk '{print $2}') -name=$(basename ${dirpath}) - -if [ ! -z ${pid} ];then - echo " ${service}/${name} is running, pid is ${pid}." - exit 0 -else - nohup ${service} --config=conf/tars.conf >>nohup.out 2>&1 & - sleep 1.5 -fi - -try_times=4 -i=0 -while [ $i -lt ${try_times} ] -do - pid=$(ps aux|grep ${service}|grep -v grep|awk '{print $2}') - if [[ ! -z ${pid} ]];then - echo -e "\033[32m ${service}/${name} start successfully pid=${pid}\033[0m" - exit 0 - fi - sleep 0.5 - ((i=i+1)) -done -echo -e "\033[31m Exceed waiting time. Please try again to start ${service}/${name} \033[0m" -tail -n20 nohup.out - diff --git a/cpp/tools/ppc-builder/src/tpl/tars_stop.sh b/cpp/tools/ppc-builder/src/tpl/tars_stop.sh deleted file mode 100644 index 3e8931ce..00000000 --- a/cpp/tools/ppc-builder/src/tpl/tars_stop.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -dirpath="$(cd "$(dirname "$0")" && pwd)" -cd ${dirpath} - -service_name='@SERVICE_NAME@' -service=${dirpath}/${service_name} -name=$(basename ${dirpath}) - -pid=$(ps aux|grep ${service}|grep -v grep|awk '{print $2}') - -if [ -z ${pid} ];then - echo " ${service}/${name} isn't running." - exit 0 -fi - -kill ${pid} > /dev/null - -i=0 -try_times=10 -while [ $i -lt ${try_times} ] -do - sleep 1 - pid=$(ps aux|grep ${service}|grep -v grep|awk '{print $2}') - if [ -z ${pid} ];then - echo -e "\033[32m stop ${service}/${name} success.\033[0m" - exit 0 - fi - ((i=i+1)) -done -echo " Exceed maximum number of retries. Please try again to stop ${service}/${name}" -exit 1