From d6fa4a5c59e6e30450f7d362eebe2477560d9e85 Mon Sep 17 00:00:00 2001 From: cyjseagull Date: Mon, 2 Dec 2024 19:55:13 +0800 Subject: [PATCH] support specify the krb5.conf --- cpp/ppc-framework/protocol/Krb5AuthConfig.h | 10 ++++- cpp/tools/build_ppc.sh | 37 ++++++++++++++++++- cpp/tools/build_wedpr_cem.sh | 6 ++- cpp/tools/build_wedpr_mpc.sh | 37 ++++++++++++++++++- .../ppc-builder/conf/config-example.toml | 14 ++++--- cpp/tools/ppc-builder/src/common/utilities.py | 2 + .../src/config/ppc_deploy_config.py | 4 ++ .../src/config/ppc_node_config_generator.py | 18 +++++++-- cpp/tools/ppc-builder/src/tpl/config.ini.node | 6 ++- cpp/tools/ppc-builder/src/tpl/krb5.conf | 22 +++++++++++ .../ppc-tools/src/config/PPCConfig.cpp | 10 +++++ .../ppc-storage/src/hdfs/HDFSStorage.cpp | 1 + .../src/hdfs/auth/Krb5CredLoader.cpp | 13 ++++++- .../src/hdfs/auth/Krb5CredLoader.h | 7 ++++ python/ppc_model/conf/application-sample.yml | 4 +- 15 files changed, 170 insertions(+), 21 deletions(-) create mode 100644 cpp/tools/ppc-builder/src/tpl/krb5.conf diff --git a/cpp/ppc-framework/protocol/Krb5AuthConfig.h b/cpp/ppc-framework/protocol/Krb5AuthConfig.h index 801119be..a7ed09b3 100644 --- a/cpp/ppc-framework/protocol/Krb5AuthConfig.h +++ b/cpp/ppc-framework/protocol/Krb5AuthConfig.h @@ -31,6 +31,7 @@ struct Krb5AuthConfig std::string principal; std::string password; std::string ccachePath; + std::string authConfigFilePath = "./conf/krb5.conf"; void check() const { if (principal.size() == 0) @@ -48,12 +49,19 @@ struct Krb5AuthConfig BOOST_THROW_EXCEPTION(WeDPRException() << bcos::errinfo_comment( "Invalid krb5 auth config: Must set the ccachePath!")); } + if (authConfigFilePath.size() == 0) + { + BOOST_THROW_EXCEPTION( + WeDPRException() << bcos::errinfo_comment( + "Invalid krb5 auth config: Must set the authConfigFilePath!")); + } } inline std::string desc() const { std::stringstream oss; - oss << LOG_KV("principal", principal) << LOG_KV("ccachePath", ccachePath); + oss << LOG_KV("principal", principal) << LOG_KV("ccachePath", ccachePath) + << LOG_KV("authConfigFilePath", authConfigFilePath); return oss.str(); } }; diff --git a/cpp/tools/build_ppc.sh b/cpp/tools/build_ppc.sh index ce20f151..9a2a9d6d 100644 --- a/cpp/tools/build_ppc.sh +++ b/cpp/tools/build_ppc.sh @@ -476,7 +476,7 @@ generate_node_config_ini() { [hdfs_storage] ; the hdfs configuration - user = app + user = root name_node = 127.0.0.1 name_node_port = 9000 token = @@ -487,11 +487,13 @@ generate_node_config_ini() { ; enable auth or not, default is false ; enable_krb5_auth = false ; the hdfs kerberos auth principal, used when enable_krb5_auth - ; auth_principal = + ; auth_principal = root@NODE.DC1.CONSUL ; the hdfs kerberos auth password, used when enable_krb5_auth ; auth_password = ; the ccache path, used when enable_krb5_auth ; ccache_path = /tmp/krb5cc_ppc_node + ; the krb5.conf path + ; krb5_conf_path = conf/krb5.conf [ra2018psi] @@ -549,6 +551,36 @@ generate_node_config_ini() { EOF } +generate_krb5_file_template() +{ + local filepath=$1 + mkdir -p $(dirname $filepath) + cat << EOF > "${filepath}" + [logging] + default = FILE:/var/log/kerberos/krb5libs.log + kdc = FILE:/var/log/kerberos/krb5kdc.log + admin_server = FILE:/var/log/kerberos/kadmind.log + +[libdefaults] + default_realm = NODE.DC1.CONSUL + dns_lookup_realm = false + dns_lookup_kdc = false + ticket_lifetime = 24h + renew_lifetime = 7d + forwardable = true + +[realms] + NODE.DC1.CONSUL = { + kdc = + admin_server = + } + +[domain_realm] + .node.dc1.consul = NODE.DC1.CONSUL + node.dc1.consul = NODE.DC1.CONSUL +EOF +} + generate_script_template() { local filepath=$1 @@ -923,6 +955,7 @@ deploy_nodes() private_key=$(generate_private_key "${node_dir}/conf") node_id=$(cat "${node_dir}/conf/node.nodeid") generate_node_config_ini "${node_dir}/config.ini" "${listen_ip}" "${gateway_port}" "${listen_ip}" "${rpc_port}" "${listen_ip}" "${grpc_port}" ${agency_id} "${count}" "${node_id}" + generate_krb5_file_template "${node_dir}/conf/krb5.conf" generate_p2p_connected_conf "${node_dir}/${p2p_connected_conf_name}" "${connected_nodes}" "false" set_value ${ip//./}_count $(($(get_value ${ip//./}_count) + 1)) ((++count)) diff --git a/cpp/tools/build_wedpr_cem.sh b/cpp/tools/build_wedpr_cem.sh index 8793893d..8299c61e 100644 --- a/cpp/tools/build_wedpr_cem.sh +++ b/cpp/tools/build_wedpr_cem.sh @@ -354,7 +354,7 @@ generate_config_ini() { [hdfs_storage] ; the hdfs configuration - user = ppc + user = root name_node = 127.0.0.1 name_node_port = 9900 token = @@ -365,11 +365,13 @@ generate_config_ini() { ; enable auth or not, default is false ; enable_krb5_auth = false ; the hdfs kerberos auth principal, used when enable_krb5_auth - ; auth_principal = + ; auth_principal = root@NODE.DC1.CONSUL ; the hdfs kerberos auth password, used when enable_krb5_auth ; auth_password = ; the ccache path, used when enable_krb5_auth ; ccache_path = /tmp/krb5cc_ppc_node + ; the krb5.conf path + ; krb5_conf_path = conf/krb5.conf [cert] ; directory the certificates located in diff --git a/cpp/tools/build_wedpr_mpc.sh b/cpp/tools/build_wedpr_mpc.sh index 7967429a..ea03934b 100644 --- a/cpp/tools/build_wedpr_mpc.sh +++ b/cpp/tools/build_wedpr_mpc.sh @@ -358,7 +358,7 @@ generate_config_ini() { [hdfs_storage] ; the hdfs configuration - user = ppc + user = root name_node = 127.0.0.1 name_node_port = 9900 token = @@ -369,11 +369,13 @@ generate_config_ini() { ; enable auth or not, default is false ; enable_krb5_auth = false ; the hdfs kerberos auth principal, used when enable_krb5_auth - ; auth_principal = + ; auth_principal = root@NODE.DC1.CONSUL ; the hdfs kerberos auth password, used when enable_krb5_auth ; auth_password = ; the ccache path, used when enable_krb5_auth ; ccache_path = /tmp/krb5cc_ppc_node + ; the krb5.conf path + ; krb5_conf_path = conf/krb5.conf [transport] ; the endpoint information @@ -404,6 +406,36 @@ generate_config_ini() { EOF } +generate_krb5_file_template() +{ + local filepath=$1 + mkdir -p $(dirname $filepath) + cat << EOF > "${filepath}" + [logging] + default = FILE:/var/log/kerberos/krb5libs.log + kdc = FILE:/var/log/kerberos/krb5kdc.log + admin_server = FILE:/var/log/kerberos/kadmind.log + +[libdefaults] + default_realm = NODE.DC1.CONSUL + dns_lookup_realm = false + dns_lookup_kdc = false + ticket_lifetime = 24h + renew_lifetime = 7d + forwardable = true + +[realms] + NODE.DC1.CONSUL = { + kdc = + admin_server = + } + +[domain_realm] + .node.dc1.consul = NODE.DC1.CONSUL + node.dc1.consul = NODE.DC1.CONSUL +EOF +} + generate_script_template() { local filepath=$1 @@ -731,6 +763,7 @@ deploy_nodes() private_key=$(generate_private_key "${output_dir}/conf") node_id=$(cat "${output_dir}/conf/node.nodeid") generate_config_ini "${output_dir}/config.ini" "${listen_ip}" "${rpc_port}" "${agency_info}" ${agency_id} "${listen_ip}" "${grpc_port}" "${node_id}" + generate_krb5_file_template "{output_dir}/conf/krb5.conf" print_result } diff --git a/cpp/tools/ppc-builder/conf/config-example.toml b/cpp/tools/ppc-builder/conf/config-example.toml index b68ba1ba..867f113a 100644 --- a/cpp/tools/ppc-builder/conf/config-example.toml +++ b/cpp/tools/ppc-builder/conf/config-example.toml @@ -90,19 +90,20 @@ holding_msg_minutes = 30 database = "" # the hdfs storage config [agency.node.hdfs_storage] - user = "" + user = "root" name_node = "127.0.0.1" name_node_port = 9000 token = "" # enable auth or not, default is false enable_krb5_auth = false # the hdfs kerberos auth principal, used when enable_krb5_auth - auth_principal = "" + auth_principal = "root@NODE.DC1.CONSUL" # the hdfs kerberos auth password, used when enable_krb5_auth auth_password = "" # the ccache path, used when enable_krb5_auth ccache_path = "/tmp/krb5cc_ppc_node" - + # the krb5 conf path + krb5_conf_path = "conf/krb5.conf" # the gateway config [agency.node.gateway] gateway_grpc_target = ["127.0.0.1:40600", "127.0.0.1:40601"] @@ -184,18 +185,21 @@ holding_msg_minutes = 30 database = "" # the hdfs storage config [agency.node.hdfs_storage] - user = "" + user = "root" name_node = "127.0.0.1" name_node_port = 9000 token = "" # enable auth or not, default is false enable_krb5_auth = false # the hdfs kerberos auth principal, used when enable_krb5_auth - auth_principal = "" + auth_principal = "root@NODE.DC1.CONSUL" # the hdfs kerberos auth password, used when enable_krb5_auth auth_password = "" # the ccache path, used when enable_krb5_auth ccache_path = "/tmp/krb5cc_ppc_node" + # the krb5 conf path + krb5_conf_path = "conf/krb5.conf" + # the gateway config [agency.node.gateway] gateway_grpc_target = ["127.0.0.1:40620", "127.0.0.1:40621"] diff --git a/cpp/tools/ppc-builder/src/common/utilities.py b/cpp/tools/ppc-builder/src/common/utilities.py index 5e4b3c32..5462a9db 100644 --- a/cpp/tools/ppc-builder/src/common/utilities.py +++ b/cpp/tools/ppc-builder/src/common/utilities.py @@ -29,6 +29,8 @@ class ConfigInfo: pwd_path, tpl_abs_path, "config.ini.node") gateway_config_tpl_path = os.path.join( pwd_path, tpl_abs_path, "config.ini.gateway") + krb5_config_tpl_path = os.path.join( + pwd_path, tpl_abs_path, "krb5.conf") ppc_gateway_binary_name = "ppc-gateway-service" ppc_node_binary_name = "ppc-pro-node" diff --git a/cpp/tools/ppc-builder/src/config/ppc_deploy_config.py b/cpp/tools/ppc-builder/src/config/ppc_deploy_config.py index 38731269..d8c642e4 100644 --- a/cpp/tools/ppc-builder/src/config/ppc_deploy_config.py +++ b/cpp/tools/ppc-builder/src/config/ppc_deploy_config.py @@ -134,6 +134,10 @@ def __init__(self, config, config_section, must_exist): self.ccache_path = utilities.get_item_value( self.config, "ccache_path", "", enable_krb5_auth, config_section) + # the krb5.conf + self.krb5_conf_path = utilities.get_item_value( + self.config, "krb5_conf_path", + "conf/krb5.conf", enable_krb5_auth, config_section) class RA2018PSIConfig: diff --git a/cpp/tools/ppc-builder/src/config/ppc_node_config_generator.py b/cpp/tools/ppc-builder/src/config/ppc_node_config_generator.py index 325dccfd..8a835ae1 100644 --- a/cpp/tools/ppc-builder/src/config/ppc_node_config_generator.py +++ b/cpp/tools/ppc-builder/src/config/ppc_node_config_generator.py @@ -109,8 +109,8 @@ def __generate_single_node_inner_config__(self, tpl_config_path, node_path, priv self.__generate_storage_config__( config_content, node_config.storage_config) # load the hdfs_storage_config - self.__generate_hdfs_storage_config__( - config_content, node_config.hdfs_storage_config) + self.__generate_hdfs_storage_config__(node_path, utilities.ConfigInfo.krb5_config_tpl_path, + config_content, node_config.hdfs_storage_config) # load the ra2018psi config self.__generate_ra2018psi_config__( config_content, node_config.ra2018psi_config) @@ -182,7 +182,7 @@ def __generate_storage_config__(self, config_content, storage_config): config_content[section_name]["password"] = storage_config.password config_content[section_name]["database"] = storage_config.database - def __generate_hdfs_storage_config__(self, config_content, hdfs_storage_config): + def __generate_hdfs_storage_config__(self, node_path, krb5_tpl_file_path, config_content, hdfs_storage_config): if hdfs_storage_config is None: return section_name = "hdfs_storage" @@ -195,6 +195,18 @@ def __generate_hdfs_storage_config__(self, config_content, hdfs_storage_config): config_content[section_name]["auth_principal"] = hdfs_storage_config.auth_principal config_content[section_name]["auth_password"] = hdfs_storage_config.auth_password config_content[section_name]["ccache_path"] = hdfs_storage_config.ccache_path + config_content[section_name]["krb5_conf_path"] = hdfs_storage_config.krb5_conf_path + # copy krb5.conf to krb5_conf_path specified path + dst_path = os.path.join(node_path, hdfs_storage_config.krb5_conf_path) + if hdfs_storage_config.krb5_conf_path.startswith("/"): + dst_path = hdfs_storage_config.krb5_conf_path + command = "cp %s %s" % (krb5_tpl_file_path, dst_path) + (ret, output) = utilities.execute_command_and_getoutput(command) + if ret is False: + utilities.log_error("copy krb5 configuration from %s to %s failed, error: %s") % ( + krb5_tpl_file_path, dst_path, output) + return False + return True def __generate_transport_config__(self, config_content, node_config, node_id, diff --git a/cpp/tools/ppc-builder/src/tpl/config.ini.node b/cpp/tools/ppc-builder/src/tpl/config.ini.node index afc7800b..5bbb29b3 100644 --- a/cpp/tools/ppc-builder/src/tpl/config.ini.node +++ b/cpp/tools/ppc-builder/src/tpl/config.ini.node @@ -53,7 +53,7 @@ [hdfs_storage] ; the hdfs configuration - user = app + user = root name_node = 127.0.0.1 name_node_port = 9000 token = @@ -64,11 +64,13 @@ ; enable auth or not, default is false ; enable_krb5_auth = false ; the hdfs kerberos auth principal, used when enable_krb5_auth - ; auth_principal = + ; auth_principal = root@NODE.DC1.CONSUL ; the hdfs kerberos auth password, used when enable_krb5_auth ; auth_password = ; the ccache path, used when enable_krb5_auth ; ccache_path = /tmp/krb5cc_ppc_node + ; the krb5.conf path + ; krb5_conf_path = conf/krb5.conf [ra2018psi] ; The database used to store cuckoo-filter diff --git a/cpp/tools/ppc-builder/src/tpl/krb5.conf b/cpp/tools/ppc-builder/src/tpl/krb5.conf new file mode 100644 index 00000000..715fe8f9 --- /dev/null +++ b/cpp/tools/ppc-builder/src/tpl/krb5.conf @@ -0,0 +1,22 @@ +[logging] + default = FILE:/var/log/kerberos/krb5libs.log + kdc = FILE:/var/log/kerberos/krb5kdc.log + admin_server = FILE:/var/log/kerberos/kadmind.log + +[libdefaults] + default_realm = NODE.DC1.CONSUL + dns_lookup_realm = false + dns_lookup_kdc = false + ticket_lifetime = 24h + renew_lifetime = 7d + forwardable = true + +[realms] + NODE.DC1.CONSUL = { + kdc = + admin_server = + } + +[domain_realm] + .node.dc1.consul = NODE.DC1.CONSUL + node.dc1.consul = NODE.DC1.CONSUL diff --git a/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.cpp b/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.cpp index 144208db..59b34c2a 100644 --- a/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.cpp +++ b/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.cpp @@ -599,6 +599,16 @@ void PPCConfig::loadKrb5AuthConfig(boost::property_tree::ptree const& pt) // the ccachePath authConfig->ccachePath = pt.get("hdfs_storage.ccache_path", "/tmp/krb5cc_ppc_node"); + // the krb5.conf path + authConfig->authConfigFilePath = + pt.get("hdfs_storage.krb5_conf_path", "./conf/krb5.conf"); + // relative path case + if (!authConfig->authConfigFilePath.starts_with("/")) + { + auto joinedPath = + boost::filesystem::absolute(boost::filesystem::path(authConfig->authConfigFilePath)); + authConfig->authConfigFilePath = joinedPath.string(); + } m_storageConfig.fileStorageConnectionOpt->authConfig = authConfig; PPCConfig_LOG(INFO) << LOG_DESC("loadKrb5AuthConfig") << LOG_KV("config", authConfig->desc()); } diff --git a/cpp/wedpr-storage/ppc-storage/src/hdfs/HDFSStorage.cpp b/cpp/wedpr-storage/ppc-storage/src/hdfs/HDFSStorage.cpp index e536433d..fe84d63d 100644 --- a/cpp/wedpr-storage/ppc-storage/src/hdfs/HDFSStorage.cpp +++ b/cpp/wedpr-storage/ppc-storage/src/hdfs/HDFSStorage.cpp @@ -77,6 +77,7 @@ HDFSStorage::HDFSStorage(FileStorageConnectionOption::Ptr const& _option) ctx->init(); HDFS_STORAGE_LOG(INFO) << LOG_DESC("SetKerbTicketCachePath") << LOG_KV("ccachePath", _option->authConfig->ccachePath); + // set the ccache file path hdfsBuilderSetKerbTicketCachePath(m_builder.get(), _option->authConfig->ccachePath.c_str()); } // connect to the hdfs, Note: the m_fs is a pointer diff --git a/cpp/wedpr-storage/ppc-storage/src/hdfs/auth/Krb5CredLoader.cpp b/cpp/wedpr-storage/ppc-storage/src/hdfs/auth/Krb5CredLoader.cpp index 17ff8cba..498aaa7e 100644 --- a/cpp/wedpr-storage/ppc-storage/src/hdfs/auth/Krb5CredLoader.cpp +++ b/cpp/wedpr-storage/ppc-storage/src/hdfs/auth/Krb5CredLoader.cpp @@ -28,9 +28,18 @@ using namespace bcos; void Krb5Context::init() { HDFS_AUTH_LOG(INFO) << LOG_DESC("init Krb5Context") << m_config->desc(); + + // init the profile + auto ret = profile_init_path(m_config->authConfigFilePath.c_str(), &m_profile); + if (ret) + { + BOOST_THROW_EXCEPTION(WeDPRException() << errinfo_comment( + "load Krb5Context failed for profile_init_path failed!")); + } + m_profilePtr = &m_profile; // load krb5 ctx - auto error = krb5_init_context(&m_ctx); - checkResult(error, "krb5_init_context"); + auto error = krb5_init_context_profile(m_profile, 1, &m_ctx); + checkResult(error, "krb5_init_context_profile"); // init the principal error = krb5_parse_name(m_ctx, m_config->principal.c_str(), &m_principal); diff --git a/cpp/wedpr-storage/ppc-storage/src/hdfs/auth/Krb5CredLoader.h b/cpp/wedpr-storage/ppc-storage/src/hdfs/auth/Krb5CredLoader.h index 5dc83831..e2caeb30 100644 --- a/cpp/wedpr-storage/ppc-storage/src/hdfs/auth/Krb5CredLoader.h +++ b/cpp/wedpr-storage/ppc-storage/src/hdfs/auth/Krb5CredLoader.h @@ -20,6 +20,7 @@ #pragma once #include "ppc-framework/protocol/Krb5AuthConfig.h" #include +#include #include namespace ppc::storage @@ -44,6 +45,10 @@ class Krb5Context { krb5_free_context(m_ctx); } + if (m_profilePtr) + { + profile_release(m_profile); + } } virtual void init(); @@ -54,6 +59,8 @@ class Krb5Context protected: ppc::protocol::Krb5AuthConfig::Ptr m_config; krb5_context m_ctx = NULL; + profile_t m_profile; + profile_t* m_profilePtr = NULL; krb5_principal m_principal = NULL; krb5_creds m_credsObj; krb5_creds* m_creds = NULL; diff --git a/python/ppc_model/conf/application-sample.yml b/python/ppc_model/conf/application-sample.yml index 3b7595fe..a472260b 100644 --- a/python/ppc_model/conf/application-sample.yml +++ b/python/ppc_model/conf/application-sample.yml @@ -17,11 +17,11 @@ HDFS_URL: "http://127.0.0.1:50070" # HDFS, STORAGE_TYPE: "HDFS" HDFS_URL: "http://127.0.0.1:9870" -HDFS_USER: "ppc" +HDFS_USER: "root" HDFS_HOME: "/user/ppc/model/webank" HDFS_ENABLE_AUTH: False # the hdfs auth principal -HDFS_AUTH_PRINCIPAL: "wedpr@NODE.DC1.CONSUL" +HDFS_AUTH_PRINCIPAL: "root@NODE.DC1.CONSUL" # the auth key-tab path HDFS_AUTH_KEYTAB_PATH: "./hdfs-wedpr.keytab"