diff --git a/ceph-nvmeof.conf b/ceph-nvmeof.conf index 9aa4af7c..079bd7df 100644 --- a/ceph-nvmeof.conf +++ b/ceph-nvmeof.conf @@ -60,11 +60,11 @@ client_cert = ./client.crt [spdk] bdevs_per_cluster = 32 tgt_path = /usr/local/bin/nvmf_tgt -#rpc_socket = /var/tmp/spdk.sock +#rpc_socket_dir = /var/tmp/ +#rpc_socket_name = spdk.sock #tgt_cmd_extra_args = --env-context="--no-huge -m1024" --iova-mode=va timeout = 60.0 -log_level = -protocol_log_level = WARNING +#log_level = WARNING # Example value: -m 0x3 -L all # tgt_cmd_extra_args = diff --git a/control/server.py b/control/server.py index 9b5d9224..ce688d4a 100644 --- a/control/server.py +++ b/control/server.py @@ -341,15 +341,7 @@ def _start_spdk(self, omap_state): self.logger.debug(f"Configuring server {self.name}") spdk_tgt_path = self.config.get("spdk", "tgt_path") self.logger.info(f"SPDK Target Path: {spdk_tgt_path}") - self.spdk_rpc_socket_path = self.config.get_with_default("spdk", "rpc_socket", "/var/tmp/spdk.sock") - self.logger.info(f"SPDK Socket: {self.spdk_rpc_socket_path}") - spdk_tgt_cmd_extra_args = self.config.get_with_default( - "spdk", "tgt_cmd_extra_args", "") - cmd = [spdk_tgt_path, "-u", "-r", self.spdk_rpc_socket_path] - if spdk_tgt_cmd_extra_args: - cmd += shlex.split(spdk_tgt_cmd_extra_args) - self.logger.info(f"Starting {' '.join(cmd)}") - sockdir = os.path.dirname(self.spdk_rpc_socket_path) + sockdir = self.config.get_with_default("spdk", "rpc_socket_dir", "/var/tmp/") if not os.path.isdir(sockdir): self.logger.warning(f"Directory {sockdir} does not exist, will create it") try: @@ -357,6 +349,20 @@ def _start_spdk(self, omap_state): except Exception: self.logger.exception(f"Error trying to create {sockdir}") raise + if not sockdir.endswith("/"): + sockdir += "/" + sockname = self.config.get_with_default("spdk", "rpc_socket_name", "spdk.sock") + if sockname.find("/") >= 0: + self.logger.error(f"Invalid SPDK socket name \"{sockname}\". Name should not contain a \"/\".") + raise(f"Invalid SPDK socket name.") + self.spdk_rpc_socket_path = sockdir + sockname + self.logger.info(f"SPDK Socket: {self.spdk_rpc_socket_path}") + spdk_tgt_cmd_extra_args = self.config.get_with_default( + "spdk", "tgt_cmd_extra_args", "") + cmd = [spdk_tgt_path, "-u", "-r", self.spdk_rpc_socket_path] + if spdk_tgt_cmd_extra_args: + cmd += shlex.split(spdk_tgt_cmd_extra_args) + self.logger.info(f"Starting {' '.join(cmd)}") try: # start spdk process time.sleep(2) # this is a temporary hack, we have a timing issue here. Once we solve it the sleep will ve removed @@ -367,7 +373,7 @@ def _start_spdk(self, omap_state): # Initialization timeout = self.config.getfloat_with_default("spdk", "timeout", 60.0) - protocol_log_level = self.config.get_with_default("spdk", "protocol_log_level", "WARNING") + protocol_log_level = self.config.get_with_default("spdk", "log_level", "WARNING") # connect timeout: spdk client retries 5 times per sec conn_retries = int(timeout * 5) self.logger.info(f"SPDK process id: {self.spdk_process.pid}") diff --git a/tests/ceph-nvmeof.tls.conf b/tests/ceph-nvmeof.tls.conf index b6843fa3..398960d6 100644 --- a/tests/ceph-nvmeof.tls.conf +++ b/tests/ceph-nvmeof.tls.conf @@ -59,10 +59,11 @@ client_cert = /etc/ceph/client.crt [spdk] bdevs_per_cluster = 32 tgt_path = /usr/local/bin/nvmf_tgt -#rpc_socket = /var/tmp/spdk.sock +#rpc_socket_dir = /var/tmp/ +#rpc_socket_name = spdk.sock #tgt_cmd_extra_args = --env-context="--no-huge -m1024" --iova-mode=va timeout = 60.0 -log_level = WARNING +#log_level = WARNING # Example value: -m 0x3 -L all # tgt_cmd_extra_args = diff --git a/tests/test_cli_change_lb.py b/tests/test_cli_change_lb.py index aaef3907..db234ac0 100644 --- a/tests/test_cli_change_lb.py +++ b/tests/test_cli_change_lb.py @@ -33,14 +33,14 @@ def two_gateways(config): configB = copy.deepcopy(config) configA.config["gateway"]["name"] = nameA configA.config["gateway"]["override_hostname"] = nameA - configA.config["spdk"]["rpc_socket"] = f"/var/tmp/{sockA}" + configA.config["spdk"]["rpc_socket_name"] = sockA portA = configA.getint("gateway", "port") + 1 configA.config["gateway"]["port"] = str(portA) discPortA = configA.getint("discovery", "port") + 1 configA.config["discovery"]["port"] = str(discPortA) configB.config["gateway"]["name"] = nameB configB.config["gateway"]["override_hostname"] = nameB - configB.config["spdk"]["rpc_socket"] = f"/var/tmp/{sockB}" + configB.config["spdk"]["rpc_socket_name"] = sockB portB = portA + 1 discPortB = discPortA + 1 configB.config["gateway"]["port"] = str(portB) diff --git a/tests/test_multi_gateway.py b/tests/test_multi_gateway.py index 249ebfc2..9f470110 100644 --- a/tests/test_multi_gateway.py +++ b/tests/test_multi_gateway.py @@ -22,7 +22,7 @@ def conn(config): configA.config["gateway"]["group"] = "Group1" configA.config["gateway"]["state_update_notify"] = str(update_notify) configA.config["gateway"]["enable_spdk_discovery_controller"] = "True" - configA.config["spdk"]["rpc_socket"] = "/var/tmp/spdk_GatewayA.sock" + configA.config["spdk"]["rpc_socket_name"] = "spdk_GatewayA.sock" configB = copy.deepcopy(configA) addr = configA.get("gateway", "addr") portA = configA.getint("gateway", "port") @@ -30,7 +30,7 @@ def conn(config): configB.config["gateway"]["name"] = "GatewayB" configB.config["gateway"]["port"] = str(portB) configB.config["gateway"]["state_update_interval_sec"] = str(update_interval_sec) - configB.config["spdk"]["rpc_socket"] = "/var/tmp/spdk_GatewayB.sock" + configB.config["spdk"]["rpc_socket_name"] = "spdk_GatewayB.sock" configB.config["spdk"]["tgt_cmd_extra_args"] = "-m 0x02" ceph_utils = CephUtils(config) diff --git a/tests/test_nsid.py b/tests/test_nsid.py index 9ac106be..6b5896dc 100644 --- a/tests/test_nsid.py +++ b/tests/test_nsid.py @@ -27,14 +27,14 @@ def setup_config(config, gw1_name, gw2_name, gw_group, update_notify, update_int configA.config["gateway"]["omap_file_disable_unlock"] = str(disable_unlock) configA.config["gateway"]["omap_file_lock_duration"] = str(lock_duration) configA.config["gateway"]["enable_spdk_discovery_controller"] = "True" - configA.config["spdk"]["rpc_socket"] = f"/var/tmp/{sock1_name}" + configA.config["spdk"]["rpc_socket_name"] = sock1_name configB = copy.deepcopy(configA) portA = configA.getint("gateway", "port") configA.config["gateway"]["port"] = str(portA) portB = portA + 2 configB.config["gateway"]["name"] = gw2_name configB.config["gateway"]["port"] = str(portB) - configB.config["spdk"]["rpc_socket"] = f"/var/tmp/{sock2_name}" + configB.config["spdk"]["rpc_socket_name"] = sock2_name configB.config["spdk"]["tgt_cmd_extra_args"] = "-m 0x02" return configA, configB diff --git a/tests/test_omap_lock.py b/tests/test_omap_lock.py index 297a2eab..826d4593 100644 --- a/tests/test_omap_lock.py +++ b/tests/test_omap_lock.py @@ -29,7 +29,7 @@ def setup_config(config, gw1_name, gw2_name, gw_group, update_notify ,update_int configA.config["gateway"]["omap_file_disable_unlock"] = str(disable_unlock) configA.config["gateway"]["omap_file_lock_duration"] = str(lock_duration) configA.config["gateway"]["enable_spdk_discovery_controller"] = "True" - configA.config["spdk"]["rpc_socket"] = f"/var/tmp/{sock1_name}" + configA.config["spdk"]["rpc_socket_name"] = sock1_name configB = copy.deepcopy(configA) portA = configA.getint("gateway", "port") + port_inc configA.config["gateway"]["port"] = str(portA) @@ -37,7 +37,7 @@ def setup_config(config, gw1_name, gw2_name, gw_group, update_notify ,update_int configB.config["gateway"]["name"] = gw2_name configB.config["gateway"]["override_hostname"] = gw2_name configB.config["gateway"]["port"] = str(portB) - configB.config["spdk"]["rpc_socket"] = f"/var/tmp/{sock2_name}" + configB.config["spdk"]["rpc_socket_name"] = sock2_name configB.config["spdk"]["tgt_cmd_extra_args"] = "-m 0x02" return configA, configB diff --git a/tests/test_server.py b/tests/test_server.py index 8e240ae0..2cd63ed5 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -57,7 +57,7 @@ def test_spdk_multi_gateway_exception(self): configB = copy.deepcopy(configA) configB.config["gateway"]["name"] = "GatewayB" configB.config["gateway"]["port"] = str(configA.getint("gateway", "port") + 1) - configB.config["spdk"]["rpc_socket"] = "/var/tmp/spdk_GatewayB.sock" + configB.config["spdk"]["rpc_socket_name"] = "spdk_GatewayB.sock" # invalid arg, spdk would exit with code 1 at start up configB.config["spdk"]["tgt_cmd_extra_args"] = "-m 0x343435545"