Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Revert SPDK RPC fields #841

Merged
merged 1 commit into from
Sep 1, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions ceph-nvmeof.conf
Original file line number Diff line number Diff line change
Expand Up @@ -60,11 +60,11 @@ client_cert = ./client.crt
[spdk]
bdevs_per_cluster = 32
tgt_path = /usr/local/bin/nvmf_tgt
#rpc_socket = /var/tmp/spdk.sock
#rpc_socket_dir = /var/tmp/
#rpc_socket_name = spdk.sock
#tgt_cmd_extra_args = --env-context="--no-huge -m1024" --iova-mode=va
timeout = 60.0
log_level =
protocol_log_level = WARNING
#log_level = WARNING

# Example value: -m 0x3 -L all
# tgt_cmd_extra_args =
Expand Down
26 changes: 16 additions & 10 deletions control/server.py
Original file line number Diff line number Diff line change
Expand Up @@ -341,22 +341,28 @@ def _start_spdk(self, omap_state):
self.logger.debug(f"Configuring server {self.name}")
spdk_tgt_path = self.config.get("spdk", "tgt_path")
self.logger.info(f"SPDK Target Path: {spdk_tgt_path}")
self.spdk_rpc_socket_path = self.config.get_with_default("spdk", "rpc_socket", "/var/tmp/spdk.sock")
self.logger.info(f"SPDK Socket: {self.spdk_rpc_socket_path}")
spdk_tgt_cmd_extra_args = self.config.get_with_default(
"spdk", "tgt_cmd_extra_args", "")
cmd = [spdk_tgt_path, "-u", "-r", self.spdk_rpc_socket_path]
if spdk_tgt_cmd_extra_args:
cmd += shlex.split(spdk_tgt_cmd_extra_args)
self.logger.info(f"Starting {' '.join(cmd)}")
sockdir = os.path.dirname(self.spdk_rpc_socket_path)
sockdir = self.config.get_with_default("spdk", "rpc_socket_dir", "/var/tmp/")
if not os.path.isdir(sockdir):
self.logger.warning(f"Directory {sockdir} does not exist, will create it")
try:
os.makedirs(sockdir, 0o755)
except Exception:
self.logger.exception(f"Error trying to create {sockdir}")
raise
if not sockdir.endswith("/"):
sockdir += "/"
sockname = self.config.get_with_default("spdk", "rpc_socket_name", "spdk.sock")
if sockname.find("/") >= 0:
self.logger.error(f"Invalid SPDK socket name \"{sockname}\". Name should not contain a \"/\".")
raise(f"Invalid SPDK socket name.")
self.spdk_rpc_socket_path = sockdir + sockname
self.logger.info(f"SPDK Socket: {self.spdk_rpc_socket_path}")
spdk_tgt_cmd_extra_args = self.config.get_with_default(
"spdk", "tgt_cmd_extra_args", "")
cmd = [spdk_tgt_path, "-u", "-r", self.spdk_rpc_socket_path]
if spdk_tgt_cmd_extra_args:
cmd += shlex.split(spdk_tgt_cmd_extra_args)
self.logger.info(f"Starting {' '.join(cmd)}")
try:
# start spdk process
time.sleep(2) # this is a temporary hack, we have a timing issue here. Once we solve it the sleep will ve removed
Expand All @@ -367,7 +373,7 @@ def _start_spdk(self, omap_state):

# Initialization
timeout = self.config.getfloat_with_default("spdk", "timeout", 60.0)
protocol_log_level = self.config.get_with_default("spdk", "protocol_log_level", "WARNING")
protocol_log_level = self.config.get_with_default("spdk", "log_level", "WARNING")
# connect timeout: spdk client retries 5 times per sec
conn_retries = int(timeout * 5)
self.logger.info(f"SPDK process id: {self.spdk_process.pid}")
Expand Down
5 changes: 3 additions & 2 deletions tests/ceph-nvmeof.tls.conf
Original file line number Diff line number Diff line change
Expand Up @@ -59,10 +59,11 @@ client_cert = /etc/ceph/client.crt
[spdk]
bdevs_per_cluster = 32
tgt_path = /usr/local/bin/nvmf_tgt
#rpc_socket = /var/tmp/spdk.sock
#rpc_socket_dir = /var/tmp/
#rpc_socket_name = spdk.sock
#tgt_cmd_extra_args = --env-context="--no-huge -m1024" --iova-mode=va
timeout = 60.0
log_level = WARNING
#log_level = WARNING

# Example value: -m 0x3 -L all
# tgt_cmd_extra_args =
Expand Down
4 changes: 2 additions & 2 deletions tests/test_cli_change_lb.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,14 +33,14 @@ def two_gateways(config):
configB = copy.deepcopy(config)
configA.config["gateway"]["name"] = nameA
configA.config["gateway"]["override_hostname"] = nameA
configA.config["spdk"]["rpc_socket"] = f"/var/tmp/{sockA}"
configA.config["spdk"]["rpc_socket_name"] = sockA
portA = configA.getint("gateway", "port") + 1
configA.config["gateway"]["port"] = str(portA)
discPortA = configA.getint("discovery", "port") + 1
configA.config["discovery"]["port"] = str(discPortA)
configB.config["gateway"]["name"] = nameB
configB.config["gateway"]["override_hostname"] = nameB
configB.config["spdk"]["rpc_socket"] = f"/var/tmp/{sockB}"
configB.config["spdk"]["rpc_socket_name"] = sockB
portB = portA + 1
discPortB = discPortA + 1
configB.config["gateway"]["port"] = str(portB)
Expand Down
4 changes: 2 additions & 2 deletions tests/test_multi_gateway.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,15 +22,15 @@ def conn(config):
configA.config["gateway"]["group"] = "Group1"
configA.config["gateway"]["state_update_notify"] = str(update_notify)
configA.config["gateway"]["enable_spdk_discovery_controller"] = "True"
configA.config["spdk"]["rpc_socket"] = "/var/tmp/spdk_GatewayA.sock"
configA.config["spdk"]["rpc_socket_name"] = "spdk_GatewayA.sock"
configB = copy.deepcopy(configA)
addr = configA.get("gateway", "addr")
portA = configA.getint("gateway", "port")
portB = portA + 2
configB.config["gateway"]["name"] = "GatewayB"
configB.config["gateway"]["port"] = str(portB)
configB.config["gateway"]["state_update_interval_sec"] = str(update_interval_sec)
configB.config["spdk"]["rpc_socket"] = "/var/tmp/spdk_GatewayB.sock"
configB.config["spdk"]["rpc_socket_name"] = "spdk_GatewayB.sock"
configB.config["spdk"]["tgt_cmd_extra_args"] = "-m 0x02"
ceph_utils = CephUtils(config)

Expand Down
4 changes: 2 additions & 2 deletions tests/test_nsid.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,14 +27,14 @@ def setup_config(config, gw1_name, gw2_name, gw_group, update_notify, update_int
configA.config["gateway"]["omap_file_disable_unlock"] = str(disable_unlock)
configA.config["gateway"]["omap_file_lock_duration"] = str(lock_duration)
configA.config["gateway"]["enable_spdk_discovery_controller"] = "True"
configA.config["spdk"]["rpc_socket"] = f"/var/tmp/{sock1_name}"
configA.config["spdk"]["rpc_socket_name"] = sock1_name
configB = copy.deepcopy(configA)
portA = configA.getint("gateway", "port")
configA.config["gateway"]["port"] = str(portA)
portB = portA + 2
configB.config["gateway"]["name"] = gw2_name
configB.config["gateway"]["port"] = str(portB)
configB.config["spdk"]["rpc_socket"] = f"/var/tmp/{sock2_name}"
configB.config["spdk"]["rpc_socket_name"] = sock2_name
configB.config["spdk"]["tgt_cmd_extra_args"] = "-m 0x02"

return configA, configB
Expand Down
4 changes: 2 additions & 2 deletions tests/test_omap_lock.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,15 +29,15 @@ def setup_config(config, gw1_name, gw2_name, gw_group, update_notify ,update_int
configA.config["gateway"]["omap_file_disable_unlock"] = str(disable_unlock)
configA.config["gateway"]["omap_file_lock_duration"] = str(lock_duration)
configA.config["gateway"]["enable_spdk_discovery_controller"] = "True"
configA.config["spdk"]["rpc_socket"] = f"/var/tmp/{sock1_name}"
configA.config["spdk"]["rpc_socket_name"] = sock1_name
configB = copy.deepcopy(configA)
portA = configA.getint("gateway", "port") + port_inc
configA.config["gateway"]["port"] = str(portA)
portB = portA + 2
configB.config["gateway"]["name"] = gw2_name
configB.config["gateway"]["override_hostname"] = gw2_name
configB.config["gateway"]["port"] = str(portB)
configB.config["spdk"]["rpc_socket"] = f"/var/tmp/{sock2_name}"
configB.config["spdk"]["rpc_socket_name"] = sock2_name
configB.config["spdk"]["tgt_cmd_extra_args"] = "-m 0x02"

return configA, configB
Expand Down
2 changes: 1 addition & 1 deletion tests/test_server.py
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ def test_spdk_multi_gateway_exception(self):
configB = copy.deepcopy(configA)
configB.config["gateway"]["name"] = "GatewayB"
configB.config["gateway"]["port"] = str(configA.getint("gateway", "port") + 1)
configB.config["spdk"]["rpc_socket"] = "/var/tmp/spdk_GatewayB.sock"
configB.config["spdk"]["rpc_socket_name"] = "spdk_GatewayB.sock"
# invalid arg, spdk would exit with code 1 at start up
configB.config["spdk"]["tgt_cmd_extra_args"] = "-m 0x343435545"

Expand Down
Loading