From 788ecfcd96a2d6c80bc159170c1006f3d73f7237 Mon Sep 17 00:00:00 2001 From: Artsiom Koltun Date: Wed, 24 Jan 2024 12:26:51 +0100 Subject: [PATCH 01/11] refactor: move spdk bindings to spdk package Signed-off-by: Artsiom Koltun --- pkg/spdk/models.go | 392 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 392 insertions(+) create mode 100644 pkg/spdk/models.go diff --git a/pkg/spdk/models.go b/pkg/spdk/models.go new file mode 100644 index 00000000..feee5f69 --- /dev/null +++ b/pkg/spdk/models.go @@ -0,0 +1,392 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright (c) 2022-2023 Dell Inc, or its subsidiaries. +// Copyright (C) 2023 Intel Corporation + +// Package spdk implements the spdk json-rpc protocol +package spdk + +const ( + // TweakModeSimpleLba represents tweak as + // Tweak[127:0] = {64'b0, LBA[63:0]} + // It is the default tweak mode if not specified + TweakModeSimpleLba = "SIMPLE_LBA" + // TweakModeJoinNegLbaWithLba represents tweak as + // Tweak[127:0] = {1’b0, ~LBA[62:0], LBA[63:0]} + TweakModeJoinNegLbaWithLba = "JOIN_NEG_LBA_WITH_LBA" + // TweakModeIncr512FullLba represents tweak as + // Tweak[127:0] = {lba[127:0]} + TweakModeIncr512FullLba = "INCR_512_FULL_LBA" + // TweakModeIncr512UpperLba represents tweak as + // Tweak[127:0] = {lba[63:0], 64'b0} + TweakModeIncr512UpperLba = "INCR_512_UPPER_LBA" +) + +// AccelCryptoKeyCreateParams holds the parameters required to create a Crypto Key +type AccelCryptoKeyCreateParams struct { + Cipher string `json:"cipher"` + Key string `json:"key"` + Key2 string `json:"key2"` + TweakMode string `json:"tweak_mode,omitempty"` + Name string `json:"name"` +} + +// AccelCryptoKeyCreateResult is the result of creating a Crypto Key +type AccelCryptoKeyCreateResult bool + +// AccelCryptoKeyDestroyParams holds the parameters required to delete a Crypto Key +type AccelCryptoKeyDestroyParams struct { + KeyName string `json:"key_name"` +} + +// AccelCryptoKeyDestroyResult is the result of deleting a Crypto Key +type AccelCryptoKeyDestroyResult bool + +// AccelCryptoKeyGetParams holds the parameters required to get a Crypto Key +type AccelCryptoKeyGetParams struct { + KeyName string `json:"key_name"` +} + +// AccelCryptoKeyGetResult is the result of getting a Crypto Key +type AccelCryptoKeyGetResult struct { + Name string `json:"name"` + Cipher string `json:"cipher"` + Key string `json:"key"` + Key2 string `json:"key2"` +} + +// GetVersionResult is the result of getting a version +type GetVersionResult struct { + Version string `json:"version"` + Fields struct { + Major int `json:"major"` + Minor int `json:"minor"` + Patch int `json:"patch"` + Suffix string `json:"suffix"` + } `json:"fields"` +} + +// BdevAioCreateParams holds the parameters required to create an AIO Block Device +type BdevAioCreateParams struct { + Name string `json:"name"` + Filename string `json:"filename"` + BlockSize int `json:"block_size"` +} + +// BdevAioCreateResult is the result of creating an AIO Block Device +type BdevAioCreateResult string + +// BdevAioDeleteParams holds the parameters required to delete an AIO Block Device +type BdevAioDeleteParams struct { + Name string `json:"name"` +} + +// BdevAioDeleteResult is the result of deleting an AIO Block Device +type BdevAioDeleteResult bool + +// BdevMalloCreateParams holds the parameters required to create a Malloc Block Device +type BdevMalloCreateParams struct { + NumBlocks int `json:"num_blocks"` + BlockSize int `json:"block_size"` + Name string `json:"name"` + UUID string `json:"uuid"` +} + +// BdevAMalloCreateResult is the result of creating a Malloc Block Device +type BdevAMalloCreateResult string + +// BdevMallocDeleteParams holds the parameters required to delete a Malloc Block Device +type BdevMallocDeleteParams struct { + Name string `json:"name"` +} + +// BdevMallocDeleteResult is the result of deleting a Malloc Block Device +type BdevMallocDeleteResult bool + +// BdevNullCreateParams holds the parameters required to create a Null Block Device +// that discards all writes and returns undefined data for reads +type BdevNullCreateParams struct { + BlockSize int `json:"block_size"` + NumBlocks int `json:"num_blocks"` + Name string `json:"name"` +} + +// BdevNullCreateResult is the result of creating a Null Block Device +type BdevNullCreateResult string + +// BdevNullDeleteParams holds the parameters required to delete a Null Block Device +type BdevNullDeleteParams struct { + Name string `json:"name"` +} + +// BdevNullDeleteResult is the result of deleting a Null Block Device +type BdevNullDeleteResult bool + +// BdevCryptoCreateParams holds the parameters required to create a Crypto Block Device +type BdevCryptoCreateParams struct { + BaseBdevName string `json:"base_bdev_name"` + Name string `json:"name"` + KeyName string `json:"key_name"` +} + +// BdevCryptoCreateResult is the result of creating a Crypto Block Device +type BdevCryptoCreateResult string + +// BdevCryptoDeleteParams holds the parameters required to delete a Crypto Block Device +type BdevCryptoDeleteParams struct { + Name string `json:"name"` +} + +// BdevCryptoDeleteResult is the result of deleting a Crypto Block Device +type BdevCryptoDeleteResult bool + +// BdevNvmeAttachControllerParams is the parameters required to create a block device based on an NVMe device +type BdevNvmeAttachControllerParams struct { + Name string `json:"name"` + Trtype string `json:"trtype"` + Traddr string `json:"traddr"` + Hostnqn string `json:"hostnqn,omitempty"` + Adrfam string `json:"adrfam,omitempty"` + Trsvcid string `json:"trsvcid,omitempty"` + Subnqn string `json:"subnqn,omitempty"` + Hdgst bool `json:"hdgst,omitempty"` + Ddgst bool `json:"ddgst,omitempty"` + Psk string `json:"psk,omitempty"` + Multipath string `json:"multipath,omitempty"` +} + +// BdevNvmeAttachControllerResult is the result of creating a block device based on an NVMe device +type BdevNvmeAttachControllerResult string + +// BdevNvmeDetachControllerParams is the parameters required to detach a block device based on an NVMe device +type BdevNvmeDetachControllerParams struct { + Name string `json:"name"` + Trtype string `json:"trtype"` + Traddr string `json:"traddr"` + Adrfam string `json:"adrfam,omitempty"` + Trsvcid string `json:"trsvcid,omitempty"` + Subnqn string `json:"subnqn,omitempty"` +} + +// BdevNvmeDetachControllerResult is the result of detaching a block device based on an NVMe device +type BdevNvmeDetachControllerResult bool + +// BdevNvmeGetControllerParams is the parameters required to get a block device based on an NVMe device +type BdevNvmeGetControllerParams struct { + Name string `json:"name"` +} + +// BdevNvmeGetControllerResult is the result of getting a block device based on an NVMe device +type BdevNvmeGetControllerResult struct { + Name string `json:"name"` + Ctrlrs []struct { + State string `json:"state"` + Trid struct { + Trtype string `json:"trtype"` + Adrfam string `json:"adrfam"` + Traddr string `json:"traddr"` + Trsvcid string `json:"trsvcid"` + Subnqn string `json:"subnqn"` + } `json:"trid"` + Cntlid int `json:"cntlid"` + Host struct { + Nqn string `json:"nqn"` + Addr string `json:"addr"` + Svcid string `json:"svcid"` + } `json:"host"` + } `json:"ctrlrs"` +} + +// BdevGetBdevsParams is the parameters required to get a block device +type BdevGetBdevsParams struct { + Name string `json:"name"` +} + +// BdevGetBdevsResult is the result of getting a block device +type BdevGetBdevsResult struct { + Name string `json:"name"` + BlockSize int64 `json:"block_size"` + NumBlocks int64 `json:"num_blocks"` + UUID string `json:"uuid"` +} + +// BdevGetIostatParams hold the parameters required to get the IO stats of a block device +type BdevGetIostatParams struct { + Name string `json:"name"` +} + +// BdevGetIostatResult hold the results of getting the IO stats of a block device +type BdevGetIostatResult struct { + TickRate int `json:"tick_rate"` + Ticks int64 `json:"ticks"` + Bdevs []struct { + Name string `json:"name"` + BytesRead int `json:"bytes_read"` + NumReadOps int `json:"num_read_ops"` + BytesWritten int `json:"bytes_written"` + NumWriteOps int `json:"num_write_ops"` + BytesUnmapped int `json:"bytes_unmapped"` + NumUnmapOps int `json:"num_unmap_ops"` + ReadLatencyTicks int `json:"read_latency_ticks"` + WriteLatencyTicks int `json:"write_latency_ticks"` + UnmapLatencyTicks int `json:"unmap_latency_ticks"` + } `json:"bdevs"` +} + +// BdevQoSParams holds the parameters required to set QoS on a Block Device +type BdevQoSParams struct { + Name string `json:"name"` + RwIosPerSec int `json:"rw_ios_per_sec"` + RwMbytesPerSec int `json:"rw_mbytes_per_sec"` + RMbytesPerSec int `json:"r_mbytes_per_sec"` + WMbytesPerSec int `json:"w_mbytes_per_sec"` +} + +// BdevQoSResult is the result of setting QoS on a Block Device +type BdevQoSResult bool + +// VhostCreateBlkControllerParams holds the parameters required to create a block device +// from a vhost controller +type VhostCreateBlkControllerParams struct { + Ctrlr string `json:"ctrlr"` + DevName string `json:"dev_name"` +} + +// VhostCreateBlkControllerResult is the result of creating a block device from a vhost controller +type VhostCreateBlkControllerResult bool + +// VhostDeleteControllerParams holds the parameters required to delete a vhost controller +type VhostDeleteControllerParams struct { + Ctrlr string `json:"ctrlr"` +} + +// VhostDeleteControllerResult is the result of deleting a vhost controller +type VhostDeleteControllerResult bool + +// VhostGetControllersParams holds the parameters required to get a vhost controller +type VhostGetControllersParams struct { + Name string `json:"name"` +} + +// VhostGetControllersResult is the result of getting a vhost controller +type VhostGetControllersResult struct { + Ctrlr string `json:"ctrlr"` + Cpumask string `json:"cpumask"` + DelayBaseUs int `json:"delay_base_us"` + IopsThreshold int `json:"iops_threshold"` + Socket string `json:"socket"` + BackendSpecific struct { + Block struct { + Readonly bool `json:"readonly"` + Bdev string `json:"bdev"` + } `json:"block"` + } `json:"backend_specific"` +} + +// VhostCreateScsiControllerParams holds the parameters required to create a SCSI controller +type VhostCreateScsiControllerParams struct { + Ctrlr string `json:"ctrlr"` +} + +// VhostCreateScsiControllerResult is the result of creating a SCSI controller +type VhostCreateScsiControllerResult bool + +// NvmfSubsystemAddNsParams holds the parameters required to add a namespace to an existing subsystem +type NvmfSubsystemAddNsParams struct { + Nqn string `json:"nqn"` + Namespace struct { + Nsid int `json:"nsid"` + BdevName string `json:"bdev_name"` + } `json:"namespace"` +} + +// NvmfSubsystemAddNsResult is the result NSID of attaching a namespace to an existing subsystem +type NvmfSubsystemAddNsResult int + +// NvmfSubsystemRemoveNsParams holds the parameters required to Delete a NVMf subsystem +type NvmfSubsystemRemoveNsParams struct { + Nqn string `json:"nqn"` + Nsid int `json:"nsid"` +} + +// NvmfSubsystemRemoveNsResult is the result of creating a NVMf subsystem +type NvmfSubsystemRemoveNsResult bool + +// NvmfCreateSubsystemParams holds the parameters required to create a NVMf subsystem +type NvmfCreateSubsystemParams struct { + Nqn string `json:"nqn"` + SerialNumber string `json:"serial_number"` + ModelNumber string `json:"model_number"` + AllowAnyHost bool `json:"allow_any_host"` + MaxNamespaces int `json:"max_namespaces"` +} + +// NvmfCreateSubsystemResult is the result of creating a NVMf subsystem +type NvmfCreateSubsystemResult bool + +// NvmfDeleteSubsystemParams holds the parameters required to Delete a NVMf subsystem +type NvmfDeleteSubsystemParams struct { + Nqn string `json:"nqn"` +} + +// NvmfDeleteSubsystemResult is the result of creating a NVMf subsystem +type NvmfDeleteSubsystemResult bool + +// NvmfGetSubsystemsResult is the result of listing all NVMf subsystems +type NvmfGetSubsystemsResult struct { + Nqn string `json:"nqn"` + Subtype string `json:"subtype"` + ListenAddresses []interface{} `json:"listen_addresses"` + AllowAnyHost bool `json:"allow_any_host"` + Hosts []interface{} `json:"hosts"` + SerialNumber string `json:"serial_number,omitempty"` + ModelNumber string `json:"model_number,omitempty"` + MaxNamespaces int `json:"max_namespaces,omitempty"` + MinCntlid int `json:"min_cntlid,omitempty"` + MaxCntlid int `json:"max_cntlid,omitempty"` + Namespaces []struct { + Nsid int `json:"nsid"` + Name string `json:"name"` + } `json:"namespaces,omitempty"` +} + +// NvmfGetSubsystemStatsResult is the result of NVMf subsystem statistics +type NvmfGetSubsystemStatsResult struct { + TickRate int `json:"tick_rate"` + PollGroups []struct { + Name string `json:"name"` + AdminQpairs int `json:"admin_qpairs"` + IoQpairs int `json:"io_qpairs"` + CurrentAdminQpairs int `json:"current_admin_qpairs"` + CurrentIoQpairs int `json:"current_io_qpairs"` + PendingBdevIo int `json:"pending_bdev_io"` + Transports []struct { + Trtype string `json:"trtype"` + } `json:"transports"` + } `json:"poll_groups"` +} + +// NvmfSubsystemAddListenerParams holds the parameters required to Delete a NVMf subsystem +type NvmfSubsystemAddListenerParams struct { + Nqn string `json:"nqn"` + SecureChannel bool `json:"secure_channel,omitempty"` + ListenAddress struct { + Trtype string `json:"trtype"` + Traddr string `json:"traddr"` + Trsvcid string `json:"trsvcid,omitempty"` + Adrfam string `json:"adrfam,omitempty"` + } `json:"listen_address"` +} + +// NvmfSubsystemAddListenerResult is the result of creating a NVMf subsystem +type NvmfSubsystemAddListenerResult bool + +// NvmfSubsystemAddHostParams holds the parameters required to add a host to NVMf subsystem +type NvmfSubsystemAddHostParams struct { + Nqn string `json:"nqn"` + Host string `json:"host"` + TgtName string `json:"tgt_name,omitempty"` + Psk string `json:"psk,omitempty"` +} + +// NvmfSubsystemAddHostResult is the result of adding host to NVMf subsystem +type NvmfSubsystemAddHostResult bool From 56e974190f4f701e69e28d3170658a51444e2d53 Mon Sep 17 00:00:00 2001 From: Artsiom Koltun Date: Wed, 24 Jan 2024 13:15:31 +0100 Subject: [PATCH 02/11] chore(deps): get spdk client Signed-off-by: Artsiom Koltun --- go.mod | 3 ++- go.sum | 22 ++++++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 2e225299..2f666039 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/opiproject/opi-spdk-bridge -go 1.19 +go 1.21 require ( github.com/digitalocean/go-qemu v0.0.0-20230711162256-2e3d0186973e @@ -14,6 +14,7 @@ require ( github.com/philippgille/gokv v0.6.0 github.com/philippgille/gokv/gomap v0.6.0 github.com/philippgille/gokv/redis v0.6.0 + github.com/spdk/spdk/go/rpc v0.0.0-20240118091712-b0eb3ed8f752 github.com/vektra/mockery/v2 v2.38.0 go.einride.tech/aip v0.66.0 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1 diff --git a/go.sum b/go.sum index 65b26c58..1081feff 100644 --- a/go.sum +++ b/go.sum @@ -29,7 +29,9 @@ cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUM cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/compute v1.23.3 h1:6sVlXXBmbd7jNX0Ipq0trII3e4n1/MsADLK6a+aiVlk= +cloud.google.com/go/compute v1.23.3/go.mod h1:VCgBUoMnIVIR0CscqQiPJLAG25E3ZRZMzcFZeQ+h8CI= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= +cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= @@ -66,9 +68,11 @@ github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF0 github.com/OpenPeeDeeP/depguard/v2 v2.1.0 h1:aQl70G173h/GZYhWf36aE5H0KaujXfVMnn/f1kSDVYY= github.com/OpenPeeDeeP/depguard/v2 v2.1.0/go.mod h1:PUBgk35fX4i7JDmwzlJwJ+GMe6NfO1723wmJMgPThNQ= github.com/alecthomas/assert/v2 v2.2.2 h1:Z/iVC0xZfWTaFNE6bA3z07T86hd45Xe2eLt6WVy2bbk= +github.com/alecthomas/assert/v2 v2.2.2/go.mod h1:pXcQ2Asjp247dahGEmsZ6ru0UVwnkhktn7S0bBDLxvQ= github.com/alecthomas/go-check-sumtype v0.1.3 h1:M+tqMxB68hcgccRXBMVCPI4UJ+QUfdSx0xdbypKCqA8= github.com/alecthomas/go-check-sumtype v0.1.3/go.mod h1:WyYPfhfkdhyrdaligV6svFopZV8Lqdzn5pyVBaV6jhQ= github.com/alecthomas/repr v0.2.0 h1:HAzS41CIzNW5syS8Mf9UwXhNH1J9aix/BvDRf1Ml2Yk= +github.com/alecthomas/repr v0.2.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -130,6 +134,7 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 h1:/inchEIKaYC1Akx+H+gqO04wryn5h75LSazbRlnya1k= +github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/coreos/go-systemd/v22 v22.3.3-0.20220203105225-a9a7ef127534/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/curioswitch/go-reassign v0.2.0 h1:G9UZyOcpk/d7Gd6mqYgd8XYWFMw/znxwGDUstnC9DIo= @@ -152,6 +157,7 @@ github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5y github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/envoyproxy/protoc-gen-validate v1.0.2 h1:QkIBuU5k+x7/QXPvPPnWXWlCdaBFApVqftFV6k087DA= +github.com/envoyproxy/protoc-gen-validate v1.0.2/go.mod h1:GpiZQP3dDbg4JouG/NNS7QWXpgx6x8QiMKdmN72jogE= github.com/esimonov/ifshort v1.0.4 h1:6SID4yGWfRae/M7hkVDVVyppy8q/v9OuxNdmjLQStBA= github.com/esimonov/ifshort v1.0.4/go.mod h1:Pe8zjlRrJ80+q2CxHLfEOfTwxCZ4O+MuhcHcfgNWTk0= github.com/ettle/strcase v0.1.1 h1:htFueZyVeE1XNnMEfbqp5r67qAN/4r6ya1ysq8Q+Zcw= @@ -163,6 +169,7 @@ github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4 github.com/firefart/nonamedreturns v1.0.4 h1:abzI1p7mAEPYuR4A+VLKn4eNDOycjYo2phmY9sfv40Y= github.com/firefart/nonamedreturns v1.0.4/go.mod h1:TDhe/tjI1BXo48CmYbUduTV7BdIga8MAO/xbKdcVsGI= github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= +github.com/frankban/quicktest v1.14.4/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= @@ -205,6 +212,7 @@ github.com/go-toolsmith/astfmt v1.1.0/go.mod h1:OrcLlRwu0CuiIBp/8b5PYF9ktGVZUjlN github.com/go-toolsmith/astp v1.1.0 h1:dXPuCl6u2llURjdPLLDxJeZInAeZ0/eZwFJmqZMnpQA= github.com/go-toolsmith/astp v1.1.0/go.mod h1:0T1xFGz9hicKs8Z5MfAqSUitoUYS30pDMsRVIDHs8CA= github.com/go-toolsmith/pkgload v1.2.2 h1:0CtmHq/02QhxcF7E9N5LIFcYFsMR5rdovfqTtRKkgIk= +github.com/go-toolsmith/pkgload v1.2.2/go.mod h1:R2hxLNRKuAsiXCo2i5J6ZQPhnPMOVtU+f0arbFPWCus= github.com/go-toolsmith/strparse v1.0.0/go.mod h1:YI2nUKP9YGZnL/L1/DLFBfixrcjslWct4wyljWhSRy8= github.com/go-toolsmith/strparse v1.1.0 h1:GAioeZUK9TGxnLS+qfdqNbA4z0SSm5zVNtCQiyP2Bvw= github.com/go-toolsmith/strparse v1.1.0/go.mod h1:7ksGy58fsaQkGQlY8WVoBFNyEPMGuJin1rfoPS4lBSQ= @@ -319,6 +327,7 @@ github.com/gostaticanalysis/nilerr v0.1.1 h1:ThE+hJP0fEp4zWLkWHWcRyI2Od0p7DlgYG3 github.com/gostaticanalysis/nilerr v0.1.1/go.mod h1:wZYb6YI5YAxxq0i1+VJbY0s2YONW0HU0GPE3+5PWN4A= github.com/gostaticanalysis/testutil v0.3.1-0.20210208050101-bfb5c8eec0e4/go.mod h1:D+FIZ+7OahH3ePw/izIEeH5I06eKs1IKI4Xr64/Am3M= github.com/gostaticanalysis/testutil v0.4.0 h1:nhdCmubdmDF6VEatUNjgUZBJKWRqugoISdUv3PPQgHY= +github.com/gostaticanalysis/testutil v0.4.0/go.mod h1:bLIoPefWXrRi/ssLFWX1dx7Repi5x3CuviD3dgAZaBU= github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.1 h1:HcUWd006luQPljE73d5sk+/VgYPGUReEVz2y1/qylwY= github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.1/go.mod h1:w9Y7gY31krpLmrVU5ZPG9H7l9fZuRu5/3R3S3FMtVQ4= github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.0 h1:Wqo399gCIufwto+VfwCSvsnfGpF/w5E9CNxSwbpD6No= @@ -377,9 +386,11 @@ github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kulti/thelper v0.6.3 h1:ElhKf+AlItIu+xGnI990no4cE2+XaSu1ULymV2Yulxs= github.com/kulti/thelper v0.6.3/go.mod h1:DsqKShOvP40epevkFrvIwkCMNYxMeTNjdWL4dqWHZ6I= github.com/kunwardeep/paralleltest v1.0.8 h1:Ul2KsqtzFxTlSU7IP0JusWlLiNqQaloB9vguyjbE558= @@ -451,12 +462,14 @@ github.com/onsi/ginkgo/v2 v2.14.0 h1:vSmGj2Z5YPb9JwCWT6z6ihcUvDhuXLc3sJiqd3jMKAY github.com/onsi/ginkgo/v2 v2.14.0/go.mod h1:JkUdW7JkN0V6rFvsHcJ478egV3XH9NxpD27Hal/PhZw= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.30.0 h1:hvMK7xYz4D3HapigLTeGdId/NcfQx1VHMJc60ew99+8= +github.com/onsi/gomega v1.30.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ= github.com/opiproject/gospdk v0.0.0-20240108075015-92d689fff706 h1:fZcUuXgfsfWIbSZTPCJG72wh7LyKIyrpPR9kH+rKEXI= github.com/opiproject/gospdk v0.0.0-20240108075015-92d689fff706/go.mod h1:9CMbTd9ptR6tl6HRRn8C33DPeWF85hTo4KZCa5iKftY= github.com/opiproject/opi-api v0.0.0-20240118183513-e44db269fba4 h1:YBjvYWQQAbNIGsAXvB6FwL9Encr1nzo3/w+bB/tXltM= github.com/opiproject/opi-api v0.0.0-20240118183513-e44db269fba4/go.mod h1:92pv4ulvvPMuxCJ9ND3aYbmBfEMLx0VCjpkiR7ZTqPY= github.com/otiai10/copy v1.2.0/go.mod h1:rrF5dJ5F0t/EWSYODDu4j9/vEeYHMkc8jt0zJChqQWw= github.com/otiai10/copy v1.11.0 h1:OKBD80J/mLBrwnzXqGtFCzprFSGioo30JcmR4APsNwc= +github.com/otiai10/copy v1.11.0/go.mod h1:rSaLseMUsZFFbsFGc7wCJnnkTAvdc5L6VWxPE4308Ww= github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE= github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6j4vs= github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo= @@ -518,6 +531,7 @@ github.com/quasilyte/stdinfo v0.0.0-20220114132959-f7386bf02567/go.mod h1:DWNGW8 github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.29.0 h1:Zes4hju04hjbvkVkOhdl2HpZa+0PmVwigmo8XoORE5w= github.com/rs/zerolog v1.29.0/go.mod h1:NILgTygv/Uej1ra5XxGf82ZFSLk58MFGAUS2o6usyD0= @@ -553,6 +567,8 @@ github.com/sonatard/noctx v0.0.2 h1:L7Dz4De2zDQhW8S0t+KUjY0MAQJd6SgVwhzNIc4ok00= github.com/sonatard/noctx v0.0.2/go.mod h1:kzFz+CzWSjQ2OzIm46uJZoXuBpa2+0y3T36U18dWqIo= github.com/sourcegraph/go-diff v0.7.0 h1:9uLlrd5T46OXs5qpp8L/MTltk0zikUGi0sNNyCpA8G0= github.com/sourcegraph/go-diff v0.7.0/go.mod h1:iBszgVvyxdc8SFZ7gm69go2KDdt3ag071iBaWPF6cjs= +github.com/spdk/spdk/go/rpc v0.0.0-20240118091712-b0eb3ed8f752 h1:i2B7K0PNHJwspiFPQmw/7LU1yu5NZ2kma+itV8dI1qs= +github.com/spdk/spdk/go/rpc v0.0.0-20240118091712-b0eb3ed8f752/go.mod h1:rpOwUxkcLJ5j1SB80YgfM6277k9RDRxvBMQomeho0EE= github.com/spf13/afero v1.9.3 h1:41FoI0fD7OR7mGcKE/aOiLkGreyf8ifIOQmJANWogMk= github.com/spf13/afero v1.9.3/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y= github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w= @@ -631,6 +647,7 @@ github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5t gitlab.com/bosi/decorder v0.4.1 h1:VdsdfxhstabyhZovHafFw+9eJ6eU0d2CkFNJcZz/NU4= gitlab.com/bosi/decorder v0.4.1/go.mod h1:jecSqWUew6Yle1pCr2eLWTensJMmsxHsBwt+PVbkAqA= go-simpler.org/assert v0.6.0 h1:QxSrXa4oRuo/1eHMXSBFHKvJIpWABayzKldqZyugG7E= +go-simpler.org/assert v0.6.0/go.mod h1:74Eqh5eI6vCK6Y5l3PI8ZYFXG4Sa+tkr70OIPJAUr28= go-simpler.org/sloglint v0.1.2 h1:IjdhF8NPxyn0Ckn2+fuIof7ntSnVUAqBFcQRrnG9AiM= go-simpler.org/sloglint v0.1.2/go.mod h1:2LL+QImPfTslD5muNPydAEYmpXIj6o/WYcqnJjLi4o4= go.einride.tech/aip v0.66.0 h1:XfV+NQX6L7EOYK11yoHHFtndeaWh3KbD9/cN/6iWEt8= @@ -663,6 +680,7 @@ go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.8.0 h1:dg6GjLku4EH+249NNmoIciG9N/jURbDG+pFlTkhzIC8= go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= @@ -780,6 +798,7 @@ golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.15.0 h1:s8pnnxNVzjWyrvYdFUQq5llS1PX2zhPXmccZv99h7uQ= +golang.org/x/oauth2 v0.15.0/go.mod h1:q48ptWNTY5XWf+JNten23lcvHpLJ0ZSxF5ttTHKVCAM= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -983,6 +1002,7 @@ google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCID google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= +google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -1062,6 +1082,7 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= @@ -1081,6 +1102,7 @@ gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU= +gotest.tools/v3 v3.5.1/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= From 6d6fedb9d63c70591fa95307fed34b095bdcbaed Mon Sep 17 00:00:00 2001 From: Artsiom Koltun Date: Wed, 24 Jan 2024 13:01:17 +0100 Subject: [PATCH 03/11] refactor: add spdk client adapter Signed-off-by: Artsiom Koltun --- pkg/spdk/adapter.go | 77 +++++++++++++++++++++++++++++++++++ pkg/spdk/adapter_test.go | 86 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 163 insertions(+) create mode 100644 pkg/spdk/adapter.go create mode 100644 pkg/spdk/adapter_test.go diff --git a/pkg/spdk/adapter.go b/pkg/spdk/adapter.go new file mode 100644 index 00000000..b237da26 --- /dev/null +++ b/pkg/spdk/adapter.go @@ -0,0 +1,77 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright (C) 2024 Intel Corporation + +// Package spdk implements the spdk json-rpc protocol +package spdk + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + + "github.com/spdk/spdk/go/rpc/client" +) + +// SpdkClientAdapter provides an adapter between old gospdk api and new spdk client +// This is needed due to: +// * awkward response unmarshaling in the new client. A Result in Response is +// provided in the form described in json.Unmarshal +// * New error codes affecting a lot of tests. The adapter converts the new +// format to the old one. It enables gradual transition. The tests will be +// reworked to new errors eliminating transformations in the adapter. +type SpdkClientAdapter struct { + client client.IClient +} + +// NewSpdkClientAdapter creates a new instance if SpdkClientAdapter +func NewSpdkClientAdapter(client client.IClient) *SpdkClientAdapter { + return &SpdkClientAdapter{client} +} + +// Call performs a call to spdk client and unmarshalls the result into requested structure +func (c *SpdkClientAdapter) Call(ctx context.Context, method string, params any, result any) error { + ch := make(chan error) + + go func() { + response, err := c.client.Call(method, params) + if err == nil { + var bytes []byte + bytes, err = json.Marshal(response.Result) + if err == nil { + err = json.Unmarshal(bytes, result) + } + } + + ch <- err + }() + + select { + case <-ctx.Done(): + return ctx.Err() + case err := <-ch: + // The unit tests check err messages. The format used in the spdk + // client is different. Convert to the old format to preserve behavior + // and gradually rework the tests + return c.convertToLegacyErrorFormat(err, method) + } +} + +func (c *SpdkClientAdapter) convertToLegacyErrorFormat(err error, method string) error { + e := errors.Unwrap(err) + spdkErr, _ := e.(*client.Error) + + switch { + case err == nil: + return nil + case strings.Contains(err.Error(), "EOF"): + return fmt.Errorf("%s: EOF", method) + case strings.Contains(err.Error(), "mismatch request and response IDs"): + return fmt.Errorf("%s: json response ID mismatch", method) + case strings.Contains(err.Error(), "error received for") && spdkErr != nil: + return fmt.Errorf("%s: json response error: %s", method, spdkErr.Message) + default: + return fmt.Errorf("%s: %s", method, err) + } +} diff --git a/pkg/spdk/adapter_test.go b/pkg/spdk/adapter_test.go new file mode 100644 index 00000000..89aec472 --- /dev/null +++ b/pkg/spdk/adapter_test.go @@ -0,0 +1,86 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright (C) 2023-2024 Intel Corporation + +// Package spdk implements the spdk json-rpc protocol +package spdk + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/spdk/spdk/go/rpc/client" +) + +type StubSpdkClient struct { + StubCall func(method string, params any) (*client.Response, error) +} + +func (m *StubSpdkClient) Call(method string, params any) (*client.Response, error) { + return m.StubCall(method, params) +} + +func TestCall(t *testing.T) { + tests := map[string]struct { + client *StubSpdkClient + wantErr bool + }{ + "successful call": { + client: &StubSpdkClient{ + StubCall: func(method string, params any) (*client.Response, error) { + return &client.Response{Result: "TestResult"}, nil + }, + }, + wantErr: false, + }, + "call error": { + client: &StubSpdkClient{ + StubCall: func(method string, params any) (*client.Response, error) { + return nil, errors.New("call error") + }, + }, + wantErr: true, + }, + "marshall error": { + client: &StubSpdkClient{ + StubCall: func(method string, params any) (*client.Response, error) { + return &client.Response{Result: make(chan int)}, nil // json.Marshal will fail for this + }, + }, + wantErr: true, + }, + "unmarshall error": { + client: &StubSpdkClient{ + StubCall: func(method string, params any) (*client.Response, error) { + return &client.Response{Result: 1.0}, nil + }, + }, + wantErr: true, + }, + "context canceled": { + client: &StubSpdkClient{ + StubCall: func(method string, params any) (*client.Response, error) { + time.Sleep(100 * time.Millisecond) + return &client.Response{Result: "TestResult"}, nil + }, + }, + wantErr: true, + }, + } + + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + c := NewSpdkClientAdapter(tt.client) + + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + + result := "" + err := c.Call(ctx, "TestMethod", "TestParams", &result) + if (err != nil) != tt.wantErr { + t.Errorf("error: %v, want error: %v", err, tt.wantErr) + } + }) + } +} From 8cb2fd3b881a1089b4f069a0539ce3481c4138c6 Mon Sep 17 00:00:00 2001 From: Artsiom Koltun Date: Mon, 29 Jan 2024 09:39:25 +0100 Subject: [PATCH 04/11] test: adjust test infrastructure to spdk client Signed-off-by: Artsiom Koltun --- pkg/utils/server.go | 89 ++++++++++++++++++++++++++++++++++----------- 1 file changed, 67 insertions(+), 22 deletions(-) diff --git a/pkg/utils/server.go b/pkg/utils/server.go index 96e4089a..f493684d 100644 --- a/pkg/utils/server.go +++ b/pkg/utils/server.go @@ -7,6 +7,7 @@ package utils import ( "crypto/rand" + "encoding/json" "fmt" "log" "math/big" @@ -19,8 +20,8 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" - "github.com/opiproject/gospdk/spdk" pb "github.com/opiproject/opi-api/storage/v1alpha1/gen/go" + "github.com/spdk/spdk/go/rpc/client" ) // ExtractPagination is a helper function for List pagination to fetch PageSize and PageToken @@ -65,13 +66,29 @@ func LimitPagination[T any](result []T, offset int, size int) ([]T, bool) { } // CreateTestSpdkServer creates a mock spdk server for testing -func CreateTestSpdkServer(socket string, spdkResponses []string) (net.Listener, spdk.JSONRPC) { - jsonRPC := spdk.NewClient(socket) - ln := jsonRPC.StartUnixListener() +func CreateTestSpdkServer(socket string, spdkResponses []string) (net.Listener, client.IClient) { + ln := startUnixListener(socket) if len(spdkResponses) > 0 { - go spdkMockServerCommunicate(jsonRPC, ln, spdkResponses) + go spdkMockServerCommunicate(ln, spdkResponses) } - return ln, jsonRPC + + spdkClient, err := client.CreateClientWithJsonCodec("unix", socket) + if err != nil { + log.Panicf("Cannot create spdk client for test: %v", err) + } + + return ln, spdkClient +} + +func startUnixListener(socket string) net.Listener { + if err := os.RemoveAll(socket); err != nil { + log.Fatal(err) + } + ln, err := net.Listen("unix", socket) + if err != nil { + log.Fatal("listen error:", err) + } + return ln } // CloseGrpcConnection is utility function used to defer grpc connection close is tests @@ -100,16 +117,15 @@ func GenerateSocketName(testType string) string { return filepath.Join(os.TempDir(), "opi-spdk-"+testType+"-test-"+fmt.Sprint(n)+".sock") } -func spdkMockServerCommunicate(rpc spdk.JSONRPC, l net.Listener, toSend []string) { +func spdkMockServerCommunicate(l net.Listener, toSend []string) { + fd, err := l.Accept() + defer func() { _ = fd.Close() }() for _, spdk := range toSend { // wait for client to connect (accept stage) - fd, err := l.Accept() if err != nil { - log.Fatal("accept error:", err) + log.Panic("accept error:", err) } log.Printf("SPDK mockup Server: client connected [%s]", fd.RemoteAddr().Network()) - id := rpc.GetID() - log.Printf("SPDK ID [%d]", id) // read from client // we just read to extract ID, rest of the data is discarded here buf := make([]byte, 512) @@ -117,31 +133,60 @@ func spdkMockServerCommunicate(rpc spdk.JSONRPC, l net.Listener, toSend []string if err != nil { log.Panic("Read: ", err) } - // fill in ID, since client expects the same ID in the response data := buf[0:nr] + req := client.Request{} + id := uint64(0) + if err := json.Unmarshal(data, &req); err != nil { + log.Printf("request parsing error: %v", err) + } else { + // fill in ID, since client expects the same ID in the response + id = req.ID + } + log.Printf("SPDK ID [%d]", id) if strings.Contains(spdk, "%") { spdk = fmt.Sprintf(spdk, id) } log.Printf("SPDK mockup Server: got : %s", string(data)) log.Printf("SPDK mockup Server: snd : %s", spdk) + + // Remove all zero code errors from test responses since JSON-RPC + // states that error member in response must not exist if there was no + // error triggered during invocation. Delete this part when unit tests + // are reworked. + spdk = removeZeroCode(spdk) + // send data back to client _, err = fd.Write([]byte(spdk)) if err != nil { log.Panic("Write: ", err) } - // close connection - switch fd := fd.(type) { - case *net.TCPConn: - err = fd.CloseWrite() - case *net.UnixConn: - err = fd.CloseWrite() - } - if err != nil { - log.Fatal(err) - } } } +func removeZeroCode(jsonSpdkResponseStr string) string { + if jsonSpdkResponseStr == "" { + return jsonSpdkResponseStr + } + + var response client.Response + err := json.Unmarshal([]byte(jsonSpdkResponseStr), &response) + if err != nil { + return jsonSpdkResponseStr + } + + // If the error code is 0, set the Error field to nil + if response.Error != nil && response.Error.Code == 0 { + response.Error = nil + } + + newResponse, err := json.Marshal(response) + if err != nil { + log.Panicf("error marshalling JSON: %v", err) + } + + return string(newResponse) +} + // OpiAdressFamilyToSpdk converts opi address family to the one used in spdk func OpiAdressFamilyToSpdk(adrfam pb.NvmeAddressFamily) string { if adrfam == pb.NvmeAddressFamily_NVME_ADDRESS_FAMILY_UNSPECIFIED { From dc09e78037a8b375519812d43b7fb0093096f112 Mon Sep 17 00:00:00 2001 From: Artsiom Koltun Date: Mon, 29 Jan 2024 09:40:46 +0100 Subject: [PATCH 05/11] refactor(backend): transition to spdk client Signed-off-by: Artsiom Koltun --- cmd/main.go | 18 +++++++++++++++++- pkg/backend/aio.go | 2 +- pkg/backend/backend.go | 13 +++++++------ pkg/backend/backend_test.go | 8 ++++---- pkg/backend/null.go | 2 +- pkg/backend/nvme_path.go | 2 +- 6 files changed, 31 insertions(+), 14 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index fc8f5d5b..8b30c116 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -24,6 +24,8 @@ import ( pc "github.com/opiproject/opi-api/inventory/v1/gen/go" pb "github.com/opiproject/opi-api/storage/v1alpha1/gen/go" + "github.com/spdk/spdk/go/rpc/client" + "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/reflection" @@ -139,7 +141,21 @@ func runGrpcServer(grpcPort int, useKvm bool, store gokv.Store, spdkAddress, qmp s := grpc.NewServer(serverOptions...) jsonRPC := spdk.NewClient(spdkAddress) - backendServer := backend.NewServer(jsonRPC, store) + protocol := client.TCP + if _, _, err := net.SplitHostPort(spdkAddress); err != nil { + protocol = client.Unix + } + spdkClient, err := client.CreateClientWithJsonCodec(protocol, spdkAddress) + if err != nil { + log.Panicf("spdk client creation failed %v", err) + } + defer func() { + if err := spdkClient.Close(); err != nil { + log.Panicf("spdk client close: %v", err) + } + }() + + backendServer := backend.NewServer(spdkClient, store) middleendServer := middleend.NewServer(jsonRPC, store) if useKvm { diff --git a/pkg/backend/aio.go b/pkg/backend/aio.go index 9ff4e429..0ed799a1 100644 --- a/pkg/backend/aio.go +++ b/pkg/backend/aio.go @@ -12,8 +12,8 @@ import ( "path" "sort" - "github.com/opiproject/gospdk/spdk" pb "github.com/opiproject/opi-api/storage/v1alpha1/gen/go" + "github.com/opiproject/opi-spdk-bridge/pkg/spdk" "github.com/opiproject/opi-spdk-bridge/pkg/utils" "github.com/google/uuid" diff --git a/pkg/backend/backend.go b/pkg/backend/backend.go index 723e7d87..73806f54 100644 --- a/pkg/backend/backend.go +++ b/pkg/backend/backend.go @@ -9,9 +9,10 @@ import ( "log" "github.com/philippgille/gokv" + "github.com/spdk/spdk/go/rpc/client" - "github.com/opiproject/gospdk/spdk" pb "github.com/opiproject/opi-api/storage/v1alpha1/gen/go" + "github.com/opiproject/opi-spdk-bridge/pkg/spdk" "github.com/opiproject/opi-spdk-bridge/pkg/utils" ) @@ -33,7 +34,7 @@ type Server struct { pb.UnimplementedNullVolumeServiceServer pb.UnimplementedAioVolumeServiceServer - rpc spdk.JSONRPC + rpc *spdk.SpdkClientAdapter store gokv.Store Volumes VolumeParameters Pagination map[string]int @@ -42,15 +43,15 @@ type Server struct { // NewServer creates initialized instance of BackEnd server communicating // with provided jsonRPC -func NewServer(jsonRPC spdk.JSONRPC, store gokv.Store) *Server { - if jsonRPC == nil { - log.Panic("nil for JSONRPC is not allowed") +func NewServer(spdkClient client.IClient, store gokv.Store) *Server { + if spdkClient == nil { + log.Panic("nil for spdkClient is not allowed") } if store == nil { log.Panic("nil for Store is not allowed") } return &Server{ - rpc: jsonRPC, + rpc: spdk.NewSpdkClientAdapter(spdkClient), store: store, Volumes: VolumeParameters{ AioVolumes: make(map[string]*pb.AioVolume), diff --git a/pkg/backend/backend_test.go b/pkg/backend/backend_test.go index da995185..ed1806d0 100644 --- a/pkg/backend/backend_test.go +++ b/pkg/backend/backend_test.go @@ -16,8 +16,8 @@ import ( "google.golang.org/grpc/test/bufconn" "github.com/philippgille/gokv/gomap" + "github.com/spdk/spdk/go/rpc/client" - "github.com/opiproject/gospdk/spdk" pb "github.com/opiproject/opi-api/storage/v1alpha1/gen/go" "github.com/opiproject/opi-spdk-bridge/pkg/utils" ) @@ -48,7 +48,7 @@ type testEnv struct { testSocket string ctx context.Context conn *grpc.ClientConn - jsonRPC spdk.JSONRPC + spdkClient client.IClient } func (e *testEnv) Close() { @@ -62,11 +62,11 @@ func (e *testEnv) Close() { func createTestEnvironment(spdkResponses []string) *testEnv { env := &testEnv{} env.testSocket = utils.GenerateSocketName("backend") - env.ln, env.jsonRPC = utils.CreateTestSpdkServer(env.testSocket, spdkResponses) + env.ln, env.spdkClient = utils.CreateTestSpdkServer(env.testSocket, spdkResponses) options := gomap.DefaultOptions options.Codec = utils.ProtoCodec{} store := gomap.NewStore(options) - env.opiSpdkServer = NewServer(env.jsonRPC, store) + env.opiSpdkServer = NewServer(env.spdkClient, store) ctx := context.Background() conn, err := grpc.DialContext(ctx, diff --git a/pkg/backend/null.go b/pkg/backend/null.go index 8f6cb0e8..7d3670f4 100644 --- a/pkg/backend/null.go +++ b/pkg/backend/null.go @@ -12,8 +12,8 @@ import ( "path" "sort" - "github.com/opiproject/gospdk/spdk" pb "github.com/opiproject/opi-api/storage/v1alpha1/gen/go" + "github.com/opiproject/opi-spdk-bridge/pkg/spdk" "github.com/opiproject/opi-spdk-bridge/pkg/utils" "github.com/google/uuid" diff --git a/pkg/backend/nvme_path.go b/pkg/backend/nvme_path.go index 4aafcebd..3b0f1f86 100644 --- a/pkg/backend/nvme_path.go +++ b/pkg/backend/nvme_path.go @@ -14,8 +14,8 @@ import ( "sort" "strings" - "github.com/opiproject/gospdk/spdk" pb "github.com/opiproject/opi-api/storage/v1alpha1/gen/go" + "github.com/opiproject/opi-spdk-bridge/pkg/spdk" "github.com/opiproject/opi-spdk-bridge/pkg/utils" "github.com/google/uuid" From cff9e85f2ce1831a2ce046240dd7420193bb498d Mon Sep 17 00:00:00 2001 From: Artsiom Koltun Date: Thu, 25 Jan 2024 16:16:14 +0100 Subject: [PATCH 06/11] refactor(frontend): transition to spdk client Signed-off-by: Artsiom Koltun --- cmd/main.go | 8 ++++---- pkg/frontend/blk.go | 6 +++--- pkg/frontend/frontend.go | 21 +++++++++++---------- pkg/frontend/frontend_test.go | 33 ++++++++++++++++++++------------- pkg/frontend/nvme_namespace.go | 2 +- pkg/frontend/nvme_subsystem.go | 2 +- pkg/frontend/scsi.go | 2 +- pkg/frontend/transport.go | 13 +++++++------ pkg/frontend/transport_test.go | 9 +++++---- 9 files changed, 53 insertions(+), 43 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index 8b30c116..bc299004 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -160,10 +160,10 @@ func runGrpcServer(grpcPort int, useKvm bool, store gokv.Store, spdkAddress, qmp if useKvm { log.Println("Creating KVM server.") - frontendServer := frontend.NewCustomizedServer(jsonRPC, + frontendServer := frontend.NewCustomizedServer(spdkClient, store, map[pb.NvmeTransportType]frontend.NvmeTransport{ - pb.NvmeTransportType_NVME_TRANSPORT_TYPE_TCP: frontend.NewNvmeTCPTransport(jsonRPC), + pb.NvmeTransportType_NVME_TRANSPORT_TYPE_TCP: frontend.NewNvmeTCPTransport(spdkClient), pb.NvmeTransportType_NVME_TRANSPORT_TYPE_PCIE: kvm.NewNvmeVfiouserTransport(ctrlrDir, jsonRPC), }, frontend.NewVhostUserBlkTransport(), @@ -174,10 +174,10 @@ func runGrpcServer(grpcPort int, useKvm bool, store gokv.Store, spdkAddress, qmp pb.RegisterFrontendVirtioBlkServiceServer(s, kvmServer) pb.RegisterFrontendVirtioScsiServiceServer(s, kvmServer) } else { - frontendServer := frontend.NewCustomizedServer(jsonRPC, + frontendServer := frontend.NewCustomizedServer(spdkClient, store, map[pb.NvmeTransportType]frontend.NvmeTransport{ - pb.NvmeTransportType_NVME_TRANSPORT_TYPE_TCP: frontend.NewNvmeTCPTransport(jsonRPC), + pb.NvmeTransportType_NVME_TRANSPORT_TYPE_TCP: frontend.NewNvmeTCPTransport(spdkClient), }, frontend.NewVhostUserBlkTransport(), ) diff --git a/pkg/frontend/blk.go b/pkg/frontend/blk.go index 10b1f0ff..da816a8e 100644 --- a/pkg/frontend/blk.go +++ b/pkg/frontend/blk.go @@ -12,8 +12,8 @@ import ( "path" "sort" - "github.com/opiproject/gospdk/spdk" pb "github.com/opiproject/opi-api/storage/v1alpha1/gen/go" + "github.com/opiproject/opi-spdk-bridge/pkg/spdk" "github.com/opiproject/opi-spdk-bridge/pkg/utils" "github.com/google/uuid" @@ -59,7 +59,7 @@ func (s *Server) CreateVirtioBlk(ctx context.Context, in *pb.CreateVirtioBlkRequ } var result spdk.VhostCreateBlkControllerResult - err = s.rpc.Call(ctx, "vhost_create_blk_controller", ¶ms, &result) + err = s.rpc.Call(ctx, "vhost_create_blk_controller", params, &result) if err != nil { return nil, err } @@ -97,7 +97,7 @@ func (s *Server) DeleteVirtioBlk(ctx context.Context, in *pb.DeleteVirtioBlkRequ } var result spdk.VhostDeleteControllerResult - err = s.rpc.Call(ctx, "vhost_delete_controller", ¶ms, &result) + err = s.rpc.Call(ctx, "vhost_delete_controller", params, &result) if err != nil { return nil, err } diff --git a/pkg/frontend/frontend.go b/pkg/frontend/frontend.go index 15003870..f3a01512 100644 --- a/pkg/frontend/frontend.go +++ b/pkg/frontend/frontend.go @@ -9,9 +9,10 @@ import ( "log" "github.com/philippgille/gokv" + "github.com/spdk/spdk/go/rpc/client" - "github.com/opiproject/gospdk/spdk" pb "github.com/opiproject/opi-api/storage/v1alpha1/gen/go" + "github.com/opiproject/opi-spdk-bridge/pkg/spdk" "github.com/opiproject/opi-spdk-bridge/pkg/utils" ) @@ -37,7 +38,7 @@ type Server struct { pb.UnimplementedFrontendVirtioBlkServiceServer pb.UnimplementedFrontendVirtioScsiServiceServer - rpc spdk.JSONRPC + rpc *spdk.SpdkClientAdapter store gokv.Store Nvme NvmeParameters Virt VirtioParameters @@ -48,22 +49,22 @@ type Server struct { // NewServer creates initialized instance of FrontEnd server communicating // with provided jsonRPC -func NewServer(jsonRPC spdk.JSONRPC, store gokv.Store) *Server { - if jsonRPC == nil { - log.Panic("nil for JSONRPC is not allowed") +func NewServer(spdkClient client.IClient, store gokv.Store) *Server { + if spdkClient == nil { + log.Panic("nil for spdkClient is not allowed") } if store == nil { log.Panic("nil for Store is not allowed") } return &Server{ - rpc: jsonRPC, + rpc: spdk.NewSpdkClientAdapter(spdkClient), store: store, Nvme: NvmeParameters{ Subsystems: make(map[string]*pb.NvmeSubsystem), Controllers: make(map[string]*pb.NvmeController), Namespaces: make(map[string]*pb.NvmeNamespace), transports: map[pb.NvmeTransportType]NvmeTransport{ - pb.NvmeTransportType_NVME_TRANSPORT_TYPE_TCP: NewNvmeTCPTransport(jsonRPC), + pb.NvmeTransportType_NVME_TRANSPORT_TYPE_TCP: NewNvmeTCPTransport(spdkClient), }, }, Virt: VirtioParameters{ @@ -81,7 +82,7 @@ func NewServer(jsonRPC spdk.JSONRPC, store gokv.Store) *Server { // NewCustomizedServer creates initialized instance of FrontEnd server communicating // with provided jsonRPC and externally created NvmeTransport and VirtioBlkTransport func NewCustomizedServer( - jsonRPC spdk.JSONRPC, + spdkClient client.IClient, store gokv.Store, nvmeTransports map[pb.NvmeTransportType]NvmeTransport, virtioBlkTransport VirtioBlkTransport, @@ -100,14 +101,14 @@ func NewCustomizedServer( log.Panic("nil for VirtioBlkTransport is not allowed") } - if jsonRPC == nil { + if spdkClient == nil { log.Panic("nil for JSONRPC is not allowed") } if store == nil { log.Panic("nil for Store is not allowed") } - server := NewServer(jsonRPC, store) + server := NewServer(spdkClient, store) server.Nvme.transports = nvmeTransports server.Virt.transport = virtioBlkTransport return server diff --git a/pkg/frontend/frontend_test.go b/pkg/frontend/frontend_test.go index 13a2a363..8ed459f8 100644 --- a/pkg/frontend/frontend_test.go +++ b/pkg/frontend/frontend_test.go @@ -17,8 +17,8 @@ import ( "google.golang.org/grpc/test/bufconn" "github.com/philippgille/gokv/gomap" + "github.com/spdk/spdk/go/rpc/client" - "github.com/opiproject/gospdk/spdk" pb "github.com/opiproject/opi-api/storage/v1alpha1/gen/go" "github.com/opiproject/opi-spdk-bridge/pkg/utils" ) @@ -45,7 +45,7 @@ type testEnv struct { testSocket string ctx context.Context conn *grpc.ClientConn - jsonRPC spdk.JSONRPC + spdkClient client.IClient } func (e *testEnv) Close() { @@ -59,11 +59,11 @@ func (e *testEnv) Close() { func createTestEnvironment(spdkResponses []string) *testEnv { env := &testEnv{} env.testSocket = utils.GenerateSocketName("frontend") - env.ln, env.jsonRPC = utils.CreateTestSpdkServer(env.testSocket, spdkResponses) + env.ln, env.spdkClient = utils.CreateTestSpdkServer(env.testSocket, spdkResponses) options := gomap.DefaultOptions options.Codec = utils.ProtoCodec{} store := gomap.NewStore(options) - env.opiSpdkServer = NewServer(env.jsonRPC, store) + env.opiSpdkServer = NewServer(env.spdkClient, store) ctx := context.Background() conn, err := grpc.DialContext(ctx, @@ -103,8 +103,15 @@ func dialer(opiSpdkServer *Server) func(context.Context, string) (net.Conn, erro } } +type spdkCLientStub struct { +} + +func (s spdkCLientStub) Call(string, any) (*client.Response, error) { + return nil, nil +} + func TestFrontEnd_NewCustomizedServer(t *testing.T) { - validJSONRPC := spdk.NewClient("/some/path") + validJSONRPC := spdkCLientStub{} validNvmeTransports := map[pb.NvmeTransportType]NvmeTransport{ pb.NvmeTransportType_NVME_TRANSPORT_TYPE_TCP: NewNvmeTCPTransport(validJSONRPC), } @@ -112,29 +119,29 @@ func TestFrontEnd_NewCustomizedServer(t *testing.T) { validStore := gomap.NewStore(gomap.DefaultOptions) tests := map[string]struct { - jsonRPC spdk.JSONRPC + spdkClient client.IClient store gomap.Store nvmeTransports map[pb.NvmeTransportType]NvmeTransport virtioBlkTransport VirtioBlkTransport wantPanic bool }{ "nil json rpc": { - jsonRPC: nil, + spdkClient: nil, store: validStore, nvmeTransports: validNvmeTransports, virtioBlkTransport: validVirtioBLkTransport, wantPanic: true, }, "nil nvme transports": { - jsonRPC: validJSONRPC, + spdkClient: validJSONRPC, store: validStore, nvmeTransports: nil, virtioBlkTransport: validVirtioBLkTransport, wantPanic: true, }, "nil one of nvme transports": { - jsonRPC: validJSONRPC, - store: validStore, + spdkClient: validJSONRPC, + store: validStore, nvmeTransports: map[pb.NvmeTransportType]NvmeTransport{ pb.NvmeTransportType_NVME_TRANSPORT_TYPE_TCP: nil, }, @@ -142,14 +149,14 @@ func TestFrontEnd_NewCustomizedServer(t *testing.T) { wantPanic: true, }, "nil virtio blk transport": { - jsonRPC: validJSONRPC, + spdkClient: validJSONRPC, store: validStore, nvmeTransports: validNvmeTransports, virtioBlkTransport: nil, wantPanic: true, }, "all valid arguments": { - jsonRPC: validJSONRPC, + spdkClient: validJSONRPC, store: validStore, nvmeTransports: validNvmeTransports, virtioBlkTransport: validVirtioBLkTransport, @@ -166,7 +173,7 @@ func TestFrontEnd_NewCustomizedServer(t *testing.T) { } }() - server := NewCustomizedServer(tt.jsonRPC, tt.store, tt.nvmeTransports, tt.virtioBlkTransport) + server := NewCustomizedServer(tt.spdkClient, tt.store, tt.nvmeTransports, tt.virtioBlkTransport) if server == nil && !tt.wantPanic { t.Error("expected non nil server or panic") } diff --git a/pkg/frontend/nvme_namespace.go b/pkg/frontend/nvme_namespace.go index 01cdbadb..773b107d 100644 --- a/pkg/frontend/nvme_namespace.go +++ b/pkg/frontend/nvme_namespace.go @@ -12,8 +12,8 @@ import ( "path" "sort" - "github.com/opiproject/gospdk/spdk" pb "github.com/opiproject/opi-api/storage/v1alpha1/gen/go" + "github.com/opiproject/opi-spdk-bridge/pkg/spdk" "github.com/opiproject/opi-spdk-bridge/pkg/utils" "github.com/google/uuid" diff --git a/pkg/frontend/nvme_subsystem.go b/pkg/frontend/nvme_subsystem.go index a0d0fe5e..62d507e9 100644 --- a/pkg/frontend/nvme_subsystem.go +++ b/pkg/frontend/nvme_subsystem.go @@ -13,8 +13,8 @@ import ( "path" "sort" - "github.com/opiproject/gospdk/spdk" pb "github.com/opiproject/opi-api/storage/v1alpha1/gen/go" + "github.com/opiproject/opi-spdk-bridge/pkg/spdk" "github.com/opiproject/opi-spdk-bridge/pkg/utils" "github.com/google/uuid" diff --git a/pkg/frontend/scsi.go b/pkg/frontend/scsi.go index 41f8c62e..01e490ec 100644 --- a/pkg/frontend/scsi.go +++ b/pkg/frontend/scsi.go @@ -12,8 +12,8 @@ import ( "path" "sort" - "github.com/opiproject/gospdk/spdk" pb "github.com/opiproject/opi-api/storage/v1alpha1/gen/go" + "github.com/opiproject/opi-spdk-bridge/pkg/spdk" "github.com/opiproject/opi-spdk-bridge/pkg/utils" "github.com/google/uuid" diff --git a/pkg/frontend/transport.go b/pkg/frontend/transport.go index 03dcaf25..cb24eda8 100644 --- a/pkg/frontend/transport.go +++ b/pkg/frontend/transport.go @@ -12,9 +12,10 @@ import ( "log" "path" - "github.com/opiproject/gospdk/spdk" pb "github.com/opiproject/opi-api/storage/v1alpha1/gen/go" + "github.com/opiproject/opi-spdk-bridge/pkg/spdk" "github.com/opiproject/opi-spdk-bridge/pkg/utils" + "github.com/spdk/spdk/go/rpc/client" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) @@ -34,20 +35,20 @@ type VirtioBlkTransport interface { } type nvmeTCPTransport struct { - rpc spdk.JSONRPC + rpc *spdk.SpdkClientAdapter } // build time check that struct implements interface var _ NvmeTransport = (*nvmeTCPTransport)(nil) // NewNvmeTCPTransport creates a new instance of nvmeTcpTransport -func NewNvmeTCPTransport(rpc spdk.JSONRPC) NvmeTransport { - if rpc == nil { - log.Panicf("rpc cannot be nil") +func NewNvmeTCPTransport(spdkClient client.IClient) NvmeTransport { + if spdkClient == nil { + log.Panicf("spdkClient cannot be nil") } return &nvmeTCPTransport{ - rpc: rpc, + rpc: spdk.NewSpdkClientAdapter(spdkClient), } } diff --git a/pkg/frontend/transport_test.go b/pkg/frontend/transport_test.go index d24cecfe..62f42c7f 100644 --- a/pkg/frontend/transport_test.go +++ b/pkg/frontend/transport_test.go @@ -9,12 +9,13 @@ import ( "reflect" "testing" - "github.com/opiproject/gospdk/spdk" + "github.com/opiproject/opi-spdk-bridge/pkg/spdk" + "github.com/spdk/spdk/go/rpc/client" ) func TestNewNvmeVfiouserTransport(t *testing.T) { tests := map[string]struct { - rpc spdk.JSONRPC + rpc client.IClient wantPanic bool }{ "nil json rpc": { @@ -22,7 +23,7 @@ func TestNewNvmeVfiouserTransport(t *testing.T) { wantPanic: true, }, "valid transport": { - rpc: spdk.NewClient("/some/path"), + rpc: spdkCLientStub{}, wantPanic: false, }, } @@ -38,7 +39,7 @@ func TestNewNvmeVfiouserTransport(t *testing.T) { gotTransport := NewNvmeTCPTransport(tt.rpc) wantTransport := &nvmeTCPTransport{ - rpc: tt.rpc, + rpc: spdk.NewSpdkClientAdapter(spdkCLientStub{}), } if !reflect.DeepEqual(gotTransport, wantTransport) { From b1666e00fa50fada9d906829ba92948cae20e9e5 Mon Sep 17 00:00:00 2001 From: Artsiom Koltun Date: Fri, 26 Jan 2024 09:34:09 +0100 Subject: [PATCH 07/11] refactor(middleend): transition to spdk client Signed-off-by: Artsiom Koltun --- cmd/main.go | 2 +- pkg/middleend/encryption.go | 2 +- pkg/middleend/middleend.go | 17 +++++++------- pkg/middleend/middleend_test.go | 8 +++---- pkg/middleend/qos.go | 2 +- pkg/middleend/qos_test.go | 41 +++++++++++++-------------------- pkg/spdk/errors.go | 17 ++++++++++++++ 7 files changed, 49 insertions(+), 40 deletions(-) create mode 100644 pkg/spdk/errors.go diff --git a/cmd/main.go b/cmd/main.go index bc299004..e15bce80 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -156,7 +156,7 @@ func runGrpcServer(grpcPort int, useKvm bool, store gokv.Store, spdkAddress, qmp }() backendServer := backend.NewServer(spdkClient, store) - middleendServer := middleend.NewServer(jsonRPC, store) + middleendServer := middleend.NewServer(spdkClient, store) if useKvm { log.Println("Creating KVM server.") diff --git a/pkg/middleend/encryption.go b/pkg/middleend/encryption.go index 22f9d48e..71a9d5db 100644 --- a/pkg/middleend/encryption.go +++ b/pkg/middleend/encryption.go @@ -14,8 +14,8 @@ import ( "sort" "github.com/google/uuid" - "github.com/opiproject/gospdk/spdk" pb "github.com/opiproject/opi-api/storage/v1alpha1/gen/go" + "github.com/opiproject/opi-spdk-bridge/pkg/spdk" "github.com/opiproject/opi-spdk-bridge/pkg/utils" "go.einride.tech/aip/fieldbehavior" diff --git a/pkg/middleend/middleend.go b/pkg/middleend/middleend.go index e46b2d81..01439ba2 100644 --- a/pkg/middleend/middleend.go +++ b/pkg/middleend/middleend.go @@ -8,9 +8,10 @@ package middleend import ( "log" + "github.com/opiproject/opi-spdk-bridge/pkg/spdk" "github.com/philippgille/gokv" + "github.com/spdk/spdk/go/rpc/client" - "github.com/opiproject/gospdk/spdk" pb "github.com/opiproject/opi-api/storage/v1alpha1/gen/go" ) @@ -25,7 +26,7 @@ type Server struct { pb.UnimplementedMiddleendEncryptionServiceServer pb.UnimplementedMiddleendQosVolumeServiceServer - rpc spdk.JSONRPC + rpc *spdk.SpdkClientAdapter store gokv.Store volumes VolumeParameters tweakMode string @@ -34,21 +35,21 @@ type Server struct { // NewServer creates initialized instance of MiddleEnd server communicating // with provided jsonRPC -func NewServer(jsonRPC spdk.JSONRPC, store gokv.Store) *Server { - return NewCustomizedServer(jsonRPC, store, spdk.TweakModeSimpleLba) +func NewServer(spdkClient client.IClient, store gokv.Store) *Server { + return NewCustomizedServer(spdkClient, store, spdk.TweakModeSimpleLba) } // NewCustomizedServer creates initialized instance of MiddleEnd server communicating // with provided jsonRPC, store and non standard tweak mode -func NewCustomizedServer(jsonRPC spdk.JSONRPC, store gokv.Store, tweakMode string) *Server { - if jsonRPC == nil { - log.Panic("nil for JSONRPC is not allowed") +func NewCustomizedServer(spdkClient client.IClient, store gokv.Store, tweakMode string) *Server { + if spdkClient == nil { + log.Panic("nil for spdkClient is not allowed") } if store == nil { log.Panic("nil for Store is not allowed") } return &Server{ - rpc: jsonRPC, + rpc: spdk.NewSpdkClientAdapter(spdkClient), store: store, volumes: VolumeParameters{ qosVolumes: make(map[string]*pb.QosVolume), diff --git a/pkg/middleend/middleend_test.go b/pkg/middleend/middleend_test.go index 5ab78731..ebc2bc08 100644 --- a/pkg/middleend/middleend_test.go +++ b/pkg/middleend/middleend_test.go @@ -16,8 +16,8 @@ import ( "google.golang.org/grpc/test/bufconn" "github.com/philippgille/gokv/gomap" + "github.com/spdk/spdk/go/rpc/client" - "github.com/opiproject/gospdk/spdk" pb "github.com/opiproject/opi-api/storage/v1alpha1/gen/go" "github.com/opiproject/opi-spdk-bridge/pkg/utils" ) @@ -42,7 +42,7 @@ type testEnv struct { testSocket string ctx context.Context conn *grpc.ClientConn - jsonRPC spdk.JSONRPC + spdkClient client.IClient } func (e *testEnv) Close() { @@ -56,11 +56,11 @@ func (e *testEnv) Close() { func createTestEnvironment(spdkResponses []string) *testEnv { env := &testEnv{} env.testSocket = utils.GenerateSocketName("middleend") - env.ln, env.jsonRPC = utils.CreateTestSpdkServer(env.testSocket, spdkResponses) + env.ln, env.spdkClient = utils.CreateTestSpdkServer(env.testSocket, spdkResponses) options := gomap.DefaultOptions options.Codec = utils.ProtoCodec{} store := gomap.NewStore(options) - env.opiSpdkServer = NewServer(env.jsonRPC, store) + env.opiSpdkServer = NewServer(env.spdkClient, store) ctx := context.Background() conn, err := grpc.DialContext(ctx, diff --git a/pkg/middleend/qos.go b/pkg/middleend/qos.go index 9f677d97..08d8c8f1 100644 --- a/pkg/middleend/qos.go +++ b/pkg/middleend/qos.go @@ -11,8 +11,8 @@ import ( "sort" "github.com/google/uuid" - "github.com/opiproject/gospdk/spdk" pb "github.com/opiproject/opi-api/storage/v1alpha1/gen/go" + "github.com/opiproject/opi-spdk-bridge/pkg/spdk" "github.com/opiproject/opi-spdk-bridge/pkg/utils" "go.einride.tech/aip/fieldbehavior" diff --git a/pkg/middleend/qos_test.go b/pkg/middleend/qos_test.go index 7bb87757..79279d00 100644 --- a/pkg/middleend/qos_test.go +++ b/pkg/middleend/qos_test.go @@ -5,14 +5,13 @@ package middleend import ( - "context" "fmt" - "net" "testing" - "github.com/opiproject/gospdk/spdk" pb "github.com/opiproject/opi-api/storage/v1alpha1/gen/go" + "github.com/opiproject/opi-spdk-bridge/pkg/spdk" "github.com/opiproject/opi-spdk-bridge/pkg/utils" + "github.com/spdk/spdk/go/rpc/client" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "google.golang.org/protobuf/proto" @@ -30,28 +29,19 @@ var ( } ) -type stubJSONRRPC struct { +type stubClient struct { params []any } // build time check that struct implements interface -var _ spdk.JSONRPC = (*stubJSONRRPC)(nil) +var _ client.IClient = (*stubClient)(nil) -func (s *stubJSONRRPC) GetID() uint64 { - return 0 -} - -func (s *stubJSONRRPC) StartUnixListener() net.Listener { - return nil -} - -func (s *stubJSONRRPC) GetVersion(_ context.Context) string { - return "" -} - -func (s *stubJSONRRPC) Call(_ context.Context, _ string, param interface{}, _ interface{}) error { - s.params = append(s.params, param) - return nil +func (s *stubClient) Call(_ string, params any) (*client.Response, error) { + s.params = append(s.params, params) + return &client.Response{ + ID: 0, + Result: true, + }, nil } func TestMiddleEnd_CreateQosVolume(t *testing.T) { @@ -327,8 +317,8 @@ func TestMiddleEnd_CreateQosVolume(t *testing.T) { t.Run("valid values are sent to SPDK", func(t *testing.T) { testEnv := createTestEnvironment([]string{}) defer testEnv.Close() - stubRPC := &stubJSONRRPC{} - testEnv.opiSpdkServer.rpc = stubRPC + stubClient := &stubClient{} + testEnv.opiSpdkServer.rpc = spdk.NewSpdkClientAdapter(stubClient) _, _ = testEnv.client.CreateQosVolume(testEnv.ctx, &pb.CreateQosVolumeRequest{ QosVolumeId: testQosVolumeID, @@ -344,10 +334,11 @@ func TestMiddleEnd_CreateQosVolume(t *testing.T) { }, }, }) - if len(stubRPC.params) != 1 { - t.Fatalf("Expect only one call to SPDK, received %v", stubRPC.params) + if len(stubClient.params) != 1 { + t.Fatalf("Expect only one call to SPDK, received %v", stubClient.params) } - qosParams := stubRPC.params[0].(*spdk.BdevQoSParams) + + qosParams := stubClient.params[0].(*spdk.BdevQoSParams) expectedParams := spdk.BdevQoSParams{ Name: "volume-42", RwIosPerSec: 1000, diff --git a/pkg/spdk/errors.go b/pkg/spdk/errors.go new file mode 100644 index 00000000..e5b2c3cd --- /dev/null +++ b/pkg/spdk/errors.go @@ -0,0 +1,17 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright (C) 2024 Intel Corporation + +// Package spdk implements the spdk json-rpc protocol +package spdk + +import ( + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +var ( + // ErrFailedSpdkCall indicates that the bridge failed to execute SPDK call + ErrFailedSpdkCall = status.Error(codes.Unknown, "Failed to execute SPDK call") + // ErrUnexpectedSpdkCallResult indicates that the bridge got an error from SPDK + ErrUnexpectedSpdkCallResult = status.Error(codes.FailedPrecondition, "Unexpected SPDK call result.") +) From 41d0d6adb756577e53b4d9ce98e591982c9e1b24 Mon Sep 17 00:00:00 2001 From: Artsiom Koltun Date: Mon, 29 Jan 2024 09:30:31 +0100 Subject: [PATCH 08/11] refactor(kvm): transition to spdk client Signed-off-by: Artsiom Koltun --- cmd/main.go | 5 +--- pkg/kvm/blk.go | 4 +++- pkg/kvm/blk_test.go | 10 ++++---- pkg/kvm/kvm_test.go | 48 ++++++++++----------------------------- pkg/kvm/nvme.go | 4 +++- pkg/kvm/nvme_test.go | 10 ++++---- pkg/kvm/transport.go | 13 ++++++----- pkg/kvm/transport_test.go | 7 +++--- 8 files changed, 40 insertions(+), 61 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index e15bce80..8ac8da0c 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -13,8 +13,6 @@ import ( "strings" "time" - "github.com/opiproject/gospdk/spdk" - "github.com/opiproject/opi-spdk-bridge/pkg/backend" "github.com/opiproject/opi-spdk-bridge/pkg/frontend" "github.com/opiproject/opi-spdk-bridge/pkg/kvm" @@ -140,7 +138,6 @@ func runGrpcServer(grpcPort int, useKvm bool, store gokv.Store, spdkAddress, qmp ) s := grpc.NewServer(serverOptions...) - jsonRPC := spdk.NewClient(spdkAddress) protocol := client.TCP if _, _, err := net.SplitHostPort(spdkAddress); err != nil { protocol = client.Unix @@ -164,7 +161,7 @@ func runGrpcServer(grpcPort int, useKvm bool, store gokv.Store, spdkAddress, qmp store, map[pb.NvmeTransportType]frontend.NvmeTransport{ pb.NvmeTransportType_NVME_TRANSPORT_TYPE_TCP: frontend.NewNvmeTCPTransport(spdkClient), - pb.NvmeTransportType_NVME_TRANSPORT_TYPE_PCIE: kvm.NewNvmeVfiouserTransport(ctrlrDir, jsonRPC), + pb.NvmeTransportType_NVME_TRANSPORT_TYPE_PCIE: kvm.NewNvmeVfiouserTransport(ctrlrDir, spdkClient), }, frontend.NewVhostUserBlkTransport(), ) diff --git a/pkg/kvm/blk.go b/pkg/kvm/blk.go index 1468a3ea..bb3fbe48 100644 --- a/pkg/kvm/blk.go +++ b/pkg/kvm/blk.go @@ -10,6 +10,8 @@ import ( "path/filepath" pb "github.com/opiproject/opi-api/storage/v1alpha1/gen/go" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" "google.golang.org/protobuf/types/known/emptypb" ) @@ -29,7 +31,7 @@ func (s *Server) CreateVirtioBlk(ctx context.Context, in *pb.CreateVirtioBlkRequ out, err := s.Server.CreateVirtioBlk(ctx, in) if err != nil { log.Println("Error running cmd on opi-spdk bridge:", err) - return out, err + return out, status.Error(codes.Unknown, err.Error()) } mon, err := newMonitor(s.qmpAddress, s.protocol, s.timeout, s.pollDevicePresenceStep) diff --git a/pkg/kvm/blk_test.go b/pkg/kvm/blk_test.go index 8f9ed95e..1b159ab6 100644 --- a/pkg/kvm/blk_test.go +++ b/pkg/kvm/blk_test.go @@ -9,8 +9,8 @@ import ( "testing" "github.com/philippgille/gokv/gomap" + "github.com/spdk/spdk/go/rpc/client" - "github.com/opiproject/gospdk/spdk" pb "github.com/opiproject/opi-api/storage/v1alpha1/gen/go" "github.com/opiproject/opi-spdk-bridge/pkg/frontend" "github.com/opiproject/opi-spdk-bridge/pkg/utils" @@ -42,7 +42,7 @@ func TestCreateVirtioBlk(t *testing.T) { t.Cleanup(checkGlobalTestProtoObjectsNotChanged(t, t.Name())) tests := map[string]struct { - jsonRPC spdk.JSONRPC + jsonRPC client.IClient errCode codes.Code errMsg string nonDefaultQmpAddress string @@ -65,8 +65,8 @@ func TestCreateVirtioBlk(t *testing.T) { "spdk failed to create virtio-blk": { in: testCreateVirtioBlkRequest, jsonRPC: alwaysFailingJSONRPC, - errCode: status.Convert(errStub).Code(), - errMsg: status.Convert(errStub).Message(), + errCode: codes.Unknown, + errMsg: "vhost_create_blk_controller: stub error", }, "qemu chardev add failed": { in: testCreateVirtioBlkRequest, @@ -207,7 +207,7 @@ func TestCreateVirtioBlk(t *testing.T) { func TestDeleteVirtioBlk(t *testing.T) { t.Cleanup(checkGlobalTestProtoObjectsNotChanged(t, t.Name())) tests := map[string]struct { - jsonRPC spdk.JSONRPC + jsonRPC client.IClient errCode codes.Code errMsg string nonDefaultQmpAddress string diff --git a/pkg/kvm/kvm_test.go b/pkg/kvm/kvm_test.go index 91b0c2fe..26810fb6 100644 --- a/pkg/kvm/kvm_test.go +++ b/pkg/kvm/kvm_test.go @@ -5,7 +5,7 @@ package kvm import ( - "context" + "errors" "fmt" "log" "net" @@ -17,18 +17,16 @@ import ( "testing" "time" - "github.com/opiproject/gospdk/spdk" + "github.com/opiproject/opi-spdk-bridge/pkg/spdk" "github.com/opiproject/opi-spdk-bridge/pkg/utils" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" + "github.com/spdk/spdk/go/rpc/client" ) const qmpID = `"id":"` var ( - errStub = status.Error(codes.Internal, "stub error") alwaysSuccessfulJSONRPC = &stubJSONRRPC{err: nil} - alwaysFailingJSONRPC = &stubJSONRRPC{err: errStub} + alwaysFailingJSONRPC = &stubJSONRRPC{err: errors.New("stub error")} genericQmpError = `{"error": {"class": "GenericError", "desc": "some error"}}` + "\n" genericQmpOk = `{"return": {}}` + "\n" @@ -53,49 +51,27 @@ type stubJSONRRPC struct { } // build time check that struct implements interface -var _ spdk.JSONRPC = (*stubJSONRRPC)(nil) +var _ client.IClient = (*stubJSONRRPC)(nil) -func (s *stubJSONRRPC) GetID() uint64 { - return 0 -} - -func (s *stubJSONRRPC) StartUnixListener() net.Listener { - return nil -} - -func (s *stubJSONRRPC) GetVersion(_ context.Context) string { - return "" -} +func (s *stubJSONRRPC) Call(method string, arg any) (*client.Response, error) { + s.arg = arg -func (s *stubJSONRRPC) Call(_ context.Context, method string, arg, result interface{}) error { + response := &client.Response{} if method == "vhost_create_blk_controller" { if s.err == nil { - resultCreateVirtioBLk, ok := result.(*spdk.VhostCreateBlkControllerResult) - if !ok { - log.Panicf("Unexpected type for virtio-blk device creation result") - } - *resultCreateVirtioBLk = spdk.VhostCreateBlkControllerResult(true) + response.Result = spdk.VhostCreateBlkControllerResult(true) } } else if method == "vhost_delete_controller" { if s.err == nil { - resultDeleteVirtioBLk, ok := result.(*spdk.VhostDeleteControllerResult) - if !ok { - log.Panicf("Unexpected type for virtio-blk device deletion result") - } - *resultDeleteVirtioBLk = spdk.VhostDeleteControllerResult(true) + response.Result = spdk.VhostDeleteControllerResult(true) } } else if method == "nvmf_subsystem_add_listener" || method == "nvmf_subsystem_remove_listener" { if s.err == nil { - resultCreateNvmeController, ok := result.(*spdk.NvmfSubsystemAddListenerResult) - if !ok { - log.Panicf("Unexpected type for add subsystem listener result") - } - *resultCreateNvmeController = spdk.NvmfSubsystemAddListenerResult(true) + response.Result = spdk.NvmfSubsystemAddListenerResult(true) } } - s.arg = arg - return s.err + return response, s.err } type mockCall struct { diff --git a/pkg/kvm/nvme.go b/pkg/kvm/nvme.go index 33237d1a..6f75ad62 100644 --- a/pkg/kvm/nvme.go +++ b/pkg/kvm/nvme.go @@ -12,6 +12,8 @@ import ( "os" "path/filepath" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" "google.golang.org/protobuf/types/known/emptypb" pb "github.com/opiproject/opi-api/storage/v1alpha1/gen/go" @@ -55,7 +57,7 @@ func (s *Server) CreateNvmeController(ctx context.Context, in *pb.CreateNvmeCont if err != nil { log.Println("Error running cmd on opi-spdk bridge:", err) _ = deleteControllerDir(s.ctrlrDir, dirName) - return out, err + return out, status.Error(codes.Unknown, err.Error()) } name := out.Name diff --git a/pkg/kvm/nvme_test.go b/pkg/kvm/nvme_test.go index 34e0dbad..05ca0480 100644 --- a/pkg/kvm/nvme_test.go +++ b/pkg/kvm/nvme_test.go @@ -14,8 +14,8 @@ import ( "testing" "github.com/philippgille/gokv/gomap" + "github.com/spdk/spdk/go/rpc/client" - "github.com/opiproject/gospdk/spdk" pb "github.com/opiproject/opi-api/storage/v1alpha1/gen/go" "github.com/opiproject/opi-spdk-bridge/pkg/frontend" "github.com/opiproject/opi-spdk-bridge/pkg/utils" @@ -71,7 +71,7 @@ func TestCreateNvmeController(t *testing.T) { t.Cleanup(checkGlobalTestProtoObjectsNotChanged(t, t.Name())) tests := map[string]struct { - jsonRPC spdk.JSONRPC + jsonRPC client.IClient nonDefaultQmpAddress string ctrlrDirExistsBeforeOperation bool ctrlrDirExistsAfterOperation bool @@ -101,8 +101,8 @@ func TestCreateNvmeController(t *testing.T) { jsonRPC: alwaysFailingJSONRPC, ctrlrDirExistsBeforeOperation: false, ctrlrDirExistsAfterOperation: false, - errCode: status.Convert(errStub).Code(), - errMsg: status.Convert(errStub).Message(), + errCode: codes.Unknown, + errMsg: "nvmf_subsystem_add_listener: stub error", }, "qemu Nvme controller add failed": { in: testCreateNvmeControllerRequest, @@ -316,7 +316,7 @@ func TestCreateNvmeController(t *testing.T) { func TestDeleteNvmeController(t *testing.T) { t.Cleanup(checkGlobalTestProtoObjectsNotChanged(t, t.Name())) tests := map[string]struct { - jsonRPC spdk.JSONRPC + jsonRPC client.IClient nonDefaultQmpAddress string ctrlrDirExistsBeforeOperation bool diff --git a/pkg/kvm/transport.go b/pkg/kvm/transport.go index 2dabf925..6c48c27a 100644 --- a/pkg/kvm/transport.go +++ b/pkg/kvm/transport.go @@ -10,24 +10,25 @@ import ( "log" "os" - "github.com/opiproject/gospdk/spdk" pb "github.com/opiproject/opi-api/storage/v1alpha1/gen/go" "github.com/opiproject/opi-spdk-bridge/pkg/frontend" + "github.com/opiproject/opi-spdk-bridge/pkg/spdk" "github.com/opiproject/opi-spdk-bridge/pkg/utils" + "github.com/spdk/spdk/go/rpc/client" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) type nvmeVfiouserTransport struct { ctrlrDir string - rpc spdk.JSONRPC + rpc *spdk.SpdkClientAdapter } // build time check that struct implements interface var _ frontend.NvmeTransport = (*nvmeVfiouserTransport)(nil) // NewNvmeVfiouserTransport creates a new instance of nvmeVfiouserTransport -func NewNvmeVfiouserTransport(ctrlrDir string, rpc spdk.JSONRPC) frontend.NvmeTransport { +func NewNvmeVfiouserTransport(ctrlrDir string, spdkClient client.IClient) frontend.NvmeTransport { if ctrlrDir == "" { log.Panicf("ctrlrDir cannot be empty") } @@ -40,13 +41,13 @@ func NewNvmeVfiouserTransport(ctrlrDir string, rpc spdk.JSONRPC) frontend.NvmeTr log.Panicf("%v is not a directory", ctrlrDir) } - if rpc == nil { - log.Panicf("rpc cannot be nil") + if spdkClient == nil { + log.Panicf("spdkClient cannot be nil") } return &nvmeVfiouserTransport{ ctrlrDir: ctrlrDir, - rpc: rpc, + rpc: spdk.NewSpdkClientAdapter(spdkClient), } } diff --git a/pkg/kvm/transport_test.go b/pkg/kvm/transport_test.go index c068bdb5..9ef9b789 100644 --- a/pkg/kvm/transport_test.go +++ b/pkg/kvm/transport_test.go @@ -11,16 +11,17 @@ import ( "reflect" "testing" - "github.com/opiproject/gospdk/spdk" pb "github.com/opiproject/opi-api/storage/v1alpha1/gen/go" + "github.com/opiproject/opi-spdk-bridge/pkg/spdk" "github.com/opiproject/opi-spdk-bridge/pkg/utils" + "github.com/spdk/spdk/go/rpc/client" "google.golang.org/protobuf/types/known/wrapperspb" ) func TestNewNvmeVfiouserTransport(t *testing.T) { tests := map[string]struct { ctrlrDir string - rpc spdk.JSONRPC + rpc client.IClient wantPanic bool }{ "valid controller dir": { @@ -62,7 +63,7 @@ func TestNewNvmeVfiouserTransport(t *testing.T) { gotTransport := NewNvmeVfiouserTransport(tt.ctrlrDir, tt.rpc) wantTransport := &nvmeVfiouserTransport{ ctrlrDir: tt.ctrlrDir, - rpc: tt.rpc, + rpc: spdk.NewSpdkClientAdapter(tt.rpc), } if !reflect.DeepEqual(gotTransport, wantTransport) { From d39270f44f66d929e3958204bbc852a90c278822 Mon Sep 17 00:00:00 2001 From: Artsiom Koltun Date: Tue, 30 Jan 2024 09:33:05 +0100 Subject: [PATCH 09/11] refactor(spdk): rename SpdkClientAdapter to ClientAdapter type name will be used as spdk.SpdkClientAdapter by other packages, and that stutters; consider calling this ClientAdapter Signed-off-by: Artsiom Koltun --- pkg/backend/backend.go | 2 +- pkg/frontend/frontend.go | 2 +- pkg/frontend/transport.go | 2 +- pkg/kvm/transport.go | 2 +- pkg/middleend/middleend.go | 2 +- pkg/spdk/adapter.go | 12 ++++++------ 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/pkg/backend/backend.go b/pkg/backend/backend.go index 73806f54..4e50ffbb 100644 --- a/pkg/backend/backend.go +++ b/pkg/backend/backend.go @@ -34,7 +34,7 @@ type Server struct { pb.UnimplementedNullVolumeServiceServer pb.UnimplementedAioVolumeServiceServer - rpc *spdk.SpdkClientAdapter + rpc *spdk.ClientAdapter store gokv.Store Volumes VolumeParameters Pagination map[string]int diff --git a/pkg/frontend/frontend.go b/pkg/frontend/frontend.go index f3a01512..9d8d0104 100644 --- a/pkg/frontend/frontend.go +++ b/pkg/frontend/frontend.go @@ -38,7 +38,7 @@ type Server struct { pb.UnimplementedFrontendVirtioBlkServiceServer pb.UnimplementedFrontendVirtioScsiServiceServer - rpc *spdk.SpdkClientAdapter + rpc *spdk.ClientAdapter store gokv.Store Nvme NvmeParameters Virt VirtioParameters diff --git a/pkg/frontend/transport.go b/pkg/frontend/transport.go index cb24eda8..231e1cb2 100644 --- a/pkg/frontend/transport.go +++ b/pkg/frontend/transport.go @@ -35,7 +35,7 @@ type VirtioBlkTransport interface { } type nvmeTCPTransport struct { - rpc *spdk.SpdkClientAdapter + rpc *spdk.ClientAdapter } // build time check that struct implements interface diff --git a/pkg/kvm/transport.go b/pkg/kvm/transport.go index 6c48c27a..01ba5580 100644 --- a/pkg/kvm/transport.go +++ b/pkg/kvm/transport.go @@ -21,7 +21,7 @@ import ( type nvmeVfiouserTransport struct { ctrlrDir string - rpc *spdk.SpdkClientAdapter + rpc *spdk.ClientAdapter } // build time check that struct implements interface diff --git a/pkg/middleend/middleend.go b/pkg/middleend/middleend.go index 01439ba2..44b97c7d 100644 --- a/pkg/middleend/middleend.go +++ b/pkg/middleend/middleend.go @@ -26,7 +26,7 @@ type Server struct { pb.UnimplementedMiddleendEncryptionServiceServer pb.UnimplementedMiddleendQosVolumeServiceServer - rpc *spdk.SpdkClientAdapter + rpc *spdk.ClientAdapter store gokv.Store volumes VolumeParameters tweakMode string diff --git a/pkg/spdk/adapter.go b/pkg/spdk/adapter.go index b237da26..de7baa56 100644 --- a/pkg/spdk/adapter.go +++ b/pkg/spdk/adapter.go @@ -14,24 +14,24 @@ import ( "github.com/spdk/spdk/go/rpc/client" ) -// SpdkClientAdapter provides an adapter between old gospdk api and new spdk client +// ClientAdapter provides an adapter between old gospdk api and new spdk client // This is needed due to: // * awkward response unmarshaling in the new client. A Result in Response is // provided in the form described in json.Unmarshal // * New error codes affecting a lot of tests. The adapter converts the new // format to the old one. It enables gradual transition. The tests will be // reworked to new errors eliminating transformations in the adapter. -type SpdkClientAdapter struct { +type ClientAdapter struct { client client.IClient } // NewSpdkClientAdapter creates a new instance if SpdkClientAdapter -func NewSpdkClientAdapter(client client.IClient) *SpdkClientAdapter { - return &SpdkClientAdapter{client} +func NewSpdkClientAdapter(client client.IClient) *ClientAdapter { + return &ClientAdapter{client} } // Call performs a call to spdk client and unmarshalls the result into requested structure -func (c *SpdkClientAdapter) Call(ctx context.Context, method string, params any, result any) error { +func (c *ClientAdapter) Call(ctx context.Context, method string, params any, result any) error { ch := make(chan error) go func() { @@ -58,7 +58,7 @@ func (c *SpdkClientAdapter) Call(ctx context.Context, method string, params any, } } -func (c *SpdkClientAdapter) convertToLegacyErrorFormat(err error, method string) error { +func (c *ClientAdapter) convertToLegacyErrorFormat(err error, method string) error { e := errors.Unwrap(err) spdkErr, _ := e.(*client.Error) From cf34d3d1e84ec53f11f44a8b1fac8633d9825816 Mon Sep 17 00:00:00 2001 From: Artsiom Koltun Date: Tue, 30 Jan 2024 09:26:31 +0100 Subject: [PATCH 10/11] chore(deps): remove github.com/opiproject/gospdk Signed-off-by: Artsiom Koltun --- go.mod | 1 - go.sum | 4 ---- 2 files changed, 5 deletions(-) diff --git a/go.mod b/go.mod index 2f666039..5c586cec 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,6 @@ require ( github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.1 github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.0 github.com/onsi/ginkgo/v2 v2.14.0 - github.com/opiproject/gospdk v0.0.0-20240108075015-92d689fff706 github.com/opiproject/opi-api v0.0.0-20240118183513-e44db269fba4 github.com/philippgille/gokv v0.6.0 github.com/philippgille/gokv/gomap v0.6.0 diff --git a/go.sum b/go.sum index 1081feff..55264f6e 100644 --- a/go.sum +++ b/go.sum @@ -84,7 +84,6 @@ github.com/alexkohler/prealloc v1.0.0 h1:Hbq0/3fJPQhNkN0dR95AVrr6R7tou91y0uHG5pO github.com/alexkohler/prealloc v1.0.0/go.mod h1:VetnK3dIgFBBKmg0YnD9F9x6Icjd+9cvfHR56wJVlKE= github.com/alingse/asasalint v0.0.11 h1:SFwnQXJ49Kx/1GghOFz1XGqHYKp21Kq1nHad/0WQRnw= github.com/alingse/asasalint v0.0.11/go.mod h1:nCaoMhw7a9kSJObvQyVzNTPBDbNpdocqrSP7t/cW5+I= -github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/ashanbrown/forbidigo v1.6.0 h1:D3aewfM37Yb3pxHujIPSpTf6oQk9sc9WZi8gerOIVIY= github.com/ashanbrown/forbidigo v1.6.0/go.mod h1:Y8j9jy9ZYAEHXdu723cUlraTqbzjKF1MUyfOKL+AjcU= github.com/ashanbrown/makezero v1.1.1 h1:iCQ87C0V0vSyO+M9E/FZYbu65auqH0lnsOkf5FcB28s= @@ -463,8 +462,6 @@ github.com/onsi/ginkgo/v2 v2.14.0/go.mod h1:JkUdW7JkN0V6rFvsHcJ478egV3XH9NxpD27H github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.30.0 h1:hvMK7xYz4D3HapigLTeGdId/NcfQx1VHMJc60ew99+8= github.com/onsi/gomega v1.30.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ= -github.com/opiproject/gospdk v0.0.0-20240108075015-92d689fff706 h1:fZcUuXgfsfWIbSZTPCJG72wh7LyKIyrpPR9kH+rKEXI= -github.com/opiproject/gospdk v0.0.0-20240108075015-92d689fff706/go.mod h1:9CMbTd9ptR6tl6HRRn8C33DPeWF85hTo4KZCa5iKftY= github.com/opiproject/opi-api v0.0.0-20240118183513-e44db269fba4 h1:YBjvYWQQAbNIGsAXvB6FwL9Encr1nzo3/w+bB/tXltM= github.com/opiproject/opi-api v0.0.0-20240118183513-e44db269fba4/go.mod h1:92pv4ulvvPMuxCJ9ND3aYbmBfEMLx0VCjpkiR7ZTqPY= github.com/otiai10/copy v1.2.0/go.mod h1:rrF5dJ5F0t/EWSYODDu4j9/vEeYHMkc8jt0zJChqQWw= @@ -528,7 +525,6 @@ github.com/quasilyte/regex/syntax v0.0.0-20210819130434-b3f0c404a727 h1:TCg2WBOl github.com/quasilyte/regex/syntax v0.0.0-20210819130434-b3f0c404a727/go.mod h1:rlzQ04UMyJXu/aOvhd8qT+hvDrFpiwqp8MRXDY9szc0= github.com/quasilyte/stdinfo v0.0.0-20220114132959-f7386bf02567 h1:M8mH9eK4OUR4lu7Gd+PU1fV2/qnDNfzT635KRSObncs= github.com/quasilyte/stdinfo v0.0.0-20220114132959-f7386bf02567/go.mod h1:DWNGW8A4Y+GyBgPuaQJuWiy0XYftx4Xm/y5Jqk9I6VQ= -github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= From 9d0d95025c26e96b67e613a38452ea7e2e95976a Mon Sep 17 00:00:00 2001 From: Artsiom Koltun Date: Wed, 31 Jan 2024 09:05:44 +0100 Subject: [PATCH 11/11] refactor: (spdk)add trace to adapter Signed-off-by: Artsiom Koltun --- pkg/spdk/adapter.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/pkg/spdk/adapter.go b/pkg/spdk/adapter.go index de7baa56..346d685d 100644 --- a/pkg/spdk/adapter.go +++ b/pkg/spdk/adapter.go @@ -12,6 +12,9 @@ import ( "strings" "github.com/spdk/spdk/go/rpc/client" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/trace" ) // ClientAdapter provides an adapter between old gospdk api and new spdk client @@ -23,15 +26,22 @@ import ( // reworked to new errors eliminating transformations in the adapter. type ClientAdapter struct { client client.IClient + tracer trace.Tracer } // NewSpdkClientAdapter creates a new instance if SpdkClientAdapter func NewSpdkClientAdapter(client client.IClient) *ClientAdapter { - return &ClientAdapter{client} + return &ClientAdapter{ + client: client, + tracer: otel.Tracer(""), + } } // Call performs a call to spdk client and unmarshalls the result into requested structure func (c *ClientAdapter) Call(ctx context.Context, method string, params any, result any) error { + _, childSpan := c.tracer.Start(ctx, "spdk."+method) + defer childSpan.End() + ch := make(chan error) go func() {