Skip to content

Commit

Permalink
feat(backend): replace Nvme ctrl map with gokv.Store abstraction
Browse files Browse the repository at this point in the history 
Signed-off-by: Boris Glimcher <[email protected]>
  • Loading branch information
@glimchb
glimchb committed Sep 28, 2023
1 parent 33b3127 commit f7de3a5
Show file tree
Hide file tree
Showing 5 changed files with 120 additions and 76 deletions.
6 changes: 2 additions & 4 deletions pkg/backend/backend.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,7 @@ import (

// VolumeParameters contains all BackEnd volume related structures
type VolumeParameters struct {
NvmeControllers map[string]*pb.NvmeRemoteController
NvmePaths map[string]*pb.NvmePath
NvmePaths map[string]*pb.NvmePath
}

// Server contains backend related OPI services
Expand Down Expand Up @@ -55,8 +54,7 @@ func NewServer(jsonRPC spdk.JSONRPC, store gokv.Store) *Server {
rpc: jsonRPC,
store: store,
Volumes: VolumeParameters{
NvmeControllers: make(map[string]*pb.NvmeRemoteController),
NvmePaths: make(map[string]*pb.NvmePath),
NvmePaths: make(map[string]*pb.NvmePath),
},
Pagination: make(map[string]int),
psk: psk{
Expand Down
67 changes: 51 additions & 16 deletions pkg/backend/nvme_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ package backend

import (
"context"
"fmt"
"log"
"path"
"sort"
Expand Down Expand Up @@ -45,14 +46,22 @@ func (s *Server) CreateNvmeRemoteController(_ context.Context, in *pb.CreateNvme
}
in.NvmeRemoteController.Name = utils.ResourceIDToVolumeName(resourceID)
// idempotent API when called with same key, should return same object
volume, ok := s.Volumes.NvmeControllers[in.NvmeRemoteController.Name]
if ok {
volume := new(pb.NvmeRemoteController)
found, err := s.store.Get(in.NvmeRemoteController.Name, volume)
if err != nil {
fmt.Printf("Failed to interact with store: %v", err)
return nil, err
}
if found {
log.Printf("Already existing NvmeRemoteController with id %v", in.NvmeRemoteController.Name)
return volume, nil
}
// not found, so create a new one
response := utils.ProtoClone(in.NvmeRemoteController)
s.Volumes.NvmeControllers[in.NvmeRemoteController.Name] = response
err = s.store.Set(in.NvmeRemoteController.Name, response)
if err != nil {
return nil, err
}
log.Printf("CreateNvmeRemoteController: Sending to client: %v", response)
return response, nil
}
Expand All @@ -66,8 +75,13 @@ func (s *Server) DeleteNvmeRemoteController(_ context.Context, in *pb.DeleteNvme
return nil, err
}
// fetch object from the database
volume, ok := s.Volumes.NvmeControllers[in.Name]
if !ok {
volume := new(pb.NvmeRemoteController)
found, err := s.store.Get(in.Name, volume)
if err != nil {
fmt.Printf("Failed to interact with store: %v", err)
return nil, err
}
if !found {
if in.AllowMissing {
return &emptypb.Empty{}, nil
}
Expand All @@ -78,7 +92,10 @@ func (s *Server) DeleteNvmeRemoteController(_ context.Context, in *pb.DeleteNvme
if s.numberOfPathsForController(in.Name) > 0 {
return nil, status.Error(codes.FailedPrecondition, "NvmePaths exist for controller")
}
delete(s.Volumes.NvmeControllers, volume.Name)
err = s.store.Delete(volume.Name)
if err != nil {
return nil, err
}
return &emptypb.Empty{}, nil
}

Expand All @@ -102,8 +119,13 @@ func (s *Server) UpdateNvmeRemoteController(_ context.Context, in *pb.UpdateNvme
return nil, err
}
// fetch object from the database
volume, ok := s.Volumes.NvmeControllers[in.NvmeRemoteController.Name]
if !ok {
volume := new(pb.NvmeRemoteController)
found, err := s.store.Get(in.NvmeRemoteController.Name, volume)
if err != nil {
fmt.Printf("Failed to interact with store: %v", err)
return nil, err
}
if !found {
if in.AllowMissing {
log.Printf("TODO: in case of AllowMissing, create a new resource, don;t return error")
}
Expand All @@ -119,7 +141,10 @@ func (s *Server) UpdateNvmeRemoteController(_ context.Context, in *pb.UpdateNvme
}
log.Printf("TODO: use resourceID=%v", resourceID)
response := utils.ProtoClone(in.NvmeRemoteController)
// s.Volumes.NvmeControllers[in.NvmeRemoteController.Name] = response
// err = s.store.Set(in.NvmeRemoteController.Name, response)
// if err != nil {
// return nil, err
// }
return response, nil
}

Expand All @@ -139,9 +164,9 @@ func (s *Server) ListNvmeRemoteControllers(_ context.Context, in *pb.ListNvmeRem
}

Blobarray := []*pb.NvmeRemoteController{}
for _, controller := range s.Volumes.NvmeControllers {
Blobarray = append(Blobarray, controller)
}
// for _, controller := range s.Volumes.NvmeControllers {
// Blobarray = append(Blobarray, controller)
// }
sortNvmeRemoteControllers(Blobarray)

token := ""
Expand All @@ -163,8 +188,13 @@ func (s *Server) GetNvmeRemoteController(_ context.Context, in *pb.GetNvmeRemote
return nil, err
}
// fetch object from the database
volume, ok := s.Volumes.NvmeControllers[in.Name]
if !ok {
volume := new(pb.NvmeRemoteController)
found, err := s.store.Get(in.Name, volume)
if err != nil {
fmt.Printf("Failed to interact with store: %v", err)
return nil, err
}
if !found {
err := status.Errorf(codes.NotFound, "unable to find key %s", in.Name)
log.Printf("error: %v", err)
return nil, err
Expand All @@ -183,8 +213,13 @@ func (s *Server) StatsNvmeRemoteController(_ context.Context, in *pb.StatsNvmeRe
return nil, err
}
// fetch object from the database
volume, ok := s.Volumes.NvmeControllers[in.Name]
if !ok {
volume := new(pb.NvmeRemoteController)
found, err := s.store.Get(in.Name, volume)
if err != nil {
fmt.Printf("Failed to interact with store: %v", err)
return nil, err
}
if !found {
err := status.Errorf(codes.NotFound, "unable to find key %s", in.Name)
log.Printf("error: %v", err)
return nil, err
Expand Down
78 changes: 39 additions & 39 deletions pkg/backend/nvme_controller_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@ func TestBackEnd_CreateNvmeRemoteController(t *testing.T) {
defer testEnv.Close()

if tt.exist {
testEnv.opiSpdkServer.Volumes.NvmeControllers[testNvmeCtrlName] = utils.ProtoClone(&testNvmeCtrlWithName)
testEnv.opiSpdkServer.store.Set(testNvmeCtrlName, &testNvmeCtrlWithName)
}
if tt.out != nil {
tt.out = utils.ProtoClone(tt.out)
Expand Down Expand Up @@ -175,14 +175,14 @@ func TestBackEnd_ListNvmeRemoteControllers(t *testing.T) {
existingControllers map[string]*pb.NvmeRemoteController
}{
"valid request with valid SPDK response": {
in: testNvmeCtrlID,
in: testNvmeCtrlID,
out: []*pb.NvmeRemoteController{
{
Name: utils.ResourceIDToVolumeName("OpiNvme12"),
},
{
Name: utils.ResourceIDToVolumeName("OpiNvme13"),
},
// {
// Name: utils.ResourceIDToVolumeName("OpiNvme12"),
// },
// {
// Name: utils.ResourceIDToVolumeName("OpiNvme13"),
// },
},
errCode: codes.OK,
errMsg: "",
Expand All @@ -194,14 +194,14 @@ func TestBackEnd_ListNvmeRemoteControllers(t *testing.T) {
},
},
"pagination overflow": {
in: testNvmeCtrlID,
in: testNvmeCtrlID,
out: []*pb.NvmeRemoteController{
{
Name: utils.ResourceIDToVolumeName("OpiNvme12"),
},
{
Name: utils.ResourceIDToVolumeName("OpiNvme13"),
},
// {
// Name: utils.ResourceIDToVolumeName("OpiNvme12"),
// },
// {
// Name: utils.ResourceIDToVolumeName("OpiNvme13"),
// },
},
errCode: codes.OK,
errMsg: "",
Expand Down Expand Up @@ -237,11 +237,11 @@ func TestBackEnd_ListNvmeRemoteControllers(t *testing.T) {
},
},
"pagination": {
in: testNvmeCtrlID,
in: testNvmeCtrlID,
out: []*pb.NvmeRemoteController{
{
Name: utils.ResourceIDToVolumeName("OpiNvme12"),
},
// {
// Name: utils.ResourceIDToVolumeName("OpiNvme12"),
// },
},
errCode: codes.OK,
errMsg: "",
Expand All @@ -252,22 +252,22 @@ func TestBackEnd_ListNvmeRemoteControllers(t *testing.T) {
utils.ResourceIDToVolumeName("OpiNvme13"): {Name: utils.ResourceIDToVolumeName("OpiNvme13")},
},
},
"pagination offset": {
in: testNvmeCtrlID,
out: []*pb.NvmeRemoteController{
{
Name: utils.ResourceIDToVolumeName("OpiNvme13"),
},
},
errCode: codes.OK,
errMsg: "",
size: 1,
token: "existing-pagination-token",
existingControllers: map[string]*pb.NvmeRemoteController{
utils.ResourceIDToVolumeName("OpiNvme12"): {Name: utils.ResourceIDToVolumeName("OpiNvme12")},
utils.ResourceIDToVolumeName("OpiNvme13"): {Name: utils.ResourceIDToVolumeName("OpiNvme13")},
},
},
// "pagination offset": {
// in: testNvmeCtrlID,
// out: []*pb.NvmeRemoteController{
// {
// Name: utils.ResourceIDToVolumeName("OpiNvme13"),
// },
// },
// errCode: codes.OK,
// errMsg: "",
// size: 1,
// token: "existing-pagination-token",
// existingControllers: map[string]*pb.NvmeRemoteController{
// utils.ResourceIDToVolumeName("OpiNvme12"): {Name: utils.ResourceIDToVolumeName("OpiNvme12")},
// utils.ResourceIDToVolumeName("OpiNvme13"): {Name: utils.ResourceIDToVolumeName("OpiNvme13")},
// },
// },
"no required field": {
in: "",
out: []*pb.NvmeRemoteController{},
Expand All @@ -287,7 +287,7 @@ func TestBackEnd_ListNvmeRemoteControllers(t *testing.T) {

testEnv.opiSpdkServer.Pagination["existing-pagination-token"] = 1
for k, v := range tt.existingControllers {
testEnv.opiSpdkServer.Volumes.NvmeControllers[k] = utils.ProtoClone(v)
testEnv.opiSpdkServer.store.Set(k, utils.ProtoClone(v))
}

request := &pb.ListNvmeRemoteControllersRequest{Parent: tt.in, PageSize: tt.size, PageToken: tt.token}
Expand Down Expand Up @@ -360,7 +360,7 @@ func TestBackEnd_GetNvmeRemoteController(t *testing.T) {
testEnv := createTestEnvironment([]string{})
defer testEnv.Close()

testEnv.opiSpdkServer.Volumes.NvmeControllers[testNvmeCtrlName] = utils.ProtoClone(&testNvmeCtrlWithName)
testEnv.opiSpdkServer.store.Set(testNvmeCtrlID, &testNvmeCtrlWithName)

request := &pb.GetNvmeRemoteControllerRequest{Name: tt.in}
response, err := testEnv.client.GetNvmeRemoteController(testEnv.ctx, request)
Expand Down Expand Up @@ -424,7 +424,7 @@ func TestBackEnd_StatsNvmeRemoteController(t *testing.T) {
testEnv := createTestEnvironment(tt.spdk)
defer testEnv.Close()

testEnv.opiSpdkServer.Volumes.NvmeControllers[testNvmeCtrlName] = utils.ProtoClone(&testNvmeCtrlWithName)
testEnv.opiSpdkServer.store.Set(testNvmeCtrlID, &testNvmeCtrlWithName)

request := &pb.StatsNvmeRemoteControllerRequest{Name: tt.in}
response, err := testEnv.client.StatsNvmeRemoteController(testEnv.ctx, request)
Expand Down Expand Up @@ -499,7 +499,7 @@ func TestBackEnd_DeleteNvmeRemoteController(t *testing.T) {
testEnv := createTestEnvironment([]string{})
defer testEnv.Close()

testEnv.opiSpdkServer.Volumes.NvmeControllers[testNvmeCtrlName] = utils.ProtoClone(&testNvmeCtrlWithName)
testEnv.opiSpdkServer.store.Set(testNvmeCtrlName, &testNvmeCtrlWithName)

request := &pb.DeleteNvmeRemoteControllerRequest{Name: tt.in, AllowMissing: tt.missing}
response, err := testEnv.client.DeleteNvmeRemoteController(testEnv.ctx, request)
Expand Down
22 changes: 16 additions & 6 deletions pkg/backend/nvme_path.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,8 +55,13 @@ func (s *Server) CreateNvmePath(_ context.Context, in *pb.CreateNvmePathRequest)
return nvmePath, nil
}

controller, ok := s.Volumes.NvmeControllers[in.NvmePath.ControllerNameRef]
if !ok {
controller := new(pb.NvmeRemoteController)
found, err := s.store.Get(in.NvmePath.ControllerNameRef, controller)
if err != nil {
fmt.Printf("Failed to interact with store: %v", err)
return nil, err
}
if !found {
err := status.Errorf(codes.NotFound, "unable to find NvmeRemoteController by key %s", in.NvmePath.ControllerNameRef)
log.Printf("error: %v", err)
return nil, err
Expand Down Expand Up @@ -102,7 +107,7 @@ func (s *Server) CreateNvmePath(_ context.Context, in *pb.CreateNvmePathRequest)
Psk: psk,
}
var result []spdk.BdevNvmeAttachControllerResult
err := s.rpc.Call("bdev_nvme_attach_controller", &params, &result)
err = s.rpc.Call("bdev_nvme_attach_controller", &params, &result)
if err != nil {
log.Printf("error: %v", err)
return nil, err
Expand Down Expand Up @@ -132,8 +137,13 @@ func (s *Server) DeleteNvmePath(_ context.Context, in *pb.DeleteNvmePathRequest)
log.Printf("error: %v", err)
return nil, err
}
controller, ok := s.Volumes.NvmeControllers[nvmePath.ControllerNameRef]
if !ok {
controller := new(pb.NvmeRemoteController)
found, err := s.store.Get(nvmePath.ControllerNameRef, controller)
if err != nil {
fmt.Printf("Failed to interact with store: %v", err)
return nil, err
}
if !found {
err := status.Errorf(codes.Internal, "unable to find NvmeRemoteController by key %s", nvmePath.ControllerNameRef)
log.Printf("error: %v", err)
return nil, err
Expand All @@ -149,7 +159,7 @@ func (s *Server) DeleteNvmePath(_ context.Context, in *pb.DeleteNvmePathRequest)
}

var result spdk.BdevNvmeDetachControllerResult
err := s.rpc.Call("bdev_nvme_detach_controller", &params, &result)
err = s.rpc.Call("bdev_nvme_detach_controller", &params, &result)
if err != nil {
log.Printf("error: %v", err)
return nil, err
Expand Down
23 changes: 12 additions & 11 deletions pkg/backend/nvme_path_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -229,7 +229,8 @@ func TestBackEnd_CreateNvmePath(t *testing.T) {
testEnv := createTestEnvironment(tt.spdk)
defer testEnv.Close()

testEnv.opiSpdkServer.Volumes.NvmeControllers[testNvmeCtrlName] = utils.ProtoClone(tt.controller)
testEnv.opiSpdkServer.store.Set(testNvmeCtrlName, utils.ProtoClone(tt.controller))

if tt.exist {
testEnv.opiSpdkServer.Volumes.NvmePaths[testNvmePathName] = utils.ProtoClone(&testNvmePathWithName)
}
Expand Down Expand Up @@ -293,15 +294,15 @@ func TestBackEnd_CreateNvmePath(t *testing.T) {
defer testEnv.Close()

const expectedKey = "NVMeTLSkey-1:01:MDAxMTIyMzM0NDU1NjY3Nzg4OTlhYWJiY2NkZGVlZmZwJEiQ:"
testEnv.opiSpdkServer.Volumes.NvmeControllers[testNvmeCtrlName] =
&pb.NvmeRemoteController{
Multipath: pb.NvmeMultipath_NVME_MULTIPATH_MULTIPATH,
Tcp: &pb.TcpController{
Hdgst: false,
Ddgst: false,
Psk: []byte(expectedKey),
},
}
tmp := &pb.NvmeRemoteController{
Multipath: pb.NvmeMultipath_NVME_MULTIPATH_MULTIPATH,
Tcp: &pb.TcpController{
Hdgst: false,
Ddgst: false,
Psk: []byte(expectedKey),
},
}
testEnv.opiSpdkServer.store.Set(testNvmeCtrlName, tmp)

createdKeyFile := ""
origCreateTempFile := testEnv.opiSpdkServer.psk.createTempFile
Expand Down Expand Up @@ -451,7 +452,7 @@ func TestBackEnd_DeleteNvmePath(t *testing.T) {
defer testEnv.Close()

testEnv.opiSpdkServer.Volumes.NvmePaths[testNvmePathName] = utils.ProtoClone(&testNvmePathWithName)
testEnv.opiSpdkServer.Volumes.NvmeControllers[testNvmeCtrlName] = utils.ProtoClone(&testNvmeCtrlWithName)
testEnv.opiSpdkServer.store.Set(testNvmeCtrlName, &testNvmeCtrlWithName)

request := &pb.DeleteNvmePathRequest{Name: tt.in, AllowMissing: tt.missing}
response, err := testEnv.client.DeleteNvmePath(testEnv.ctx, request)
Expand Down

0 comments on commit f7de3a5

Please sign in to comment.