diff --git a/charts/aws-ebs-csi-driver/templates/clusterrole-csi-node.yaml b/charts/aws-ebs-csi-driver/templates/clusterrole-csi-node.yaml index 3e5382216c..a48153bb37 100644 --- a/charts/aws-ebs-csi-driver/templates/clusterrole-csi-node.yaml +++ b/charts/aws-ebs-csi-driver/templates/clusterrole-csi-node.yaml @@ -9,3 +9,6 @@ rules: - apiGroups: [""] resources: ["nodes"] verbs: ["get", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["list"] diff --git a/charts/aws-ebs-csi-driver/templates/node.yaml b/charts/aws-ebs-csi-driver/templates/node.yaml index 511d31a557..ac6118dceb 100644 --- a/charts/aws-ebs-csi-driver/templates/node.yaml +++ b/charts/aws-ebs-csi-driver/templates/node.yaml @@ -121,6 +121,10 @@ spec: securityContext: {{- toYaml . | nindent 12 }} {{- end }} + lifecycle: + preStop: + exec: + command: ["/bin/aws-ebs-csi-driver", "pre-stop-hook"] - name: node-driver-registrar image: {{ printf "%s%s:%s" (default "" .Values.image.containerRegistry) .Values.sidecars.nodeDriverRegistrar.image.repository .Values.sidecars.nodeDriverRegistrar.image.tag }} imagePullPolicy: {{ default .Values.image.pullPolicy .Values.sidecars.nodeDriverRegistrar.image.pullPolicy }} diff --git a/cmd/hooks/prestop.go b/cmd/hooks/prestop.go new file mode 100644 index 0000000000..ae44f65019 --- /dev/null +++ b/cmd/hooks/prestop.go @@ -0,0 +1,113 @@ +package hooks + +import ( + "context" + "fmt" + "os" + "time" + + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/cache" + "k8s.io/klog/v2" +) + +func PreStop(clientset kubernetes.Interface, timeout time.Duration) error { + klog.InfoS("PreStop: executing PreStop lifecycle hook") + + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + nodeName := os.Getenv("CSI_NODE_NAME") + if nodeName == "" { + return fmt.Errorf("PreStop: CSI_NODE_NAME missing") + } + + node, err := fetchNodeInfo(ctx, clientset, nodeName) + if err != nil { + return err + } + + if isNodeBeingDrained(node) { + klog.InfoS("PreStop: node is being drained, checking for remaining VolumeAttachments", "node", nodeName) + return waitForVolumeAttachments(ctx, clientset, nodeName) + } + + klog.InfoS("PreStop: node is not being drained, skipping VolumeAttachments check", "node", nodeName) + return nil +} + +func fetchNodeInfo(ctx context.Context, clientset kubernetes.Interface, nodeName string) (*v1.Node, error) { + node, err := clientset.CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("fetchNodeInfo: failed to retrieve node information: %w", err) + } + return node, nil +} + +func isNodeBeingDrained(node *v1.Node) bool { + for _, taint := range node.Spec.Taints { + if taint.Key == v1.TaintNodeUnschedulable && taint.Effect == v1.TaintEffectNoSchedule { + return true + } + } + return false +} + +func waitForVolumeAttachments(ctx context.Context, clientset kubernetes.Interface, nodeName string) error { + stopCh := make(chan struct{}) + defer close(stopCh) + + allAttachmentsDeleted := make(chan struct{}) + + factory := informers.NewSharedInformerFactory(clientset, 0) + informer := factory.Storage().V1().VolumeAttachments().Informer() + + _, err := informer.AddEventHandler(cache.ResourceEventHandlerFuncs{ + DeleteFunc: func(obj interface{}) { + klog.V(4).InfoS("DeleteFunc: VolumeAttachment deleted", "node", nodeName) + if err := checkVolumeAttachments(ctx, clientset, nodeName, allAttachmentsDeleted); err != nil { + klog.ErrorS(err, "DeleteFunc: error checking VolumeAttachments") + return + } + }, + }) + + if err != nil { + return fmt.Errorf("failed to add event handler to VolumeAttachment informer: %w", err) + } + + go informer.Run(stopCh) + + if err := checkVolumeAttachments(ctx, clientset, nodeName, allAttachmentsDeleted); err != nil { + return err + } + + select { + case <-allAttachmentsDeleted: + klog.InfoS("waitForVolumeAttachments: finished waiting for VolumeAttachments to be deleted. preStopHook completed") + return nil + + case <-ctx.Done(): + return fmt.Errorf("waitForVolumeAttachments: timed out waiting for preStopHook to complete: %w", ctx.Err()) + } +} + +func checkVolumeAttachments(ctx context.Context, clientset kubernetes.Interface, nodeName string, allAttachmentsDeleted chan struct{}) error { + allAttachments, err := clientset.StorageV1().VolumeAttachments().List(ctx, metav1.ListOptions{}) + if err != nil { + return fmt.Errorf("checkVolumeAttachments: failed to list VolumeAttachments: %w", err) + } + + for _, attachment := range allAttachments.Items { + if attachment.Spec.NodeName == nodeName { + klog.InfoS("isVolumeAttachmentEmpty: found VolumeAttachment", "attachment", attachment, "node", nodeName) + return nil + } + } + + close(allAttachmentsDeleted) + return nil +} diff --git a/cmd/hooks/prestop_test.go b/cmd/hooks/prestop_test.go new file mode 100644 index 0000000000..a52eaaa1cc --- /dev/null +++ b/cmd/hooks/prestop_test.go @@ -0,0 +1,316 @@ +package hooks + +import ( + "fmt" + "testing" + "time" + + "github.com/golang/mock/gomock" + "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/driver" + "github.com/stretchr/testify/assert" + v1 "k8s.io/api/core/v1" + storagev1 "k8s.io/api/storage/v1" + "k8s.io/apimachinery/pkg/watch" +) + +func TestPreStopHook(t *testing.T) { + testCases := []struct { + name string + nodeName string + expErr error + mockFunc func(string, *driver.MockKubernetesClient, *driver.MockCoreV1Interface, *driver.MockNodeInterface, *driver.MockVolumeAttachmentInterface, *driver.MockStorageV1Interface) error + }{ + { + name: "TestPreStopHook: CSI_NODE_NAME not set", + nodeName: "", + expErr: fmt.Errorf("PreStop: CSI_NODE_NAME missing"), + mockFunc: func(nodeName string, mockClient *driver.MockKubernetesClient, mockCoreV1 *driver.MockCoreV1Interface, mockNode *driver.MockNodeInterface, mockStorageV1 *driver.MockVolumeAttachmentInterface, mockStorageV1Interface *driver.MockStorageV1Interface) error { + return nil + }, + }, + { + name: "TestPreStopHook: failed to retrieve node information", + nodeName: "test-node", + expErr: fmt.Errorf("fetchNodeInfo: failed to retrieve node information: non-existent node"), + mockFunc: func(nodeName string, mockClient *driver.MockKubernetesClient, mockCoreV1 *driver.MockCoreV1Interface, mockNode *driver.MockNodeInterface, mockStorageV1 *driver.MockVolumeAttachmentInterface, mockStorageV1Interface *driver.MockStorageV1Interface) error { + mockClient.EXPECT().CoreV1().Return(mockCoreV1).Times(1) + mockCoreV1.EXPECT().Nodes().Return(mockNode).Times(1) + mockNode.EXPECT().Get(gomock.Any(), gomock.Eq(nodeName), gomock.Any()).Return(nil, fmt.Errorf("non-existent node")).Times(1) + + return nil + }, + }, + { + name: "TestPreStopHook: node is not being drained, skipping VolumeAttachments check - missing TaintNodeUnschedulable", + nodeName: "test-node", + expErr: nil, + mockFunc: func(nodeName string, mockClient *driver.MockKubernetesClient, mockCoreV1 *driver.MockCoreV1Interface, mockNode *driver.MockNodeInterface, mockStorageV1 *driver.MockVolumeAttachmentInterface, mockStorageV1Interface *driver.MockStorageV1Interface) error { + mockNodeObj := &v1.Node{ + Spec: v1.NodeSpec{ + Taints: []v1.Taint{}, + }, + } + + mockClient.EXPECT().CoreV1().Return(mockCoreV1).Times(1) + mockCoreV1.EXPECT().Nodes().Return(mockNode).Times(1) + mockNode.EXPECT().Get(gomock.Any(), gomock.Eq(nodeName), gomock.Any()).Return(mockNodeObj, nil).Times(1) + + return nil + }, + }, + { + name: "TestPreStopHook: node is not being drained, skipping VolumeAttachments check - missing TaintEffectNoSchedule", + nodeName: "test-node", + expErr: nil, + mockFunc: func(nodeName string, mockClient *driver.MockKubernetesClient, mockCoreV1 *driver.MockCoreV1Interface, mockNode *driver.MockNodeInterface, mockStorageV1 *driver.MockVolumeAttachmentInterface, mockStorageV1Interface *driver.MockStorageV1Interface) error { + mockNodeObj := &v1.Node{ + Spec: v1.NodeSpec{ + Taints: []v1.Taint{ + { + Key: v1.TaintNodeUnschedulable, + Effect: "", + }, + }, + }, + } + + mockClient.EXPECT().CoreV1().Return(mockCoreV1).Times(1) + mockCoreV1.EXPECT().Nodes().Return(mockNode).Times(1) + mockNode.EXPECT().Get(gomock.Any(), gomock.Eq(nodeName), gomock.Any()).Return(mockNodeObj, nil).Times(1) + + return nil + }, + }, + { + name: "TestPreStopHook: VolumeAttachmentList returns error", + nodeName: "test-node", + expErr: fmt.Errorf("checkVolumeAttachments: failed to list VolumeAttachments: error listing VolumeAttachments"), + mockFunc: func(nodeName string, mockClient *driver.MockKubernetesClient, mockCoreV1 *driver.MockCoreV1Interface, mockNode *driver.MockNodeInterface, mockVolumeAttachments *driver.MockVolumeAttachmentInterface, mockStorageV1Interface *driver.MockStorageV1Interface) error { + + fakeNode := &v1.Node{ + Spec: v1.NodeSpec{ + Taints: []v1.Taint{ + { + Key: v1.TaintNodeUnschedulable, + Effect: v1.TaintEffectNoSchedule, + }, + }, + }, + } + + mockClient.EXPECT().CoreV1().Return(mockCoreV1).AnyTimes() + mockClient.EXPECT().StorageV1().Return(mockStorageV1Interface).AnyTimes() + + mockCoreV1.EXPECT().Nodes().Return(mockNode).AnyTimes() + mockNode.EXPECT().Get(gomock.Any(), gomock.Eq(nodeName), gomock.Any()).Return(fakeNode, nil).AnyTimes() + + mockStorageV1Interface.EXPECT().VolumeAttachments().Return(mockVolumeAttachments).AnyTimes() + mockVolumeAttachments.EXPECT().List(gomock.Any(), gomock.Any()).Return(nil, fmt.Errorf("error listing VolumeAttachments")).AnyTimes() + + return nil + }, + }, + { + name: "TestPreStopHook: node is being drained, no volume attachments remain", + nodeName: "test-node", + expErr: nil, + mockFunc: func(nodeName string, mockClient *driver.MockKubernetesClient, mockCoreV1 *driver.MockCoreV1Interface, mockNode *driver.MockNodeInterface, mockVolumeAttachments *driver.MockVolumeAttachmentInterface, mockStorageV1Interface *driver.MockStorageV1Interface) error { + + fakeNode := &v1.Node{ + Spec: v1.NodeSpec{ + Taints: []v1.Taint{ + { + Key: v1.TaintNodeUnschedulable, + Effect: v1.TaintEffectNoSchedule, + }, + }, + }, + } + + emptyVolumeAttachments := &storagev1.VolumeAttachmentList{Items: []storagev1.VolumeAttachment{}} + + mockClient.EXPECT().CoreV1().Return(mockCoreV1).AnyTimes() + mockClient.EXPECT().StorageV1().Return(mockStorageV1Interface).AnyTimes() + + mockCoreV1.EXPECT().Nodes().Return(mockNode).AnyTimes() + mockNode.EXPECT().Get(gomock.Any(), gomock.Eq(nodeName), gomock.Any()).Return(fakeNode, nil).AnyTimes() + + mockStorageV1Interface.EXPECT().VolumeAttachments().Return(mockVolumeAttachments).AnyTimes() + mockVolumeAttachments.EXPECT().List(gomock.Any(), gomock.Any()).Return(emptyVolumeAttachments, nil).AnyTimes() + mockVolumeAttachments.EXPECT().Watch(gomock.Any(), gomock.Any()).Return(watch.NewFake(), nil).AnyTimes() + + return nil + }, + }, + { + name: "TestPreStopHook: node is being drained, no volume attachments associated with node", + nodeName: "test-node", + expErr: nil, + mockFunc: func(nodeName string, mockClient *driver.MockKubernetesClient, mockCoreV1 *driver.MockCoreV1Interface, mockNode *driver.MockNodeInterface, mockVolumeAttachments *driver.MockVolumeAttachmentInterface, mockStorageV1Interface *driver.MockStorageV1Interface) error { + + fakeNode := &v1.Node{ + Spec: v1.NodeSpec{ + Taints: []v1.Taint{ + { + Key: v1.TaintNodeUnschedulable, + Effect: v1.TaintEffectNoSchedule, + }, + }, + }, + } + + fakeVolumeAttachments := &storagev1.VolumeAttachmentList{ + Items: []storagev1.VolumeAttachment{ + { + Spec: storagev1.VolumeAttachmentSpec{ + NodeName: "test-node-2", + }, + }, + }, + } + + mockClient.EXPECT().CoreV1().Return(mockCoreV1).AnyTimes() + mockClient.EXPECT().StorageV1().Return(mockStorageV1Interface).AnyTimes() + + mockCoreV1.EXPECT().Nodes().Return(mockNode).AnyTimes() + mockNode.EXPECT().Get(gomock.Any(), gomock.Eq(nodeName), gomock.Any()).Return(fakeNode, nil).AnyTimes() + + mockStorageV1Interface.EXPECT().VolumeAttachments().Return(mockVolumeAttachments).AnyTimes() + mockVolumeAttachments.EXPECT().List(gomock.Any(), gomock.Any()).Return(fakeVolumeAttachments, nil).AnyTimes() + mockVolumeAttachments.EXPECT().Watch(gomock.Any(), gomock.Any()).Return(watch.NewFake(), nil).AnyTimes() + + return nil + }, + }, + { + name: "TestPreStopHook: node is being drained, volume attachments remain -- timeout exceeded", + nodeName: "test-node", + expErr: fmt.Errorf("waitForVolumeAttachments: timed out waiting for preStopHook to complete: context deadline exceeded"), + mockFunc: func(nodeName string, mockClient *driver.MockKubernetesClient, mockCoreV1 *driver.MockCoreV1Interface, mockNode *driver.MockNodeInterface, mockVolumeAttachments *driver.MockVolumeAttachmentInterface, mockStorageV1Interface *driver.MockStorageV1Interface) error { + + fakeNode := &v1.Node{ + Spec: v1.NodeSpec{ + Taints: []v1.Taint{ + { + Key: v1.TaintNodeUnschedulable, + Effect: v1.TaintEffectNoSchedule, + }, + }, + }, + } + + fakeVolumeAttachments := &storagev1.VolumeAttachmentList{ + Items: []storagev1.VolumeAttachment{ + { + Spec: storagev1.VolumeAttachmentSpec{ + NodeName: "test-node", + }, + }, + }, + } + + mockClient.EXPECT().CoreV1().Return(mockCoreV1).AnyTimes() + mockClient.EXPECT().StorageV1().Return(mockStorageV1Interface).AnyTimes() + + mockCoreV1.EXPECT().Nodes().Return(mockNode).AnyTimes() + mockNode.EXPECT().Get(gomock.Any(), gomock.Eq(nodeName), gomock.Any()).Return(fakeNode, nil).AnyTimes() + + mockStorageV1Interface.EXPECT().VolumeAttachments().Return(mockVolumeAttachments).AnyTimes() + mockVolumeAttachments.EXPECT().List(gomock.Any(), gomock.Any()).Return(fakeVolumeAttachments, nil).AnyTimes() + mockVolumeAttachments.EXPECT().Watch(gomock.Any(), gomock.Any()).Return(watch.NewFake(), nil).AnyTimes() + + return nil + }, + }, + { + name: "TestPreStopHook: Node is drained before timeout", + nodeName: "test-node", + expErr: nil, + mockFunc: func(nodeName string, mockClient *driver.MockKubernetesClient, mockCoreV1 *driver.MockCoreV1Interface, mockNode *driver.MockNodeInterface, mockVolumeAttachments *driver.MockVolumeAttachmentInterface, mockStorageV1Interface *driver.MockStorageV1Interface) error { + + fakeNode := &v1.Node{ + Spec: v1.NodeSpec{ + Taints: []v1.Taint{ + { + Key: v1.TaintNodeUnschedulable, + Effect: v1.TaintEffectNoSchedule, + }, + }, + }, + } + + fakeVolumeAttachments := &storagev1.VolumeAttachmentList{ + Items: []storagev1.VolumeAttachment{ + { + Spec: storagev1.VolumeAttachmentSpec{ + NodeName: "test-node", + }, + }, + }, + } + + fakeWatcher := watch.NewFake() + deleteSignal := make(chan bool, 1) + + mockClient.EXPECT().CoreV1().Return(mockCoreV1).AnyTimes() + mockClient.EXPECT().StorageV1().Return(mockStorageV1Interface).AnyTimes() + + mockCoreV1.EXPECT().Nodes().Return(mockNode).AnyTimes() + mockNode.EXPECT().Get(gomock.Any(), gomock.Eq(nodeName), gomock.Any()).Return(fakeNode, nil).AnyTimes() + + mockStorageV1Interface.EXPECT().VolumeAttachments().Return(mockVolumeAttachments).AnyTimes() + gomock.InOrder( + mockVolumeAttachments.EXPECT().List(gomock.Any(), gomock.Any()).Return(fakeVolumeAttachments, nil).AnyTimes(), + mockVolumeAttachments.EXPECT().Watch(gomock.Any(), gomock.Any()).DoAndReturn(func(signal, watchSignal interface{}) (watch.Interface, error) { + deleteSignal <- true + return fakeWatcher, nil + }).AnyTimes(), + mockVolumeAttachments.EXPECT().List(gomock.Any(), gomock.Any()).Return(&storagev1.VolumeAttachmentList{Items: []storagev1.VolumeAttachment{}}, nil).AnyTimes(), + ) + + go func() { + <-deleteSignal + fakeWatcher.Delete(&storagev1.VolumeAttachment{ + Spec: storagev1.VolumeAttachmentSpec{ + NodeName: "test-node", + }, + }) + }() + return nil + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + mockCtl := gomock.NewController(t) + defer mockCtl.Finish() + + mockClient := driver.NewMockKubernetesClient(mockCtl) + mockCoreV1 := driver.NewMockCoreV1Interface(mockCtl) + mockStorageV1 := driver.NewMockStorageV1Interface(mockCtl) + mockNode := driver.NewMockNodeInterface(mockCtl) + mockVolumeAttachments := driver.NewMockVolumeAttachmentInterface(mockCtl) + + if tc.mockFunc != nil { + err := tc.mockFunc(tc.nodeName, mockClient, mockCoreV1, mockNode, mockVolumeAttachments, mockStorageV1) + if err != nil { + t.Fatalf("TestPreStopHook: mockFunc returned error: %v", err) + } + } + + if tc.nodeName != "" { + t.Setenv("CSI_NODE_NAME", tc.nodeName) + } + + err := PreStop(mockClient, 5*time.Second) + + if tc.expErr != nil { + assert.Error(t, err) + assert.Equal(t, tc.expErr.Error(), err.Error()) + } else { + assert.NoError(t, err) + } + }) + } +} diff --git a/cmd/options.go b/cmd/options.go index 370ff1e1a2..ad318a662d 100644 --- a/cmd/options.go +++ b/cmd/options.go @@ -20,10 +20,13 @@ import ( "fmt" "os" "strings" + "time" flag "github.com/spf13/pflag" + "github.com/kubernetes-sigs/aws-ebs-csi-driver/cmd/hooks" "github.com/kubernetes-sigs/aws-ebs-csi-driver/cmd/options" + "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/cloud" "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/driver" "k8s.io/component-base/featuregate" @@ -95,6 +98,18 @@ func GetOptions(fs *flag.FlagSet) *Options { nodeOptions.AddFlags(fs) args = os.Args[1:] + case cmd == "pre-stop-hook": + clientset, clientErr := cloud.DefaultKubernetesAPIClient() + if clientErr != nil { + klog.ErrorS(err, "unable to communicate with k8s API") + } else { + err = hooks.PreStop(clientset, 1*time.Minute) + if err != nil { + klog.ErrorS(err, "failed to execute PreStop lifecycle hook") + } + } + klog.FlushAndExit(klog.ExitFlushTimeout, 0) + default: fmt.Printf("unknown command: %s: expected %q, %q or %q", cmd, driver.ControllerMode, driver.NodeMode, driver.AllMode) os.Exit(1) diff --git a/deploy/kubernetes/base/clusterrole-csi-node.yaml b/deploy/kubernetes/base/clusterrole-csi-node.yaml index 91c581e387..9ce5b9d704 100644 --- a/deploy/kubernetes/base/clusterrole-csi-node.yaml +++ b/deploy/kubernetes/base/clusterrole-csi-node.yaml @@ -10,3 +10,6 @@ rules: - apiGroups: [""] resources: ["nodes"] verbs: ["get", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["list"] diff --git a/deploy/kubernetes/base/node.yaml b/deploy/kubernetes/base/node.yaml index e25a724c06..6d35cdfbd2 100644 --- a/deploy/kubernetes/base/node.yaml +++ b/deploy/kubernetes/base/node.yaml @@ -87,6 +87,10 @@ spec: securityContext: privileged: true readOnlyRootFilesystem: true + lifecycle: + preStop: + exec: + command: ["/bin/aws-ebs-csi-driver", "pre-stop-hook"] - name: node-driver-registrar image: public.ecr.aws/eks-distro/kubernetes-csi/node-driver-registrar:v2.8.0-eks-1-27-9 imagePullPolicy: IfNotPresent diff --git a/hack/update-gomock b/hack/update-gomock index f458589c3c..ee6600dda4 100755 --- a/hack/update-gomock +++ b/hack/update-gomock @@ -25,6 +25,7 @@ mockgen -package mounter -destination=./pkg/mounter/mock_mount_windows.go -sourc mockgen -package cloud -destination=./pkg/cloud/mock_ec2.go github.com/aws/aws-sdk-go/service/ec2/ec2iface EC2API mockgen -package driver -destination=./pkg/driver/mock_k8s_client.go -mock_names='Interface=MockKubernetesClient' k8s.io/client-go/kubernetes Interface mockgen -package driver -destination=./pkg/driver/mock_k8s_corev1.go k8s.io/client-go/kubernetes/typed/core/v1 CoreV1Interface,NodeInterface +mockgen -package driver -destination=./pkg/driver/mock_k8s_storagev1.go k8s.io/client-go/kubernetes/typed/storage/v1 VolumeAttachmentInterface,StorageV1Interface # Fixes "Mounter Type cannot implement 'Mounter' as it has a non-exported method and is defined in a different package" # See https://github.com/kubernetes/mount-utils/commit/a20fcfb15a701977d086330b47b7efad51eb608e for context. diff --git a/pkg/driver/mock_k8s_storagev1.go b/pkg/driver/mock_k8s_storagev1.go new file mode 100644 index 0000000000..ea452a3e56 --- /dev/null +++ b/pkg/driver/mock_k8s_storagev1.go @@ -0,0 +1,317 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: k8s.io/client-go/kubernetes/typed/storage/v1 (interfaces: VolumeAttachmentInterface,StorageV1Interface) + +// Package driver is a generated GoMock package. +package driver + +import ( + context "context" + reflect "reflect" + + gomock "github.com/golang/mock/gomock" + v1 "k8s.io/api/storage/v1" + v10 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + v11 "k8s.io/client-go/applyconfigurations/storage/v1" + v12 "k8s.io/client-go/kubernetes/typed/storage/v1" + rest "k8s.io/client-go/rest" +) + +// MockVolumeAttachmentInterface is a mock of VolumeAttachmentInterface interface. +type MockVolumeAttachmentInterface struct { + ctrl *gomock.Controller + recorder *MockVolumeAttachmentInterfaceMockRecorder +} + +// MockVolumeAttachmentInterfaceMockRecorder is the mock recorder for MockVolumeAttachmentInterface. +type MockVolumeAttachmentInterfaceMockRecorder struct { + mock *MockVolumeAttachmentInterface +} + +// NewMockVolumeAttachmentInterface creates a new mock instance. +func NewMockVolumeAttachmentInterface(ctrl *gomock.Controller) *MockVolumeAttachmentInterface { + mock := &MockVolumeAttachmentInterface{ctrl: ctrl} + mock.recorder = &MockVolumeAttachmentInterfaceMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockVolumeAttachmentInterface) EXPECT() *MockVolumeAttachmentInterfaceMockRecorder { + return m.recorder +} + +// Apply mocks base method. +func (m *MockVolumeAttachmentInterface) Apply(arg0 context.Context, arg1 *v11.VolumeAttachmentApplyConfiguration, arg2 v10.ApplyOptions) (*v1.VolumeAttachment, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Apply", arg0, arg1, arg2) + ret0, _ := ret[0].(*v1.VolumeAttachment) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Apply indicates an expected call of Apply. +func (mr *MockVolumeAttachmentInterfaceMockRecorder) Apply(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Apply", reflect.TypeOf((*MockVolumeAttachmentInterface)(nil).Apply), arg0, arg1, arg2) +} + +// ApplyStatus mocks base method. +func (m *MockVolumeAttachmentInterface) ApplyStatus(arg0 context.Context, arg1 *v11.VolumeAttachmentApplyConfiguration, arg2 v10.ApplyOptions) (*v1.VolumeAttachment, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ApplyStatus", arg0, arg1, arg2) + ret0, _ := ret[0].(*v1.VolumeAttachment) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ApplyStatus indicates an expected call of ApplyStatus. +func (mr *MockVolumeAttachmentInterfaceMockRecorder) ApplyStatus(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ApplyStatus", reflect.TypeOf((*MockVolumeAttachmentInterface)(nil).ApplyStatus), arg0, arg1, arg2) +} + +// Create mocks base method. +func (m *MockVolumeAttachmentInterface) Create(arg0 context.Context, arg1 *v1.VolumeAttachment, arg2 v10.CreateOptions) (*v1.VolumeAttachment, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Create", arg0, arg1, arg2) + ret0, _ := ret[0].(*v1.VolumeAttachment) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Create indicates an expected call of Create. +func (mr *MockVolumeAttachmentInterfaceMockRecorder) Create(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Create", reflect.TypeOf((*MockVolumeAttachmentInterface)(nil).Create), arg0, arg1, arg2) +} + +// Delete mocks base method. +func (m *MockVolumeAttachmentInterface) Delete(arg0 context.Context, arg1 string, arg2 v10.DeleteOptions) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Delete", arg0, arg1, arg2) + ret0, _ := ret[0].(error) + return ret0 +} + +// Delete indicates an expected call of Delete. +func (mr *MockVolumeAttachmentInterfaceMockRecorder) Delete(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockVolumeAttachmentInterface)(nil).Delete), arg0, arg1, arg2) +} + +// DeleteCollection mocks base method. +func (m *MockVolumeAttachmentInterface) DeleteCollection(arg0 context.Context, arg1 v10.DeleteOptions, arg2 v10.ListOptions) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteCollection", arg0, arg1, arg2) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteCollection indicates an expected call of DeleteCollection. +func (mr *MockVolumeAttachmentInterfaceMockRecorder) DeleteCollection(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteCollection", reflect.TypeOf((*MockVolumeAttachmentInterface)(nil).DeleteCollection), arg0, arg1, arg2) +} + +// Get mocks base method. +func (m *MockVolumeAttachmentInterface) Get(arg0 context.Context, arg1 string, arg2 v10.GetOptions) (*v1.VolumeAttachment, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Get", arg0, arg1, arg2) + ret0, _ := ret[0].(*v1.VolumeAttachment) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Get indicates an expected call of Get. +func (mr *MockVolumeAttachmentInterfaceMockRecorder) Get(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockVolumeAttachmentInterface)(nil).Get), arg0, arg1, arg2) +} + +// List mocks base method. +func (m *MockVolumeAttachmentInterface) List(arg0 context.Context, arg1 v10.ListOptions) (*v1.VolumeAttachmentList, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "List", arg0, arg1) + ret0, _ := ret[0].(*v1.VolumeAttachmentList) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// List indicates an expected call of List. +func (mr *MockVolumeAttachmentInterfaceMockRecorder) List(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "List", reflect.TypeOf((*MockVolumeAttachmentInterface)(nil).List), arg0, arg1) +} + +// Patch mocks base method. +func (m *MockVolumeAttachmentInterface) Patch(arg0 context.Context, arg1 string, arg2 types.PatchType, arg3 []byte, arg4 v10.PatchOptions, arg5 ...string) (*v1.VolumeAttachment, error) { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1, arg2, arg3, arg4} + for _, a := range arg5 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "Patch", varargs...) + ret0, _ := ret[0].(*v1.VolumeAttachment) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Patch indicates an expected call of Patch. +func (mr *MockVolumeAttachmentInterfaceMockRecorder) Patch(arg0, arg1, arg2, arg3, arg4 interface{}, arg5 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1, arg2, arg3, arg4}, arg5...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Patch", reflect.TypeOf((*MockVolumeAttachmentInterface)(nil).Patch), varargs...) +} + +// Update mocks base method. +func (m *MockVolumeAttachmentInterface) Update(arg0 context.Context, arg1 *v1.VolumeAttachment, arg2 v10.UpdateOptions) (*v1.VolumeAttachment, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Update", arg0, arg1, arg2) + ret0, _ := ret[0].(*v1.VolumeAttachment) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Update indicates an expected call of Update. +func (mr *MockVolumeAttachmentInterfaceMockRecorder) Update(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Update", reflect.TypeOf((*MockVolumeAttachmentInterface)(nil).Update), arg0, arg1, arg2) +} + +// UpdateStatus mocks base method. +func (m *MockVolumeAttachmentInterface) UpdateStatus(arg0 context.Context, arg1 *v1.VolumeAttachment, arg2 v10.UpdateOptions) (*v1.VolumeAttachment, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UpdateStatus", arg0, arg1, arg2) + ret0, _ := ret[0].(*v1.VolumeAttachment) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// UpdateStatus indicates an expected call of UpdateStatus. +func (mr *MockVolumeAttachmentInterfaceMockRecorder) UpdateStatus(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateStatus", reflect.TypeOf((*MockVolumeAttachmentInterface)(nil).UpdateStatus), arg0, arg1, arg2) +} + +// Watch mocks base method. +func (m *MockVolumeAttachmentInterface) Watch(arg0 context.Context, arg1 v10.ListOptions) (watch.Interface, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Watch", arg0, arg1) + ret0, _ := ret[0].(watch.Interface) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Watch indicates an expected call of Watch. +func (mr *MockVolumeAttachmentInterfaceMockRecorder) Watch(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Watch", reflect.TypeOf((*MockVolumeAttachmentInterface)(nil).Watch), arg0, arg1) +} + +// MockStorageV1Interface is a mock of StorageV1Interface interface. +type MockStorageV1Interface struct { + ctrl *gomock.Controller + recorder *MockStorageV1InterfaceMockRecorder +} + +// MockStorageV1InterfaceMockRecorder is the mock recorder for MockStorageV1Interface. +type MockStorageV1InterfaceMockRecorder struct { + mock *MockStorageV1Interface +} + +// NewMockStorageV1Interface creates a new mock instance. +func NewMockStorageV1Interface(ctrl *gomock.Controller) *MockStorageV1Interface { + mock := &MockStorageV1Interface{ctrl: ctrl} + mock.recorder = &MockStorageV1InterfaceMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockStorageV1Interface) EXPECT() *MockStorageV1InterfaceMockRecorder { + return m.recorder +} + +// CSIDrivers mocks base method. +func (m *MockStorageV1Interface) CSIDrivers() v12.CSIDriverInterface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CSIDrivers") + ret0, _ := ret[0].(v12.CSIDriverInterface) + return ret0 +} + +// CSIDrivers indicates an expected call of CSIDrivers. +func (mr *MockStorageV1InterfaceMockRecorder) CSIDrivers() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CSIDrivers", reflect.TypeOf((*MockStorageV1Interface)(nil).CSIDrivers)) +} + +// CSINodes mocks base method. +func (m *MockStorageV1Interface) CSINodes() v12.CSINodeInterface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CSINodes") + ret0, _ := ret[0].(v12.CSINodeInterface) + return ret0 +} + +// CSINodes indicates an expected call of CSINodes. +func (mr *MockStorageV1InterfaceMockRecorder) CSINodes() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CSINodes", reflect.TypeOf((*MockStorageV1Interface)(nil).CSINodes)) +} + +// CSIStorageCapacities mocks base method. +func (m *MockStorageV1Interface) CSIStorageCapacities(arg0 string) v12.CSIStorageCapacityInterface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CSIStorageCapacities", arg0) + ret0, _ := ret[0].(v12.CSIStorageCapacityInterface) + return ret0 +} + +// CSIStorageCapacities indicates an expected call of CSIStorageCapacities. +func (mr *MockStorageV1InterfaceMockRecorder) CSIStorageCapacities(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CSIStorageCapacities", reflect.TypeOf((*MockStorageV1Interface)(nil).CSIStorageCapacities), arg0) +} + +// RESTClient mocks base method. +func (m *MockStorageV1Interface) RESTClient() rest.Interface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RESTClient") + ret0, _ := ret[0].(rest.Interface) + return ret0 +} + +// RESTClient indicates an expected call of RESTClient. +func (mr *MockStorageV1InterfaceMockRecorder) RESTClient() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RESTClient", reflect.TypeOf((*MockStorageV1Interface)(nil).RESTClient)) +} + +// StorageClasses mocks base method. +func (m *MockStorageV1Interface) StorageClasses() v12.StorageClassInterface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "StorageClasses") + ret0, _ := ret[0].(v12.StorageClassInterface) + return ret0 +} + +// StorageClasses indicates an expected call of StorageClasses. +func (mr *MockStorageV1InterfaceMockRecorder) StorageClasses() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StorageClasses", reflect.TypeOf((*MockStorageV1Interface)(nil).StorageClasses)) +} + +// VolumeAttachments mocks base method. +func (m *MockStorageV1Interface) VolumeAttachments() v12.VolumeAttachmentInterface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "VolumeAttachments") + ret0, _ := ret[0].(v12.VolumeAttachmentInterface) + return ret0 +} + +// VolumeAttachments indicates an expected call of VolumeAttachments. +func (mr *MockStorageV1InterfaceMockRecorder) VolumeAttachments() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "VolumeAttachments", reflect.TypeOf((*MockStorageV1Interface)(nil).VolumeAttachments)) +}