Skip to content

Commit

Permalink
1.分离 client 中的真实和 mock 调用 k8s api,以便于 ut 不依赖于当前的 k8s 环境。
Browse files Browse the repository at this point in the history
factory.go: 将client的实例管理逻辑统一,方便调用和切换fake。兼容当前 KubeClient 全局变量的使用。
k8s_client.go: 统一实现 k8s 真实调用行为, 记录调用日志,去掉全局变量。

2.修改 util 和 util_test 中的相关调用, 统一 util_test 中测试用例的命名风格。
本次修改仅关注于 real 和 mock 分离,后续将完善更多 ut 测试,逐步替换当前 client 中的直接调用方式。

3.兼容当前新老调用 KubeClient 的用法,特别是 fake 替换全部变量时。

Signed-off-by: yangshiqi <[email protected]>
  • Loading branch information
yangshiqi committed Mar 6, 2025
1 parent 857ee57 commit e68be47
Show file tree
Hide file tree
Showing 7 changed files with 315 additions and 23 deletions.
2 changes: 2 additions & 0 deletions pkg/device/hygon/device_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import (
"gotest.tools/v3/assert"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes/fake"

"k8s.io/apimachinery/pkg/api/resource"
"k8s.io/klog/v2"
Expand Down Expand Up @@ -673,6 +674,7 @@ func Test_GenerateResourceRequests(t *testing.T) {
}

func Test_NodeCleanUp(t *testing.T) {
client.KubeClient = fake.NewSimpleClientset()
tests := []struct {
name string
args string
Expand Down
12 changes: 9 additions & 3 deletions pkg/util/client/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,12 +33,18 @@ var (
once sync.Once
)

func init() {
KubeClient = nil
}

func GetClient() kubernetes.Interface {
once.Do(func() {
var err error
KubeClient, err = newClient()
if err != nil {
klog.Fatalf("Failed to create Kubernetes client: %v", err)
if KubeClient == nil {
KubeClient, err = newClient()
if err != nil {
klog.Fatalf("Failed to create Kubernetes client: %v", err)
}

Check warning on line 47 in pkg/util/client/client.go

View check run for this annotation

Codecov / codecov/patch

pkg/util/client/client.go#L46-L47

Added lines #L46 - L47 were not covered by tests
}
})
return KubeClient
Expand Down
88 changes: 88 additions & 0 deletions pkg/util/client/factory.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
/*
Copyright 2024 The HAMi Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package client

import (
"sync"

"k8s.io/client-go/kubernetes/fake"
)

// KubeClientFactory is a factory class (singleton pattern).
type KubeClientFactory struct {
client KubeInterface
}

var factoryOnce sync.Once

// GetFactory gets the singleton factory object.
// Uses closure to maintain instance as a local variable.
var GetFactory = (func() func() *KubeClientFactory {
var instance *KubeClientFactory

return func() *KubeClientFactory {
factoryOnce.Do(func() {
instance = &KubeClientFactory{}
instance.SetReal() // Use the real client by default.
})
return instance

Check warning on line 42 in pkg/util/client/factory.go

View check run for this annotation

Codecov / codecov/patch

pkg/util/client/factory.go#L38-L42

Added lines #L38 - L42 were not covered by tests
}
})()

// NewInstance directly gets the Kubernetes client instance.
func NewInstance() KubeInterface {
return GetFactory().GetClient()

Check warning on line 48 in pkg/util/client/factory.go

View check run for this annotation

Codecov / codecov/patch

pkg/util/client/factory.go#L47-L48

Added lines #L47 - L48 were not covered by tests
}

func (f *KubeClientFactory) GetClient() KubeInterface {
//TODO: For compatibility with other direct "KubeClient" assignment call points, this line needs to be removed after replacement.
if KubeClient == nil {
f.client = &K8sClient{
client: GetK8sClient().client,
}
KubeClient = GetK8sClient().client
} else {
f.client = &K8sClient{
client: KubeClient,
}
}
return f.client

Check warning on line 63 in pkg/util/client/factory.go

View check run for this annotation

Codecov / codecov/patch

pkg/util/client/factory.go#L51-L63

Added lines #L51 - L63 were not covered by tests
}

func (f *KubeClientFactory) SetFake() *KubeClientFactory {
f.client = &K8sClient{
client: fake.NewSimpleClientset(),
}
//TODO: For compatibility with other direct "KubeClient" assignment call points, this line needs to be removed after replacement.
KubeClient = fake.NewSimpleClientset()
return f

Check warning on line 72 in pkg/util/client/factory.go

View check run for this annotation

Codecov / codecov/patch

pkg/util/client/factory.go#L66-L72

Added lines #L66 - L72 were not covered by tests
}

func (f *KubeClientFactory) SetReal() *KubeClientFactory {
//TODO: For compatibility with other direct "KubeClient" assignment call points, this line needs to be removed after replacement.
if KubeClient == nil {
f.client = &K8sClient{
client: GetK8sClient().client,
}
KubeClient = GetK8sClient().client
} else {
f.client = &K8sClient{
client: KubeClient,
}
}
return f

Check warning on line 87 in pkg/util/client/factory.go

View check run for this annotation

Codecov / codecov/patch

pkg/util/client/factory.go#L75-L87

Added lines #L75 - L87 were not covered by tests
}
34 changes: 34 additions & 0 deletions pkg/util/client/interface.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
/*
Copyright 2024 The HAMi Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package client

import (
"context"

corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
)

type KubeInterface interface {
GetNode(ctx context.Context, name string, opts metav1.GetOptions) (*corev1.Node, error)
ListPods(ctx context.Context, namespace string, opts metav1.ListOptions) (*corev1.PodList, error)
GetPod(ctx context.Context, namespace, name string, opts metav1.GetOptions) (*corev1.Pod, error)
PatchNode(ctx context.Context, name string, pt types.PatchType, data []byte, opts metav1.PatchOptions) (*corev1.Node, error)
PatchPod(ctx context.Context, namespace string, name string, pt types.PatchType, data []byte, opts metav1.PatchOptions) (*corev1.Pod, error)
CreateNode(ctx context.Context, node *corev1.Node, opts metav1.CreateOptions) (*corev1.Node, error)
}
158 changes: 158 additions & 0 deletions pkg/util/client/k8s_client.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,158 @@
/*
Copyright 2024 The HAMi Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package client

import (
"context"
"fmt"
"os"
"path/filepath"
"sync"

corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
"k8s.io/klog/v2"
)

var ClientOnce sync.Once

// GetK8sClient creates a real Kubernetes client (singleton pattern).
// Uses closure to maintain client as a local variable.
var GetK8sClient = (func() func() *K8sClient {
var client *K8sClient

return func() *K8sClient {
var err error
ClientOnce.Do(func() {
client, err = createRealClient()
if err != nil {
klog.Fatalf("Failed to create Kubernetes client: %v", err)
}

Check warning on line 48 in pkg/util/client/k8s_client.go

View check run for this annotation

Codecov / codecov/patch

pkg/util/client/k8s_client.go#L43-L48

Added lines #L43 - L48 were not covered by tests
})
return client

Check warning on line 50 in pkg/util/client/k8s_client.go

View check run for this annotation

Codecov / codecov/patch

pkg/util/client/k8s_client.go#L50

Added line #L50 was not covered by tests
}
})()

type K8sClient struct {
client kubernetes.Interface
}

var _ KubeInterface = (*K8sClient)(nil)

func createRealClient() (*K8sClient, error) {
kubeConfigPath := os.Getenv("KUBECONFIG")
if kubeConfigPath == "" {
kubeConfigPath = filepath.Join(os.Getenv("HOME"), ".kube", "config")
}

Check warning on line 64 in pkg/util/client/k8s_client.go

View check run for this annotation

Codecov / codecov/patch

pkg/util/client/k8s_client.go#L60-L64

Added lines #L60 - L64 were not covered by tests

config, err := clientcmd.BuildConfigFromFlags("", kubeConfigPath)
if err != nil {
klog.ErrorS(err, "BuildConfigFromFlags failed for file %s: %v. Using in-cluster config.", kubeConfigPath, err)
config, err = rest.InClusterConfig()
if err != nil {
return nil, fmt.Errorf("failed to get in-cluster config: %w", err)
}

Check warning on line 72 in pkg/util/client/k8s_client.go

View check run for this annotation

Codecov / codecov/patch

pkg/util/client/k8s_client.go#L66-L72

Added lines #L66 - L72 were not covered by tests
}

clientset, err := kubernetes.NewForConfig(config)
if err != nil {
return nil, fmt.Errorf("failed to create kubernetes client: %w", err)
}

Check warning on line 78 in pkg/util/client/k8s_client.go

View check run for this annotation

Codecov / codecov/patch

pkg/util/client/k8s_client.go#L75-L78

Added lines #L75 - L78 were not covered by tests

return &K8sClient{client: clientset}, nil

Check warning on line 80 in pkg/util/client/k8s_client.go

View check run for this annotation

Codecov / codecov/patch

pkg/util/client/k8s_client.go#L80

Added line #L80 was not covered by tests
}

func (c *K8sClient) GetNode(ctx context.Context, name string, opts metav1.GetOptions) (*corev1.Node, error) {
klog.V(4).InfoS("Retrieving node", "node", name)
node, err := c.client.CoreV1().Nodes().Get(ctx, name, opts)
if err != nil {
klog.ErrorS(err, "Failed to get node", "node", name)
return nil, fmt.Errorf("failed to get node %s: %w", name, err)
}
klog.V(4).InfoS("Successfully retrieved node", "node", name)
return node, nil

Check warning on line 91 in pkg/util/client/k8s_client.go

View check run for this annotation

Codecov / codecov/patch

pkg/util/client/k8s_client.go#L83-L91

Added lines #L83 - L91 were not covered by tests
}

func (c *K8sClient) GetPod(ctx context.Context, namespace, name string, opts metav1.GetOptions) (*corev1.Pod, error) {
klog.V(4).InfoS("Retrieving pod", "namespace", namespace, "pod", name)
pod, err := c.client.CoreV1().Pods(namespace).Get(ctx, name, opts)
if err != nil {
klog.ErrorS(err, "Failed to get pod", "namespace", namespace, "pod", name)
return nil, fmt.Errorf("failed to get pod %s/%s: %w", namespace, name, err)
}
klog.V(4).InfoS("Successfully retrieved pod", "namespace", namespace, "pod", name)
return pod, nil

Check warning on line 102 in pkg/util/client/k8s_client.go

View check run for this annotation

Codecov / codecov/patch

pkg/util/client/k8s_client.go#L94-L102

Added lines #L94 - L102 were not covered by tests
}

func (c *K8sClient) ListPods(ctx context.Context, namespace string, opts metav1.ListOptions) (*corev1.PodList, error) {
klog.V(4).InfoS("Listing pods", "namespace", namespace, "options", opts)
pods, err := c.client.CoreV1().Pods(namespace).List(ctx, opts)
if err != nil {
klog.ErrorS(err, "Failed to list pods", "namespace", namespace)
return nil, fmt.Errorf("failed to list pods in namespace %s: %w", namespace, err)
}
klog.V(4).InfoS("Successfully listed pods", "namespace", namespace, "count", len(pods.Items))
return pods, nil

Check warning on line 113 in pkg/util/client/k8s_client.go

View check run for this annotation

Codecov / codecov/patch

pkg/util/client/k8s_client.go#L105-L113

Added lines #L105 - L113 were not covered by tests
}

func (c *K8sClient) PatchNode(ctx context.Context, name string, pt types.PatchType, data []byte, opts metav1.PatchOptions) (*corev1.Node, error) {
klog.V(4).InfoS("Patching node", "node", name, "patchType", pt)
node, err := c.client.CoreV1().Nodes().Patch(ctx, name, pt, data, opts)
if err != nil {
klog.ErrorS(err, "Failed to patch node", "node", name)
return nil, fmt.Errorf("failed to patch node %s: %w", name, err)
}
klog.V(4).InfoS("Successfully patched node", "node", name)
return node, nil

Check warning on line 124 in pkg/util/client/k8s_client.go

View check run for this annotation

Codecov / codecov/patch

pkg/util/client/k8s_client.go#L116-L124

Added lines #L116 - L124 were not covered by tests
}

func (c *K8sClient) PatchPod(ctx context.Context, namespace string, name string, pt types.PatchType, data []byte, opts metav1.PatchOptions) (*corev1.Pod, error) {
klog.V(4).InfoS("Patching pod", "namespace", namespace, "pod", name, "patchType", pt)
pod, err := c.client.CoreV1().Pods(namespace).Patch(ctx, name, pt, data, opts)
if err != nil {
klog.ErrorS(err, "Failed to patch pod", "namespace", namespace, "pod", name)
return nil, fmt.Errorf("failed to patch pod %s/%s: %w", namespace, name, err)
}
klog.V(4).InfoS("Successfully patched pod", "namespace", namespace, "pod", name)
return pod, nil

Check warning on line 135 in pkg/util/client/k8s_client.go

View check run for this annotation

Codecov / codecov/patch

pkg/util/client/k8s_client.go#L127-L135

Added lines #L127 - L135 were not covered by tests
}

func (c *K8sClient) CreateNode(ctx context.Context, node *corev1.Node, opts metav1.CreateOptions) (*corev1.Node, error) {
klog.V(4).InfoS("Creating node", "node", node.Name)
createdNode, err := c.client.CoreV1().Nodes().Create(ctx, node, opts)
if err != nil {
klog.ErrorS(err, "Failed to create node", "node", node.Name)
return nil, fmt.Errorf("failed to create node %s: %w", node.Name, err)
}
klog.V(4).InfoS("Successfully created node", "node", node.Name)
return createdNode, nil

Check warning on line 146 in pkg/util/client/k8s_client.go

View check run for this annotation

Codecov / codecov/patch

pkg/util/client/k8s_client.go#L138-L146

Added lines #L138 - L146 were not covered by tests
}

func (c *K8sClient) UpdateNode(ctx context.Context, node *corev1.Node, opts metav1.UpdateOptions) (*corev1.Node, error) {
klog.V(4).InfoS("Updating node", "node", node.Name)
updatedNode, err := c.client.CoreV1().Nodes().Update(ctx, node, opts)
if err != nil {
klog.ErrorS(err, "Failed to update node", "node", node.Name)
return nil, fmt.Errorf("failed to update node %s: %w", node.Name, err)
}
klog.V(4).InfoS("Successfully updated node", "node", node.Name)
return updatedNode, nil

Check warning on line 157 in pkg/util/client/k8s_client.go

View check run for this annotation

Codecov / codecov/patch

pkg/util/client/k8s_client.go#L149-L157

Added lines #L149 - L157 were not covered by tests
}
15 changes: 7 additions & 8 deletions pkg/util/util.go
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,8 @@ func GetNode(nodename string) (*corev1.Node, error) {
}

klog.InfoS("Fetching node", "nodeName", nodename)
n, err := client.GetClient().CoreV1().Nodes().Get(context.Background(), nodename, metav1.GetOptions{})

n, err := client.NewInstance().GetNode(context.Background(), nodename, metav1.GetOptions{})
if err != nil {
switch {
case apierrors.IsNotFound(err):
Expand Down Expand Up @@ -95,7 +96,7 @@ func GetPendingPod(ctx context.Context, node string) (*corev1.Pod, error) {
podListOptions := metav1.ListOptions{
FieldSelector: selector,
}
podlist, err := client.GetClient().CoreV1().Pods("").List(ctx, podListOptions)
podlist, err := client.NewInstance().ListPods(ctx, "", podListOptions)

Check warning on line 99 in pkg/util/util.go

View check run for this annotation

Codecov / codecov/patch

pkg/util/util.go#L99

Added line #L99 was not covered by tests
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -125,7 +126,7 @@ func GetPendingPod(ctx context.Context, node string) (*corev1.Pod, error) {
}

func GetAllocatePodByNode(ctx context.Context, nodeName string) (*corev1.Pod, error) {
node, err := client.GetClient().CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{})
node, err := client.NewInstance().GetNode(ctx, nodeName, metav1.GetOptions{})

Check warning on line 129 in pkg/util/util.go

View check run for this annotation

Codecov / codecov/patch

pkg/util/util.go#L129

Added line #L129 was not covered by tests
if err != nil {
return nil, err
}
Expand All @@ -138,7 +139,7 @@ func GetAllocatePodByNode(ctx context.Context, nodeName string) (*corev1.Pod, er
if ns == "" || name == "" {
return nil, nil
}
return client.GetClient().CoreV1().Pods(ns).Get(ctx, name, metav1.GetOptions{})
return client.NewInstance().GetPod(ctx, ns, name, metav1.GetOptions{})

Check warning on line 142 in pkg/util/util.go

View check run for this annotation

Codecov / codecov/patch

pkg/util/util.go#L142

Added line #L142 was not covered by tests
}
return nil, nil
}
Expand Down Expand Up @@ -342,8 +343,7 @@ func PatchNodeAnnotations(node *corev1.Node, annotations map[string]string) erro
if err != nil {
return err
}
_, err = client.GetClient().CoreV1().Nodes().
Patch(context.Background(), node.Name, k8stypes.StrategicMergePatchType, bytes, metav1.PatchOptions{})
_, err = client.NewInstance().PatchNode(context.Background(), node.Name, k8stypes.StrategicMergePatchType, bytes, metav1.PatchOptions{})
if err != nil {
klog.Infoln("annotations=", annotations)
klog.Infof("patch pod %v failed, %v", node.Name, err)
Expand Down Expand Up @@ -374,8 +374,7 @@ func PatchPodAnnotations(pod *corev1.Pod, annotations map[string]string) error {
return err
}
klog.V(5).Infof("patch pod %s/%s annotation content is %s", pod.Namespace, pod.Name, string(bytes))
_, err = client.GetClient().CoreV1().Pods(pod.Namespace).
Patch(context.Background(), pod.Name, k8stypes.StrategicMergePatchType, bytes, metav1.PatchOptions{})
_, err = client.NewInstance().PatchPod(context.Background(), pod.Namespace, pod.Name, k8stypes.StrategicMergePatchType, bytes, metav1.PatchOptions{})

Check warning on line 377 in pkg/util/util.go

View check run for this annotation

Codecov / codecov/patch

pkg/util/util.go#L377

Added line #L377 was not covered by tests
if err != nil {
klog.Infof("patch pod %v failed, %v", pod.Name, err)
}
Expand Down
Loading

0 comments on commit e68be47

Please sign in to comment.