From c744fff9e844baf3c85b43f2b61c73a1db22a8f3 Mon Sep 17 00:00:00 2001 From: Xiao Liu <46879761+liangzai006@users.noreply.github.com> Date: Thu, 26 Oct 2023 11:28:01 +0800 Subject: [PATCH] optimize nodeinfo inspect and add nodeinfo html out --- apis/kubeeye/v1alpha2/inspectresult_types.go | 15 +- apis/kubeeye/v1alpha2/inspectrule_types.go | 3 +- .../kubeeye/v1alpha2/zz_generated.deepcopy.go | 37 +++-- .../kubeeye.kubesphere.io_inspectresults.yaml | 4 + .../kubeeye.kubesphere.io_inspectrules.yaml | 6 +- config/rbac/role.yaml | 1 + deploy/kubeeye_v1alpha2_inspectplan.yaml | 8 ++ pkg/controllers/inspectresult_controller.go | 3 +- pkg/controllers/inspecttask_controller.go | 2 +- pkg/inspect/file_change_Inspect.go | 3 + pkg/inspect/file_filter_inspect.go | 2 + pkg/inspect/nodeinfo_inspect.go | 130 +++++++++--------- pkg/output/html.go | 35 +++++ 13 files changed, 147 insertions(+), 102 deletions(-) diff --git a/apis/kubeeye/v1alpha2/inspectresult_types.go b/apis/kubeeye/v1alpha2/inspectresult_types.go index 989365a4..f838ddb8 100644 --- a/apis/kubeeye/v1alpha2/inspectresult_types.go +++ b/apis/kubeeye/v1alpha2/inspectresult_types.go @@ -69,15 +69,15 @@ type PrometheusResult struct { } type NodeInfoResultItem struct { - Name string `json:"name,omitempty"` - FileSystem FileSystemLabel `json:",inline"` - Value string `json:"value,omitempty"` - Assert bool `json:"assert,omitempty"` - Level Level `json:"level,omitempty"` - NodeName string `json:"nodeName,omitempty"` + Name string `json:"name,omitempty"` + ResourcesType ResourcesType `json:",inline"` + Value string `json:"value,omitempty"` + Assert bool `json:"assert,omitempty"` + Level Level `json:"level,omitempty"` + NodeName string `json:"nodeName,omitempty"` } -type FileSystemLabel struct { +type ResourcesType struct { Mount string `json:"mount,omitempty"` Type string `json:"type,omitempty"` } @@ -88,6 +88,7 @@ type FileChangeResultItem struct { Path string `json:"path,omitempty"` Level Level `json:"level,omitempty"` NodeName string `json:"nodeName,omitempty"` + Assert bool `json:"assert,omitempty"` } type NodeMetricsResultItem struct { Name string `json:"name,omitempty"` diff --git a/apis/kubeeye/v1alpha2/inspectrule_types.go b/apis/kubeeye/v1alpha2/inspectrule_types.go index 677b4abd..28855b77 100644 --- a/apis/kubeeye/v1alpha2/inspectrule_types.go +++ b/apis/kubeeye/v1alpha2/inspectrule_types.go @@ -59,7 +59,8 @@ type Node struct { type NodeInfo struct { RuleItemBases `json:",inline"` Node `json:",inline"` - Mount []string `json:"mount,omitempty"` + ResourcesType string `json:"resourcesType,omitempty"` + Mount string `json:"mount,omitempty"` } type FileFilterRule struct { diff --git a/apis/kubeeye/v1alpha2/zz_generated.deepcopy.go b/apis/kubeeye/v1alpha2/zz_generated.deepcopy.go index 97615eaa..e08ad797 100644 --- a/apis/kubeeye/v1alpha2/zz_generated.deepcopy.go +++ b/apis/kubeeye/v1alpha2/zz_generated.deepcopy.go @@ -201,21 +201,6 @@ func (in *FileFilterRule) DeepCopy() *FileFilterRule { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *FileSystemLabel) DeepCopyInto(out *FileSystemLabel) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FileSystemLabel. -func (in *FileSystemLabel) DeepCopy() *FileSystemLabel { - if in == nil { - return nil - } - out := new(FileSystemLabel) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *InspectPlan) DeepCopyInto(out *InspectPlan) { *out = *in @@ -914,11 +899,6 @@ func (in *NodeInfo) DeepCopyInto(out *NodeInfo) { *out = *in in.RuleItemBases.DeepCopyInto(&out.RuleItemBases) in.Node.DeepCopyInto(&out.Node) - if in.Mount != nil { - in, out := &in.Mount, &out.Mount - *out = make([]string, len(*in)) - copy(*out, *in) - } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NodeInfo. @@ -934,7 +914,7 @@ func (in *NodeInfo) DeepCopy() *NodeInfo { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NodeInfoResultItem) DeepCopyInto(out *NodeInfoResultItem) { *out = *in - out.FileSystem = in.FileSystem + out.ResourcesType = in.ResourcesType } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NodeInfoResultItem. @@ -1034,6 +1014,21 @@ func (in *ResourceResult) DeepCopy() *ResourceResult { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ResourcesType) DeepCopyInto(out *ResourcesType) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ResourcesType. +func (in *ResourcesType) DeepCopy() *ResourcesType { + if in == nil { + return nil + } + out := new(ResourcesType) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ResultItem) DeepCopyInto(out *ResultItem) { *out = *in diff --git a/config/crd/bases/kubeeye.kubesphere.io_inspectresults.yaml b/config/crd/bases/kubeeye.kubesphere.io_inspectresults.yaml index 27c5a7b9..0917fed3 100644 --- a/config/crd/bases/kubeeye.kubesphere.io_inspectresults.yaml +++ b/config/crd/bases/kubeeye.kubesphere.io_inspectresults.yaml @@ -70,6 +70,8 @@ spec: fileChangeResult: items: properties: + assert: + type: boolean fileName: type: string issues: @@ -87,6 +89,8 @@ spec: fileFilterResult: items: properties: + assert: + type: boolean fileName: type: string issues: diff --git a/config/crd/bases/kubeeye.kubesphere.io_inspectrules.yaml b/config/crd/bases/kubeeye.kubesphere.io_inspectrules.yaml index 6525db6b..1f2a06ce 100644 --- a/config/crd/bases/kubeeye.kubesphere.io_inspectrules.yaml +++ b/config/crd/bases/kubeeye.kubesphere.io_inspectrules.yaml @@ -117,9 +117,7 @@ spec: level: type: string mount: - items: - type: string - type: array + type: string name: type: string nodeName: @@ -128,6 +126,8 @@ spec: additionalProperties: type: string type: object + resourcesType: + type: string rule: type: string type: object diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 590eb7e7..8de94209 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -25,6 +25,7 @@ rules: resources: - namespaces - nodes + - secrets - services verbs: - get diff --git a/deploy/kubeeye_v1alpha2_inspectplan.yaml b/deploy/kubeeye_v1alpha2_inspectplan.yaml index f482ba7f..52311a12 100644 --- a/deploy/kubeeye_v1alpha2_inspectplan.yaml +++ b/deploy/kubeeye_v1alpha2_inspectplan.yaml @@ -7,3 +7,11 @@ spec: ruleNames: - name: inspect-rule-namespace - name: inspect-rule-systemd + - name: inspect-rule-filechange + - name: inspect-rule-filter-file + - name: inspect-rule-node-info + - name: inspect-rule-prometheus + - name: inspect-rule-sbnormalpodstatus + - name: inspect-rule-sysctl + timeout: 30m + diff --git a/pkg/controllers/inspectresult_controller.go b/pkg/controllers/inspectresult_controller.go index 4e2ab6ae..09341d6d 100644 --- a/pkg/controllers/inspectresult_controller.go +++ b/pkg/controllers/inspectresult_controller.go @@ -140,6 +140,7 @@ func (r *InspectResultReconciler) Reconcile(ctx context.Context, req ctrl.Reques klog.Error("Failed to update inspect result status", err) return ctrl.Result{}, err } + go r.SendMessage(ctx, result) return ctrl.Result{}, nil } @@ -251,7 +252,7 @@ func (r *InspectResultReconciler) SendMessage(ctx context.Context, result *kubee messageHandler := message.NewEmailMessageOptions(&kc.Message.Email, r.Client) dispatcher := message.RegisterHandler(messageHandler) dispatcher.DispatchMessageEvent(&conf.MessageEvent{ - Title: fmt.Sprintf("巡检完成,共发现%d个问题", n), + Title: fmt.Sprintf("%s集群巡检完成,共发现%d个问题", result.Spec.InspectCluster.Name, n), Timestamp: time.Now(), Content: data.Bytes(), }) diff --git a/pkg/controllers/inspecttask_controller.go b/pkg/controllers/inspecttask_controller.go index a794207b..2d489516 100644 --- a/pkg/controllers/inspecttask_controller.go +++ b/pkg/controllers/inspecttask_controller.go @@ -57,7 +57,7 @@ type InspectTaskReconciler struct { //+kubebuilder:rbac:groups=cluster.kubesphere.io,resources=clusters,verbs=get //+kubebuilder:rbac:groups=kubeeye.kubesphere.io,resources=inspecttasks/status,verbs=get;update;patch //+kubebuilder:rbac:groups=kubeeye.kubesphere.io,resources=inspecttasks/finalizers,verbs=update -//+kubebuilder:rbac:groups="",resources=nodes;namespaces;services,verbs=list;get +//+kubebuilder:rbac:groups="",resources=nodes;namespaces;services;secrets,verbs=list;get //+kubebuilder:rbac:groups="",resources=namespaces,verbs=create //+kubebuilder:rbac:groups="",resources=serviceaccounts,verbs=create;delete //+kubebuilder:rbac:groups="",resources=configmaps,verbs=deletecollection;list;get;watch diff --git a/pkg/inspect/file_change_Inspect.go b/pkg/inspect/file_change_Inspect.go index 9f830cef..7e6b7273 100644 --- a/pkg/inspect/file_change_Inspect.go +++ b/pkg/inspect/file_change_Inspect.go @@ -82,6 +82,7 @@ func (o *fileChangeInspect) RunInspect(ctx context.Context, rules []kubeeyev1alp klog.Errorf("Failed to open base file path:%s,error:%s", baseFile, fileErr) resultItem.Issues = []string{fmt.Sprintf("%s:The file does not exist", file.Name)} resultItem.Level = file.Level + resultItem.Assert = true fileResults = append(fileResults, resultItem) continue } @@ -95,6 +96,7 @@ func (o *fileChangeInspect) RunInspect(ctx context.Context, rules []kubeeyev1alp if createErr != nil { resultItem.Issues = []string{fmt.Sprintf("%s:create configMap failed", file.Name)} resultItem.Level = file.Level + resultItem.Assert = true fileResults = append(fileResults, resultItem) } continue @@ -112,6 +114,7 @@ func (o *fileChangeInspect) RunInspect(ctx context.Context, rules []kubeeyev1alp resultItem.Issues = diffResult if len(resultItem.Issues) > 0 { resultItem.Level = file.Level + resultItem.Assert = true } fileResults = append(fileResults, resultItem) } diff --git a/pkg/inspect/file_filter_inspect.go b/pkg/inspect/file_filter_inspect.go index 758667e6..6f08a8a6 100644 --- a/pkg/inspect/file_filter_inspect.go +++ b/pkg/inspect/file_filter_inspect.go @@ -78,6 +78,7 @@ func (o *fileFilterInspect) RunInspect(ctx context.Context, rules []kubeeyev1alp klog.Errorf(" Failed to open file . err:%s", err) filterR.Issues = append(filterR.Issues, fmt.Sprintf("Failed to open file for %s.", rule.Name)) filterR.Level = rule.Level + filterR.Assert = true filterResult = append(filterResult, filterR) continue } @@ -94,6 +95,7 @@ func (o *fileFilterInspect) RunInspect(ctx context.Context, rules []kubeeyev1alp } } if len(filterR.Issues) > 0 { + filterR.Assert = true filterR.Level = rule.Level } filterResult = append(filterResult, filterR) diff --git a/pkg/inspect/nodeinfo_inspect.go b/pkg/inspect/nodeinfo_inspect.go index 93211d86..49f16ccb 100644 --- a/pkg/inspect/nodeinfo_inspect.go +++ b/pkg/inspect/nodeinfo_inspect.go @@ -19,7 +19,6 @@ import ( "k8s.io/klog/v2" "path" - "math" "strings" ) @@ -79,80 +78,66 @@ func (o *nodeInfoInspect) RunInspect(ctx context.Context, rules []kubeeyev1alpha for _, info := range nodeInfo { ok := false resultItem := kubeeyev1alpha2.NodeInfoResultItem{ - Name: info.Name, - Level: info.Level, + Name: info.Name, } - switch strings.ToLower(info.Name) { + switch strings.ToLower(info.ResourcesType) { case constant.Cpu: data := GetCpu(fs) - resultItem.Value = fmt.Sprintf("%.0f%%", data) - err, ok = visitor.EventRuleEvaluate(map[string]interface{}{constant.Cpu: data}, *info.Rule) + resultItem.Value = fmt.Sprintf("%.0f%%", data[constant.Cpu]) + resultItem.ResourcesType.Type = constant.Cpu + err, ok = visitor.EventRuleEvaluate(data, *info.Rule) if err != nil { resultItem.Value = err.Error() + resultItem.Assert = true } case constant.Memory: data := GetMemory(fs) - resultItem.Value = fmt.Sprintf("%.0f%%", data) - err, ok = visitor.EventRuleEvaluate(map[string]interface{}{constant.Memory: data}, *info.Rule) + resultItem.Value = fmt.Sprintf("%.0f%%", data[constant.Cpu]) + resultItem.ResourcesType.Type = constant.Memory + err, ok = visitor.EventRuleEvaluate(data, *info.Rule) if err != nil { resultItem.Value = err.Error() + resultItem.Assert = true } case constant.Filesystem: - if info.Mount == nil { - info.Mount = append(info.Mount, "/") + if utils.IsEmptyValue(info.Mount) { + info.Mount = "/" + } + storage := GetFileSystem(info.Mount) + resultItem.ResourcesType.Type = constant.Filesystem + resultItem.ResourcesType.Mount = info.Mount + resultItem.Value = fmt.Sprintf("%.0f%%", storage[constant.Filesystem]) + err, ok = visitor.EventRuleEvaluate(storage, *info.Rule) + if err != nil { + resultItem.Value = err.Error() + resultItem.Assert = true } - for _, m := range info.Mount { - resultItem.FileSystem.Mount = m - storage, inode := GetFileSystem(m) - resultItem.Value = fmt.Sprintf("%.0f%%", storage) - err, ok = visitor.EventRuleEvaluate(map[string]interface{}{constant.Filesystem: storage}, *info.Rule) - if err != nil { - resultItem.Value = err.Error() - resultItem.Assert = true - resultItem.Level = info.Level - } else { - if ok { - resultItem.Level = info.Level - } - resultItem.Assert = ok - } - - resultItem.FileSystem.Type = constant.Filesystem - nodeInfoResult = append(nodeInfoResult, resultItem) - resultItem.Value = fmt.Sprintf("%.0f%%", inode) - err, ok = visitor.EventRuleEvaluate(map[string]interface{}{constant.Inode: inode}, *info.Rule) - if err != nil { - resultItem.Value = err.Error() - resultItem.Assert = true - resultItem.Level = info.Level - } else { - if ok { - resultItem.Level = info.Level - } - resultItem.Assert = ok - } - - resultItem.FileSystem.Type = constant.Inode - nodeInfoResult = append(nodeInfoResult, resultItem) + case constant.Inode: + if utils.IsEmptyValue(info.Mount) { + info.Mount = "/" + } + inodes := GetInodes(info.Mount) + resultItem.ResourcesType.Type = constant.Inode + resultItem.ResourcesType.Mount = info.Mount + resultItem.Value = fmt.Sprintf("%.0f%%", inodes[constant.Inode]) + err, ok = visitor.EventRuleEvaluate(inodes, *info.Rule) + if err != nil { + resultItem.Value = err.Error() + resultItem.Assert = true } - continue case constant.LoadAvg: - a, b, c := GetLoadAvg(fs) - resultItem.Value = fmt.Sprintf("load1:%.0f,load5:%.0f,load15:%.0f", a, b, c) - var data = make(map[string]interface{}) - data["load1"] = a - data["load5"] = b - data["load15"] = c - err, ok = visitor.EventRuleEvaluate(data, *info.Rule) + loadAvg := GetLoadAvg(fs) + resultItem.Value = fmt.Sprintf("load1:%.0f,load5:%.0f,load15:%.0f", loadAvg["load1"], loadAvg["load5"], loadAvg["load15"]) + err, ok = visitor.EventRuleEvaluate(loadAvg, *info.Rule) if err != nil { resultItem.Value = err.Error() + resultItem.Assert = true } } - if ok { + if ok || resultItem.Assert { resultItem.Level = info.Level + resultItem.Assert = true } - resultItem.Assert = ok - nodeInfoResult = append(nodeInfoResult, resultItem) } } @@ -182,11 +167,11 @@ func (o *nodeInfoInspect) GetResult(runNodeName string, resultCm *corev1.ConfigM } -func GetCpu(fs procfs.FS) float64 { +func GetCpu(fs procfs.FS) map[string]interface{} { stat, err := fs.Stat() if err != nil { klog.Error("failed to get pu info") - return 0 + return nil } totalUsage := 0.0 totalIdle := 0.0 @@ -195,47 +180,56 @@ func GetCpu(fs procfs.FS) float64 { totalIdle += cpuStat.Idle } usage := totalUsage / (totalUsage + totalIdle) - return math.Round(usage * 100) + return map[string]interface{}{constant.Cpu: usage * 100} } -func GetMemory(fs procfs.FS) float64 { +func GetMemory(fs procfs.FS) map[string]interface{} { meminfo, err := fs.Meminfo() if err != nil { klog.Error("failed to get meminfo") - return 0 + return nil } totalMemory := *meminfo.MemTotal freeMemory := *meminfo.MemFree + *meminfo.Buffers + *meminfo.Cached usedMemory := totalMemory - freeMemory memoryUsage := float64(usedMemory) / float64(totalMemory) - return math.Round(memoryUsage * 100) + return map[string]interface{}{constant.Memory: memoryUsage * 100} } -func GetLoadAvg(fs procfs.FS) (float64, float64, float64) { - +func GetLoadAvg(fs procfs.FS) map[string]interface{} { avg, err := fs.LoadAvg() if err != nil { klog.Error("failed to loadavg") - return 0, 0, 0 + return nil } - - return avg.Load1, avg.Load5, avg.Load15 + return map[string]interface{}{"load1": avg.Load1, "load5": avg.Load5, "load15": avg.Load15} } -func GetFileSystem(p string) (float64, float64) { +func GetFileSystem(p string) map[string]interface{} { u := new(unix.Statfs_t) err := unix.Statfs(path.Join(constant.RootPathPrefix, p), u) if err != nil { klog.Error("failed to get filesystem info") - return 0, 0 + return nil } total := float64(u.Blocks) * float64(u.Bsize) useBy := float64(u.Bavail) * float64(u.Bsize) storageUse := (total - useBy) / total + return map[string]interface{}{constant.Filesystem: storageUse * 100} +} + +func GetInodes(p string) map[string]interface{} { + u := new(unix.Statfs_t) + err := unix.Statfs(path.Join(constant.RootPathPrefix, p), u) + if err != nil { + klog.Error("failed to get filesystem info") + return nil + } + inodeUse := u.Files - u.Ffree inodeUseRate := float64(inodeUse) / float64(u.Files) - return math.Round(storageUse * 100), math.Round(inodeUseRate * 100) + return map[string]interface{}{constant.Inode: inodeUseRate * 100} } diff --git a/pkg/output/html.go b/pkg/output/html.go index fffaf0f2..378be370 100644 --- a/pkg/output/html.go +++ b/pkg/output/html.go @@ -68,6 +68,10 @@ func HtmlOut(resultName string) (error, map[string]interface{}) { if results.Spec.CommandResult != nil { resultCollection[constant.CustomCommand] = getCommand(results.Spec.CommandResult) + } + if results.Spec.NodeInfo != nil { + resultCollection[constant.NodeInfo] = getNodeInfo(results.Spec.NodeInfo) + } if results.Spec.ComponentResult != nil { @@ -219,6 +223,37 @@ func getSysctl(sysctlResult []v1alpha2.NodeMetricsResultItem) []renderNode { return villeinage } +func getNodeInfo(nodeInfo []v1alpha2.NodeInfoResultItem) []renderNode { + var villeinage []renderNode + header := renderNode{Header: true, + Children: []renderNode{ + {Text: "name"}, + {Text: "nodeName"}, + {Text: "resourcesType"}, + {Text: "mount"}, + {Text: "value"}, + }} + villeinage = append(villeinage, header) + + for _, item := range nodeInfo { + if item.Assert { + val := renderNode{ + Issues: item.Assert, + Children: []renderNode{ + {Text: item.Name}, + {Text: item.NodeName}, + {Text: item.ResourcesType.Type}, + {Text: item.ResourcesType.Mount}, + {Text: item.Value}, + }} + villeinage = append(villeinage, val) + } + + } + + return villeinage +} + func getSystemd(systemdResult []v1alpha2.NodeMetricsResultItem) []renderNode { var villeinage []renderNode header := renderNode{Header: true,