From c683dd0d880981f3e076a19c6126a7776e80d88a Mon Sep 17 00:00:00 2001 From: CeerDecy <1748788674@qq.com> Date: Wed, 15 Jan 2025 15:51:11 +0800 Subject: [PATCH] feat: sync general monitor config --- ...115-sp_monitor_config_register-add-log.sql | 1 + .../common/db/monitor_config_register.go | 18 +++++++++++ .../tools/monitor/core/settings/monitor.go | 25 ++++++++++++++-- .../retention-strategy/routers_test.go | 30 +++++++++++++++++++ .../monitor/core/settings/settings.service.go | 3 ++ .../monitor/core/settings/sync_config.go | 30 +++++++++++++------ 6 files changed, 96 insertions(+), 11 deletions(-) create mode 100644 .erda/migrations/monitor/20250115-sp_monitor_config_register-add-log.sql create mode 100644 internal/tools/monitor/core/settings/retention-strategy/routers_test.go diff --git a/.erda/migrations/monitor/20250115-sp_monitor_config_register-add-log.sql b/.erda/migrations/monitor/20250115-sp_monitor_config_register-add-log.sql new file mode 100644 index 00000000000..ac764dc24b4 --- /dev/null +++ b/.erda/migrations/monitor/20250115-sp_monitor_config_register-add-log.sql @@ -0,0 +1 @@ +INSERT INTO `sp_monitor_config_register` (`scope`, `scope_id`, `namespace`, `type`, `names`, `filters`, `enable`, `update_time`, `desc`, `hash`) VALUES ('org','','','log','*','[]',1,'2025-01-15 00:00:00','','log_common') \ No newline at end of file diff --git a/internal/tools/monitor/common/db/monitor_config_register.go b/internal/tools/monitor/common/db/monitor_config_register.go index 59e92374489..775d120ad20 100644 --- a/internal/tools/monitor/common/db/monitor_config_register.go +++ b/internal/tools/monitor/common/db/monitor_config_register.go @@ -16,6 +16,7 @@ package db import ( "errors" + "fmt" "github.com/jinzhu/gorm" ) @@ -49,3 +50,20 @@ func (m *MonitorConfigRegisterDB) ListRegisterByType(tpy string) ([]SpMonitorCon } return res, nil } + +func (m *MonitorConfigRegisterDB) ListRegisterWithFilter(filters map[string]any) ([]SpMonitorConfigRegister, error) { + var res = make([]SpMonitorConfigRegister, 0) + tx := m.Model(&SpMonitorConfigRegister{}) + for k, v := range filters { + key := k + value := v + tx = tx.Where(fmt.Sprintf("`%s` = ?", key), value) + } + if err := tx.Find(&res).Error; err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return make([]SpMonitorConfigRegister, 0), nil + } + return nil, err + } + return res, nil +} diff --git a/internal/tools/monitor/core/settings/monitor.go b/internal/tools/monitor/core/settings/monitor.go index 3581213e628..ff8e8e567ac 100644 --- a/internal/tools/monitor/core/settings/monitor.go +++ b/internal/tools/monitor/core/settings/monitor.go @@ -76,6 +76,24 @@ func (s *settingsService) monitorConfigMap(ns string) *configDefine { if ns == "general" { cd.defaults = map[string]func(langs i18n.LanguageCodes) *pb.ConfigItem{ + "logs_ttl": func(langs i18n.LanguageCodes) *pb.ConfigItem { + return &pb.ConfigItem{ + Key: "logs_ttl", + Name: s.t.Text(langs, "base") + " " + s.t.Text(langs, "logs_ttl"), + Type: "number", + Value: structpb.NewNumberValue(float64(log.TTL)), + Unit: s.t.Text(langs, "days"), + } + }, + "logs_hot_ttl": func(langs i18n.LanguageCodes) *pb.ConfigItem { + return &pb.ConfigItem{ + Key: "logs_hot_ttl", + Name: s.t.Text(langs, "base") + " " + s.t.Text(langs, "logs_hot_ttl"), + Type: "number", + Value: structpb.NewNumberValue(float64(log.HotTTL)), + Unit: s.t.Text(langs, "days"), + } + }, "metrics_ttl": func(langs i18n.LanguageCodes) *pb.ConfigItem { return &pb.ConfigItem{ Key: "metrics_ttl", @@ -186,7 +204,10 @@ type monitorConfigRegister struct { } func (s *settingsService) generateKey(orgID, ns string) string { - return md5x.SumString(orgID + "/" + ns).String16() + if ns != "" { + return md5x.SumString(orgID + "/" + ns).String16() + } + return md5x.SumString(orgID).String16() } func (s *settingsService) updateMonitorConfig(tx *gorm.DB, orgid int64, orgName, ns, group string, keys map[string]interface{}) error { @@ -194,7 +215,7 @@ func (s *settingsService) updateMonitorConfig(tx *gorm.DB, orgid int64, orgName, ns = "" } orgID := strconv.FormatInt(orgid, 10) - key := md5x.SumString(orgID + "/" + ns).String16() + key := s.generateKey(orgID, ns) ttl := ttl{} diff --git a/internal/tools/monitor/core/settings/retention-strategy/routers_test.go b/internal/tools/monitor/core/settings/retention-strategy/routers_test.go new file mode 100644 index 00000000000..d4ca090d555 --- /dev/null +++ b/internal/tools/monitor/core/settings/retention-strategy/routers_test.go @@ -0,0 +1,30 @@ +package retention + +import ( + "fmt" + "github.com/erda-project/erda/pkg/router" + "testing" +) + +func TestRouter(t *testing.T) { + matcher := router.New() + matcher.Add("application_*", []*router.KeyValue{ + { + Key: "application_key", + Value: "application", + }, + }, "application") + matcher.Add("*", []*router.KeyValue{ + { + Key: "terminus_key", + Value: "terminus", + }, + }, "") + + find := matcher.Find("applicatdfsdfffion_sss*", map[string]string{ + "terminus_key": "terminus", + //"terminus_value": "123", + }) + + fmt.Println(find) +} diff --git a/internal/tools/monitor/core/settings/settings.service.go b/internal/tools/monitor/core/settings/settings.service.go index 03294ca82b3..21af0850a0d 100644 --- a/internal/tools/monitor/core/settings/settings.service.go +++ b/internal/tools/monitor/core/settings/settings.service.go @@ -65,6 +65,9 @@ type settingsService struct { } func (s *settingsService) PutSettingsWithType(ctx context.Context, req *pb.PutSettingsWithTypeRequest) (*pb.PutSettingsWithTypeResponse, error) { + if req.Namespace == "general" { + req.Namespace = "" + } orgName, err := s.getOrgName(req.OrgID) if err != nil { return nil, err diff --git a/internal/tools/monitor/core/settings/sync_config.go b/internal/tools/monitor/core/settings/sync_config.go index 963df092560..94308461139 100644 --- a/internal/tools/monitor/core/settings/sync_config.go +++ b/internal/tools/monitor/core/settings/sync_config.go @@ -51,17 +51,29 @@ func (p *provider) syncCreateOrgMonitorConfig() error { p.Log.Errorf("failed to get monitor config register by type: log, err: %v", err) return err } - registers = make([]db.SpMonitorConfigRegister, 0, len(logRegisters)+len(monitorRegisters)) + + generalRegisters, err := client.MonitorConfigRegister.ListRegisterWithFilter(map[string]any{ + "scope": "org", + "scope_id": "", + "namespace": "", + }) + registers = make([]db.SpMonitorConfigRegister, 0, len(logRegisters)+len(monitorRegisters)+len(generalRegisters)) registers = append(registers, monitorRegisters...) registers = append(registers, logRegisters...) + registers = append(registers, generalRegisters...) for _, register := range registers { if !p.isEmptyConfig(®ister, org) { continue } - nsConfig := defaultConfig[register.Namespace] - defConfig := nsConfig["monitor"] + var nsConfig map[string]map[string]*pb.ConfigItem + if register.Namespace == "" { + nsConfig = defaultConfig["general"] + } else { + nsConfig = defaultConfig[register.Namespace] + } + monitorConfig := nsConfig["monitor"] req := &pb.PutSettingsWithTypeRequest{ OrgID: int64(org.ID), @@ -78,20 +90,20 @@ func (p *provider) syncCreateOrgMonitorConfig() error { switch register.Type { case "log": - ttlItem = defConfig[LogsTTLKey] - hotTTLItem = defConfig[LogsHotTTLKey] + ttlItem = monitorConfig[LogsTTLKey] + hotTTLItem = monitorConfig[LogsHotTTLKey] case "metric": - ttlItem = defConfig[MetricsTTLKey] - hotTTLItem = defConfig[MetricsHotTTLKey] + ttlItem = monitorConfig[MetricsTTLKey] + hotTTLItem = monitorConfig[MetricsHotTTLKey] } if ttlItem == nil { - err = fmt.Errorf("ttl item is nil, monitor type: %s", register.Type) + err = fmt.Errorf("ttl item is nil, monitor type: %v", register) p.Log.Error(err) return err } if hotTTLItem == nil { - err = fmt.Errorf("hot_ttl item is nil, monitor type: %s", register.Type) + err = fmt.Errorf("hot_ttl item is nil, monitor type: %v", register) p.Log.Error(err) return err }