Skip to content

Commit

Permalink
feat: target add host tags (#2120)
Browse files Browse the repository at this point in the history 
* update target tags (#2091)

---------

Co-authored-by: flashbo <[email protected]>
  • Loading branch information
@710leo @lwb0214
710leo and lwb0214 authored Aug 26, 2024
1 parent 9ea5de1 commit 9c79233
Show file tree
Hide file tree
Showing 6 changed files with 102 additions and 33 deletions.
76 changes: 54 additions & 22 deletions center/router/router_heartbeat.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"encoding/json"
"errors"
"io/ioutil"
"sort"
"strings"
"time"

Expand Down Expand Up @@ -82,56 +83,87 @@ func HandleHeartbeat(c *gin.Context, ctx *ctx.Context, engineName string, metaSe
gid := ginx.QueryInt64(c, "gid", 0)
hostIp := strings.TrimSpace(req.HostIp)

field := make(map[string]interface{})
newTarget := models.Target{}
targetNeedUpdate := false
if gid != 0 && gid != target.GroupId {
field["group_id"] = gid
newTarget.GroupId = gid
targetNeedUpdate = true
}

if hostIp != "" && hostIp != target.HostIp {
field["host_ip"] = hostIp
newTarget.HostIp = hostIp
targetNeedUpdate = true
}

tagsMap := target.GetTagsMap()
tagNeedUpdate := false
for k, v := range req.GlobalLabels {
hostTagsMap := target.GetHostTagsMap()
hostTagNeedUpdate := false
if len(hostTagsMap) != len(req.GlobalLabels) {
hostTagNeedUpdate = true
} else {
for k, v := range req.GlobalLabels {
if v == "" {
continue
}

if tagv, ok := hostTagsMap[k]; !ok || tagv != v {
hostTagNeedUpdate = true
break
}
}
}

if hostTagNeedUpdate {
lst := []string{}
for k, v := range req.GlobalLabels {
lst = append(lst, k+"="+v)
}
sort.Strings(lst)
newTarget.HostTags = lst
targetNeedUpdate = true
}

userTagsMap := target.GetTagsMap()
userTagNeedUpdate := false
userTags := []string{}
for k, v := range userTagsMap {
if v == "" {
continue
}

if tagv, ok := tagsMap[k]; !ok || tagv != v {
tagNeedUpdate = true
tagsMap[k] = v
if _, ok := req.GlobalLabels[k]; !ok {
userTags = append(userTags, k+"="+v)
} else { // 该key在hostTags中已经存在
userTagNeedUpdate = true
}
}

if tagNeedUpdate {
lst := []string{}
for k, v := range tagsMap {
lst = append(lst, k+"="+v)
}
labels := strings.Join(lst, " ") + " "
field["tags"] = labels
if userTagNeedUpdate {
newTarget.Tags = strings.Join(userTags, " ") + " "
targetNeedUpdate = true
}

if req.EngineName != "" && req.EngineName != target.EngineName {
field["engine_name"] = req.EngineName
newTarget.EngineName = req.EngineName
targetNeedUpdate = true
}

if req.AgentVersion != "" && req.AgentVersion != target.AgentVersion {
field["agent_version"] = req.AgentVersion
newTarget.AgentVersion = req.AgentVersion
targetNeedUpdate = true
}

if req.OS != "" && req.OS != target.OS {
field["os"] = req.OS
newTarget.OS = req.OS
targetNeedUpdate = true
}

if len(field) > 0 {
err := target.UpdateFieldsMap(ctx, field)
if targetNeedUpdate {
err := models.DB(ctx).Model(&target).Updates(newTarget).Error
if err != nil {
logger.Errorf("update target fields failed, err: %v", err)
}
}
logger.Debugf("heartbeat field:%+v target: %v", field, *target)
logger.Debugf("heartbeat field:%+v target: %v", newTarget, *target)
}

return req, nil
Expand Down
4 changes: 3 additions & 1 deletion center/router/router_target.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -260,9 +260,11 @@ func (rt *Router) validateTags(tags []string) error {
}

func (rt *Router) addTagsToTarget(target *models.Target, tags []string) error {
hostTagsMap := target.GetHostTagsMap()
for _, tag := range tags {
tagKey := strings.Split(tag, "=")[0]
if strings.Contains(target.Tags, tagKey+"=") {
if _, ok := hostTagsMap[tagKey]; ok ||
strings.Contains(target.Tags, tagKey+"=") {
return fmt.Errorf("duplicate tagkey(%s)", tagKey)
}
}
Expand Down
6 changes: 5 additions & 1 deletion docker/migratesql/migrate.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,4 +99,8 @@ CREATE TABLE notification_record (
`details` VARCHAR(2048),
`created_at` BIGINT NOT NULL,
INDEX idx_evt (event_id)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;


/* v7.3.0 2024-08-26 */
ALTER TABLE `target` ADD COLUMN `host_tags` TEXT COMMENT 'global labels set in conf file';
2 changes: 2 additions & 0 deletions models/alert_rule.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -192,12 +192,14 @@ func GetHostsQuery(queries []HostQuery) []map[string]interface{} {
blank := " "
for _, tag := range lst {
m["tags like ?"+blank] = "%" + tag + "%"
m["host_tags like ?"+blank] = "%" + tag + "%"
blank += " "
}
} else {
blank := " "
for _, tag := range lst {
m["tags not like ?"+blank] = "%" + tag + "%"
m["host_tags not like ?"+blank] = "%" + tag + "%"
blank += " "
}
}
Expand Down
9 changes: 5 additions & 4 deletions models/migrate/migrate.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -227,10 +227,11 @@ type AlertCurEvent struct {
}

type Target struct {
HostIp string `gorm:"column:host_ip;varchar(15);default:'';comment:IPv4 string;index:idx_host_ip"`
AgentVersion string `gorm:"column:agent_version;varchar(255);default:'';comment:agent version;index:idx_agent_version"`
EngineName string `gorm:"column:engine_name;varchar(255);default:'';comment:engine name;index:idx_engine_name"`
OS string `gorm:"column:os;varchar(31);default:'';comment:os type;index:idx_os"`
HostIp string `gorm:"column:host_ip;type:varchar(15);default:'';comment:IPv4 string;index:idx_host_ip"`
AgentVersion string `gorm:"column:agent_version;type:varchar(255);default:'';comment:agent version;index:idx_agent_version"`
EngineName string `gorm:"column:engine_name;type:varchar(255);default:'';comment:engine name;index:idx_engine_name"`
OS string `gorm:"column:os;type:varchar(31);default:'';comment:os type;index:idx_os"`
HostTags []string `gorm:"column:host_tags;type:text;comment:global labels set in conf file;serializer:json"`
}

type Datasource struct {
Expand Down
38 changes: 33 additions & 5 deletions models/target.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (

"github.com/pkg/errors"
"github.com/toolkits/pkg/container/set"

"gorm.io/gorm"
)

Expand All @@ -19,14 +20,15 @@ type Target struct {
GroupObj *BusiGroup `json:"group_obj" gorm:"-"`
Ident string `json:"ident"`
Note string `json:"note"`
Tags string `json:"-"`
Tags string `json:"-"` // user tags
TagsJSON []string `json:"tags" gorm:"-"`
TagsMap map[string]string `json:"tags_maps" gorm:"-"` // internal use, append tags to series
UpdateAt int64 `json:"update_at"`
HostIp string `json:"host_ip"` //ipv4，do not needs range select
AgentVersion string `json:"agent_version"`
EngineName string `json:"engine_name"`
OS string `json:"os" gorm:"column:os"`
HostTags []string `json:"host_tags" gorm:"serializer:json"`

UnixTime int64 `json:"unixtime" gorm:"-"`
Offset int64 `json:"offset" gorm:"-"`
Expand Down Expand Up @@ -335,12 +337,12 @@ func TargetsGetIdentsByIdentsAndHostIps(ctx *ctx.Context, idents, hostIps []stri
func TargetGetTags(ctx *ctx.Context, idents []string) ([]string, error) {
session := DB(ctx).Model(new(Target))

var arr []string
var arr []*Target
if len(idents) > 0 {
session = session.Where("ident in ?", idents)
}

err := session.Select("distinct(tags) as tags").Pluck("tags", &arr).Error
err := session.Select("tags", "host_tags").Find(&arr).Error
if err != nil {
return nil, err
}
Expand All @@ -352,10 +354,13 @@ func TargetGetTags(ctx *ctx.Context, idents []string) ([]string, error) {

set := make(map[string]struct{})
for i := 0; i < cnt; i++ {
tags := strings.Fields(arr[i])
tags := strings.Fields(arr[i].Tags)
for j := 0; j < len(tags); j++ {
set[tags[j]] = struct{}{}
}
for _, ht := range arr[i].HostTags {
set[ht] = struct{}{}
}
}

cnt = len(set)
Expand Down Expand Up @@ -398,7 +403,18 @@ func (t *Target) DelTags(ctx *ctx.Context, tags []string) error {

func (t *Target) FillTagsMap() {
t.TagsJSON = strings.Fields(t.Tags)
t.TagsMap = t.GetTagsMap()
t.TagsMap = make(map[string]string)
m := make(map[string]string)
allTags := append(t.TagsJSON, t.HostTags...)
for _, item := range allTags {
arr := strings.Split(item, "=")
if len(arr) != 2 {
continue
}
m[arr[0]] = arr[1]
}

t.TagsMap = m
}

func (t *Target) GetTagsMap() map[string]string {
Expand All @@ -412,6 +428,18 @@ func (t *Target) GetTagsMap() map[string]string {
return m
}

func (t *Target) GetHostTagsMap() map[string]string {
m := make(map[string]string)
for _, item := range t.HostTags {
arr := strings.Split(item, "=")
if len(arr) != 2 {
continue
}
m[arr[0]] = arr[1]
}
return m
}

func (t *Target) FillMeta(meta *HostMeta) {
t.MemUtil = meta.MemUtil
t.CpuUtil = meta.CpuUtil
Expand Down

0 comments on commit 9c79233

Please sign in to comment.