From b0c81a9b3afbffadd5e91f854ce4d3699614fb5e Mon Sep 17 00:00:00 2001 From: Itamar Smirra Date: Thu, 11 Apr 2024 19:06:16 +0300 Subject: [PATCH 1/4] add items to parse functionallity to the mappings --- pkg/jq/parser.go | 22 +++++++++++++++++++--- pkg/k8s/controller.go | 35 ++++++++++++++++++++++++++--------- pkg/k8s/controller_test.go | 9 +++++---- pkg/port/models.go | 3 ++- 4 files changed, 52 insertions(+), 17 deletions(-) diff --git a/pkg/jq/parser.go b/pkg/jq/parser.go index 8a02b2c..bd3e3ee 100644 --- a/pkg/jq/parser.go +++ b/pkg/jq/parser.go @@ -2,11 +2,12 @@ package jq import ( "fmt" - "github.com/itchyny/gojq" - "k8s.io/klog/v2" "os" "strings" "sync" + + "github.com/itchyny/gojq" + "k8s.io/klog/v2" ) var mutex = &sync.Mutex{} @@ -66,7 +67,7 @@ func ParseString(jqQuery string, obj interface{}) (string, error) { str, ok := queryRes.(string) if !ok { - return "", fmt.Errorf("failed to parse string: %#v", queryRes) + return "", fmt.Errorf("failed to parse string with jq '%#v': %#v", jqQuery, queryRes) } return strings.Trim(str, "\""), nil @@ -81,6 +82,21 @@ func ParseInterface(jqQuery string, obj interface{}) (interface{}, error) { return queryRes, nil } +func ParseArray(jqQuery string, obj interface{}) ([]interface{}, error) { + queryRes, err := runJQQuery(jqQuery, obj) + + if err != nil { + return nil, err + } + + items, ok := queryRes.([]interface{}) + if !ok { + return nil, fmt.Errorf("failed to parse array with jq '%#v': %#v", jqQuery, queryRes) + } + + return items, nil +} + func ParseMapInterface(jqQueries map[string]string, obj interface{}) (map[string]interface{}, error) { mapInterface := make(map[string]interface{}, len(jqQueries)) diff --git a/pkg/k8s/controller.go b/pkg/k8s/controller.go index dde3813..35dc143 100644 --- a/pkg/k8s/controller.go +++ b/pkg/k8s/controller.go @@ -3,6 +3,8 @@ package k8s import ( "context" "fmt" + "time" + "github.com/port-labs/port-k8s-exporter/pkg/config" "github.com/port-labs/port-k8s-exporter/pkg/jq" "github.com/port-labs/port-k8s-exporter/pkg/port" @@ -12,7 +14,6 @@ import ( "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/informers" - "time" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/wait" @@ -184,7 +185,7 @@ func (c *Controller) objectHandler(obj interface{}, item EventItem) error { errors := make([]error, 0) for _, kindConfig := range c.resource.KindConfigs { - portEntities, err := c.getObjectEntities(obj, kindConfig.Selector, kindConfig.Port.Entity.Mappings) + portEntities, err := c.getObjectEntities(obj, kindConfig.Selector, kindConfig.Port.Entity.Mappings, kindConfig.Port.ItemsToParse) if err != nil { utilruntime.HandleError(fmt.Errorf("error getting entities for object key '%s': %v", item.Key, err)) continue @@ -205,7 +206,7 @@ func (c *Controller) objectHandler(obj interface{}, item EventItem) error { return nil } -func (c *Controller) getObjectEntities(obj interface{}, selector port.Selector, mappings []port.EntityMapping) ([]port.Entity, error) { +func (c *Controller) getObjectEntities(obj interface{}, selector port.Selector, mappings []port.EntityMapping, itemsToParse string) ([]port.Entity, error) { unstructuredObj, ok := obj.(*unstructured.Unstructured) if !ok { return nil, fmt.Errorf("error casting to unstructured") @@ -229,12 +230,28 @@ func (c *Controller) getObjectEntities(obj interface{}, selector port.Selector, entities := make([]port.Entity, 0, len(mappings)) for _, entityMapping := range mappings { - var portEntity *port.Entity - portEntity, err = mapping.NewEntity(structuredObj, entityMapping) - if err != nil { - return nil, fmt.Errorf("invalid entity mapping '%#v': %v", entityMapping, err) + if itemsToParse != "" { + items, parseItemsError := jq.ParseArray(itemsToParse, structuredObj) + if parseItemsError != nil { + return nil, parseItemsError + } else { + for _, item := range items { + var portEntity *port.Entity + portEntity, err = mapping.NewEntity(item, entityMapping) + if err != nil { + return nil, fmt.Errorf("invalid entity mapping '%#v': %v", entityMapping, err) + } + entities = append(entities, *portEntity) + } + } + } else { + var portEntity *port.Entity + portEntity, err = mapping.NewEntity(structuredObj, entityMapping) + if err != nil { + return nil, fmt.Errorf("invalid entity mapping '%#v': %v", entityMapping, err) + } + entities = append(entities, *portEntity) } - entities = append(entities, *portEntity) } return entities, nil @@ -323,7 +340,7 @@ func (c *Controller) GetEntitiesSet() (map[string]interface{}, error) { Blueprint: m.Blueprint, }) } - entities, err := c.getObjectEntities(obj, kindConfig.Selector, mappings) + entities, err := c.getObjectEntities(obj, kindConfig.Selector, mappings, kindConfig.Port.ItemsToParse) if err != nil { return nil, fmt.Errorf("error getting entities of object: %v", err) } diff --git a/pkg/k8s/controller_test.go b/pkg/k8s/controller_test.go index 7ab154c..4440789 100644 --- a/pkg/k8s/controller_test.go +++ b/pkg/k8s/controller_test.go @@ -3,6 +3,11 @@ package k8s import ( "context" "fmt" + "reflect" + "strings" + "testing" + "time" + "github.com/port-labs/port-k8s-exporter/pkg/config" "github.com/port-labs/port-k8s-exporter/pkg/port" "github.com/port-labs/port-k8s-exporter/pkg/port/cli" @@ -11,10 +16,6 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" k8sfake "k8s.io/client-go/dynamic/fake" - "reflect" - "strings" - "testing" - "time" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" diff --git a/pkg/port/models.go b/pkg/port/models.go index 661bbef..1438554 100644 --- a/pkg/port/models.go +++ b/pkg/port/models.go @@ -193,7 +193,8 @@ type EntityMappings struct { } type Port struct { - Entity EntityMappings `json:"entity"` + Entity EntityMappings `json:"entity"` + ItemsToParse string `json:"ItemsToParse"` } type Selector struct { From b6fcd8658b92d61276a8902bccba5bfc28169307 Mon Sep 17 00:00:00 2001 From: Itamar Smirra Date: Sun, 14 Apr 2024 14:47:43 +0300 Subject: [PATCH 2/4] add the item to the upper context instead of switch the context --- pkg/k8s/controller.go | 89 ++++++++++++++++++++++++++++++------------- 1 file changed, 62 insertions(+), 27 deletions(-) diff --git a/pkg/k8s/controller.go b/pkg/k8s/controller.go index 35dc143..a3219ab 100644 --- a/pkg/k8s/controller.go +++ b/pkg/k8s/controller.go @@ -206,6 +206,32 @@ func (c *Controller) objectHandler(obj interface{}, item EventItem) error { return nil } +func isPassSelector(obj interface{}, selector port.Selector) (bool, error) { + if selector.Query == "" { + return true, nil + } + + selectorResult, err := jq.ParseBool(selector.Query, obj) + if err != nil { + return false, fmt.Errorf("invalid selector query '%s': %v", selector.Query, err) + } + + return selectorResult, err +} + +func mapEntities(obj interface{}, mappings []port.EntityMapping) ([]port.Entity, error) { + entities := make([]port.Entity, 0, len(mappings)) + for _, entityMapping := range mappings { + portEntity, err := mapping.NewEntity(obj, entityMapping) + if err != nil { + return nil, fmt.Errorf("invalid entity mapping '%#v': %v", entityMapping, err) + } + entities = append(entities, *portEntity) + } + + return entities, nil +} + func (c *Controller) getObjectEntities(obj interface{}, selector port.Selector, mappings []port.EntityMapping, itemsToParse string) ([]port.Entity, error) { unstructuredObj, ok := obj.(*unstructured.Unstructured) if !ok { @@ -217,40 +243,49 @@ func (c *Controller) getObjectEntities(obj interface{}, selector port.Selector, return nil, fmt.Errorf("error converting from unstructured: %v", err) } - var selectorResult = true - if selector.Query != "" { - selectorResult, err = jq.ParseBool(selector.Query, structuredObj) - if err != nil { - return nil, fmt.Errorf("invalid selector query '%s': %v", selector.Query, err) - } - } - if !selectorResult { - return nil, nil - } - entities := make([]port.Entity, 0, len(mappings)) - for _, entityMapping := range mappings { - if itemsToParse != "" { - items, parseItemsError := jq.ParseArray(itemsToParse, structuredObj) - if parseItemsError != nil { - return nil, parseItemsError - } else { - for _, item := range items { - var portEntity *port.Entity - portEntity, err = mapping.NewEntity(item, entityMapping) + if itemsToParse != "" { + items, parseItemsError := jq.ParseArray(itemsToParse, structuredObj) + if parseItemsError != nil { + return nil, parseItemsError + } else { + editedObject, ok := structuredObj.(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("error parsing object '%#v'", structuredObj) + } + + for _, item := range items { + editedObject["item"] = item + selectorResult, err := isPassSelector(editedObject, selector) + + if err != nil { + return nil, err + } + + if selectorResult { + currentEntities, err := mapEntities(editedObject, mappings) if err != nil { - return nil, fmt.Errorf("invalid entity mapping '%#v': %v", entityMapping, err) + return nil, err } - entities = append(entities, *portEntity) + + entities = append(entities, currentEntities...) } } - } else { - var portEntity *port.Entity - portEntity, err = mapping.NewEntity(structuredObj, entityMapping) + } + } else { + selectorResult, err := isPassSelector(structuredObj, selector) + + if err != nil { + return nil, err + } + + if selectorResult { + currentEntities, err := mapEntities(structuredObj, mappings) if err != nil { - return nil, fmt.Errorf("invalid entity mapping '%#v': %v", entityMapping, err) + return nil, err } - entities = append(entities, *portEntity) + + entities = append(entities, currentEntities...) } } From b217defc1699f7ce7843ca69cc2becdc68df94cd Mon Sep 17 00:00:00 2001 From: Itamar Smirra Date: Thu, 25 Apr 2024 13:27:15 +0300 Subject: [PATCH 3/4] revert last changes --- pkg/k8s/controller.go | 31 +++++++++++++++---------------- pkg/port/models.go | 2 +- 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/pkg/k8s/controller.go b/pkg/k8s/controller.go index a3219ab..048334f 100644 --- a/pkg/k8s/controller.go +++ b/pkg/k8s/controller.go @@ -248,28 +248,27 @@ func (c *Controller) getObjectEntities(obj interface{}, selector port.Selector, items, parseItemsError := jq.ParseArray(itemsToParse, structuredObj) if parseItemsError != nil { return nil, parseItemsError - } else { - editedObject, ok := structuredObj.(map[string]interface{}) - if !ok { - return nil, fmt.Errorf("error parsing object '%#v'", structuredObj) - } + } + editedObject, ok := structuredObj.(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("error parsing object '%#v'", structuredObj) + } - for _, item := range items { - editedObject["item"] = item - selectorResult, err := isPassSelector(editedObject, selector) + for _, item := range items { + editedObject["item"] = item + selectorResult, err := isPassSelector(editedObject, selector) + if err != nil { + return nil, err + } + + if selectorResult { + currentEntities, err := mapEntities(editedObject, mappings) if err != nil { return nil, err } - if selectorResult { - currentEntities, err := mapEntities(editedObject, mappings) - if err != nil { - return nil, err - } - - entities = append(entities, currentEntities...) - } + entities = append(entities, currentEntities...) } } } else { diff --git a/pkg/port/models.go b/pkg/port/models.go index 024cd54..4d97b4e 100644 --- a/pkg/port/models.go +++ b/pkg/port/models.go @@ -194,7 +194,7 @@ type EntityMappings struct { type Port struct { Entity EntityMappings `json:"entity"` - ItemsToParse string `json:"ItemsToParse"` + ItemsToParse string `json:"itemsToParse"` } type Selector struct { From 02877a2040f91578d170d20471f8915e8cfa7f72 Mon Sep 17 00:00:00 2001 From: Itamar Smirra Date: Sun, 5 May 2024 17:30:56 +0300 Subject: [PATCH 4/4] insert the objects to map to array to remove the duplicate mapping code --- pkg/k8s/controller.go | 36 +++++++++++++++++------------------- pkg/port/models.go | 26 +++++++++++++------------- 2 files changed, 30 insertions(+), 32 deletions(-) diff --git a/pkg/k8s/controller.go b/pkg/k8s/controller.go index 048334f..4da895b 100644 --- a/pkg/k8s/controller.go +++ b/pkg/k8s/controller.go @@ -244,42 +244,40 @@ func (c *Controller) getObjectEntities(obj interface{}, selector port.Selector, } entities := make([]port.Entity, 0, len(mappings)) - if itemsToParse != "" { + objectsToMap := make([]interface{}, 0) + + if (itemsToParse == "") { + objectsToMap = append(objectsToMap, structuredObj) + } else { items, parseItemsError := jq.ParseArray(itemsToParse, structuredObj) if parseItemsError != nil { return nil, parseItemsError } - editedObject, ok := structuredObj.(map[string]interface{}) + + mappedObject, ok := structuredObj.(map[string]interface{}) if !ok { return nil, fmt.Errorf("error parsing object '%#v'", structuredObj) } for _, item := range items { - editedObject["item"] = item - selectorResult, err := isPassSelector(editedObject, selector) - - if err != nil { - return nil, err - } - - if selectorResult { - currentEntities, err := mapEntities(editedObject, mappings) - if err != nil { - return nil, err - } - - entities = append(entities, currentEntities...) + copiedObject := make(map[string]interface{}) + for key, value := range mappedObject { + copiedObject[key] = value } + copiedObject["item"] = item + objectsToMap = append(objectsToMap, copiedObject) } - } else { - selectorResult, err := isPassSelector(structuredObj, selector) + } + + for _, objectToMap := range objectsToMap { + selectorResult, err := isPassSelector(objectToMap, selector) if err != nil { return nil, err } if selectorResult { - currentEntities, err := mapEntities(structuredObj, mappings) + currentEntities, err := mapEntities(objectToMap, mappings) if err != nil { return nil, err } diff --git a/pkg/port/models.go b/pkg/port/models.go index 4d97b4e..722f7d9 100644 --- a/pkg/port/models.go +++ b/pkg/port/models.go @@ -179,22 +179,22 @@ type ResponseBody struct { } type EntityMapping struct { - Identifier string `json:"identifier"` - Title string `json:"title"` - Blueprint string `json:"blueprint"` - Icon string `json:"icon,omitempty"` - Team string `json:"team,omitempty"` - Properties map[string]string `json:"properties,omitempty"` - Relations map[string]string `json:"relations,omitempty"` + Identifier string `json:"identifier" yaml:"identifier"` + Title string `json:"title" yaml:"title"` + Blueprint string `json:"blueprint" yaml:"blueprint"` + Icon string `json:"icon,omitempty" yaml:"icon,omitempty"` + Team string `json:"team,omitempty" yaml:"team,omitempty"` + Properties map[string]string `json:"properties,omitempty" yaml:"properties,omitempty"` + Relations map[string]string `json:"relations,omitempty" yaml:"relations,omitempty"` } type EntityMappings struct { - Mappings []EntityMapping `json:"mappings"` + Mappings []EntityMapping `json:"mappings" yaml:"mappings"` } type Port struct { - Entity EntityMappings `json:"entity"` - ItemsToParse string `json:"itemsToParse"` + Entity EntityMappings `json:"entity" yaml:"entity"` + ItemsToParse string `json:"itemsToParse" yaml:"itemsToParse"` } type Selector struct { @@ -202,9 +202,9 @@ type Selector struct { } type Resource struct { - Kind string `json:"kind"` - Selector Selector `json:"selector,omitempty"` - Port Port `json:"port"` + Kind string `json:"kind" yaml:"kind"` + Selector Selector `json:"selector,omitempty" yaml:"selector,omitempty"` + Port Port `json:"port" yaml:"port"` } type EventListenerSettings struct {