Merge pull request #44 from port-labs/items-to-parse

Add items to parse functionallity to the mappings

pkg/jq/parser.go

@@ -2,11 +2,12 @@ package jq
 
 import (
 	"fmt"
-	"github.com/itchyny/gojq"
-	"k8s.io/klog/v2"
 	"os"
 	"strings"
 	"sync"
+
+	"github.com/itchyny/gojq"
+	"k8s.io/klog/v2"
 )
 
 var mutex = &sync.Mutex{}
@@ -66,7 +67,7 @@ func ParseString(jqQuery string, obj interface{}) (string, error) {
 
 	str, ok := queryRes.(string)
 	if !ok {
-		return "", fmt.Errorf("failed to parse string: %#v", queryRes)
+		return "", fmt.Errorf("failed to parse string with jq '%#v': %#v", jqQuery, queryRes)
 	}
 
 	return strings.Trim(str, "\""), nil
@@ -81,6 +82,21 @@ func ParseInterface(jqQuery string, obj interface{}) (interface{}, error) {
 	return queryRes, nil
 }
 
+func ParseArray(jqQuery string, obj interface{}) ([]interface{}, error) {
+	queryRes, err := runJQQuery(jqQuery, obj)
+
+	if err != nil {
+		return nil, err
+	}
+
+	items, ok := queryRes.([]interface{})
+	if !ok {
+		return nil, fmt.Errorf("failed to parse array with jq '%#v': %#v", jqQuery, queryRes)
+	}
+
+	return items, nil
+}
+
 func ParseMapInterface(jqQueries map[string]string, obj interface{}) (map[string]interface{}, error) {
 	mapInterface := make(map[string]interface{}, len(jqQueries))
 

pkg/k8s/controller.go

@@ -3,6 +3,8 @@ package k8s
 import (
 	"context"
 	"fmt"
+	"time"
+
 	"github.com/port-labs/port-k8s-exporter/pkg/config"
 	"github.com/port-labs/port-k8s-exporter/pkg/jq"
 	"github.com/port-labs/port-k8s-exporter/pkg/port"
@@ -12,7 +14,6 @@ import (
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/client-go/informers"
-	"time"
 
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/apimachinery/pkg/util/wait"
@@ -184,7 +185,7 @@ func (c *Controller) objectHandler(obj interface{}, item EventItem) error {
 
 	errors := make([]error, 0)
 	for _, kindConfig := range c.resource.KindConfigs {
-		portEntities, err := c.getObjectEntities(obj, kindConfig.Selector, kindConfig.Port.Entity.Mappings)
+		portEntities, err := c.getObjectEntities(obj, kindConfig.Selector, kindConfig.Port.Entity.Mappings, kindConfig.Port.ItemsToParse)
 		if err != nil {
 			utilruntime.HandleError(fmt.Errorf("error getting entities for object key '%s': %v", item.Key, err))
 			continue
@@ -205,7 +206,33 @@ func (c *Controller) objectHandler(obj interface{}, item EventItem) error {
 	return nil
 }
 
-func (c *Controller) getObjectEntities(obj interface{}, selector port.Selector, mappings []port.EntityMapping) ([]port.Entity, error) {
+func isPassSelector(obj interface{}, selector port.Selector) (bool, error) {
+	if selector.Query == "" {
+		return true, nil
+	}
+
+	selectorResult, err := jq.ParseBool(selector.Query, obj)
+	if err != nil {
+		return false, fmt.Errorf("invalid selector query '%s': %v", selector.Query, err)
+	}
+
+	return selectorResult, err
+}
+
+func mapEntities(obj interface{}, mappings []port.EntityMapping) ([]port.Entity, error) {
+	entities := make([]port.Entity, 0, len(mappings))
+	for _, entityMapping := range mappings {
+		portEntity, err := mapping.NewEntity(obj, entityMapping)
+		if err != nil {
+			return nil, fmt.Errorf("invalid entity mapping '%#v': %v", entityMapping, err)
+		}
+		entities = append(entities, *portEntity)
+	}
+
+	return entities, nil
+}
+
+func (c *Controller) getObjectEntities(obj interface{}, selector port.Selector, mappings []port.EntityMapping, itemsToParse string) ([]port.Entity, error) {
 	unstructuredObj, ok := obj.(*unstructured.Unstructured)
 	if !ok {
 		return nil, fmt.Errorf("error casting to unstructured")
@@ -216,25 +243,47 @@ func (c *Controller) getObjectEntities(obj interface{}, selector port.Selector,
 		return nil, fmt.Errorf("error converting from unstructured: %v", err)
 	}
 
-	var selectorResult = true
-	if selector.Query != "" {
-		selectorResult, err = jq.ParseBool(selector.Query, structuredObj)
-		if err != nil {
-			return nil, fmt.Errorf("invalid selector query '%s': %v", selector.Query, err)
+	entities := make([]port.Entity, 0, len(mappings))
+	objectsToMap := make([]interface{}, 0)
+
+	if (itemsToParse == "") {
+		objectsToMap = append(objectsToMap, structuredObj)
+	} else {
+		items, parseItemsError := jq.ParseArray(itemsToParse, structuredObj)
+		if parseItemsError != nil {
+			return nil, parseItemsError
+		}
+
+		mappedObject, ok := structuredObj.(map[string]interface{})
+		if !ok {
+			return nil, fmt.Errorf("error parsing object '%#v'", structuredObj)
+		}
+
+		for _, item := range items {
+			copiedObject := make(map[string]interface{})
+			for key, value := range mappedObject {
+				copiedObject[key] = value
+			}
+			copiedObject["item"] = item
+			objectsToMap = append(objectsToMap, copiedObject)
 		}
 	}
-	if !selectorResult {
-		return nil, nil
-	}
+
+	for _, objectToMap := range objectsToMap {
+		selectorResult, err := isPassSelector(objectToMap, selector)
 
-	entities := make([]port.Entity, 0, len(mappings))
-	for _, entityMapping := range mappings {
-		var portEntity *port.Entity
-		portEntity, err = mapping.NewEntity(structuredObj, entityMapping)
 		if err != nil {
-			return nil, fmt.Errorf("invalid entity mapping '%#v': %v", entityMapping, err)
+			return nil, err
+		}
+
+		if selectorResult {
+			currentEntities, err := mapEntities(objectToMap, mappings)
+			if err != nil {
+				return nil, err
+			}
+
+			entities = append(entities, currentEntities...)
 		}
-		entities = append(entities, *portEntity)
 	}
 
 	return entities, nil
@@ -323,7 +372,7 @@ func (c *Controller) GetEntitiesSet() (map[string]interface{}, error) {
 					Blueprint:  m.Blueprint,
 				})
 			}
-			entities, err := c.getObjectEntities(obj, kindConfig.Selector, mappings)
+			entities, err := c.getObjectEntities(obj, kindConfig.Selector, mappings, kindConfig.Port.ItemsToParse)
 			if err != nil {
 				return nil, fmt.Errorf("error getting entities of object: %v", err)
 			}

pkg/k8s/controller_test.go

@@ -3,6 +3,11 @@ package k8s
 import (
 	"context"
 	"fmt"
+	"reflect"
+	"strings"
+	"testing"
+	"time"
+
 	"github.com/port-labs/port-k8s-exporter/pkg/config"
 	"github.com/port-labs/port-k8s-exporter/pkg/port"
 	"github.com/port-labs/port-k8s-exporter/pkg/port/cli"
@@ -11,10 +16,6 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	k8sfake "k8s.io/client-go/dynamic/fake"
-	"reflect"
-	"strings"
-	"testing"
-	"time"
 
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"

pkg/port/models.go

@@ -179,31 +179,32 @@ type ResponseBody struct {
 }
 
 type EntityMapping struct {
-	Identifier string            `json:"identifier"`
-	Title      string            `json:"title"`
-	Blueprint  string            `json:"blueprint"`
-	Icon       string            `json:"icon,omitempty"`
-	Team       string            `json:"team,omitempty"`
-	Properties map[string]string `json:"properties,omitempty"`
-	Relations  map[string]string `json:"relations,omitempty"`
+	Identifier string            `json:"identifier" yaml:"identifier"`
+	Title      string            `json:"title" yaml:"title"`
+	Blueprint  string            `json:"blueprint" yaml:"blueprint"`
+	Icon       string            `json:"icon,omitempty" yaml:"icon,omitempty"`
+	Team       string            `json:"team,omitempty" yaml:"team,omitempty"`
+	Properties map[string]string `json:"properties,omitempty" yaml:"properties,omitempty"`
+	Relations  map[string]string `json:"relations,omitempty" yaml:"relations,omitempty"`
 }
 
 type EntityMappings struct {
-	Mappings []EntityMapping `json:"mappings"`
+	Mappings []EntityMapping `json:"mappings" yaml:"mappings"`
 }
 
 type Port struct {
-	Entity EntityMappings `json:"entity"`
+	Entity       EntityMappings `json:"entity" yaml:"entity"`
+	ItemsToParse string         `json:"itemsToParse" yaml:"itemsToParse"`
 }
 
 type Selector struct {
 	Query string
 }
 
 type Resource struct {
-	Kind     string   `json:"kind"`
-	Selector Selector `json:"selector,omitempty"`
-	Port     Port     `json:"port"`
+	Kind     string   `json:"kind" yaml:"kind"`
+	Selector Selector `json:"selector,omitempty" yaml:"selector,omitempty"`
+	Port     Port     `json:"port" yaml:"port"`
 }
 
 type EventListenerSettings struct {