Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add items to parse functionallity to the mappings #44

Merged
merged 5 commits into from
May 6, 2024
Merged

Add items to parse functionallity to the mappings #44

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
b0c81a9
add items to parse functionallity to the mappings
Apr 11, 2024
b6fcd86
add the item to the upper context instead of switch the context
Apr 14, 2024
55567a8
Merge branch 'main' of https://github.com/port-labs/port-k8s-exporter…
itamar-smirra-port Apr 25, 2024
b217def
revert last changes
itamar-smirra-port Apr 25, 2024
02877a2
insert the objects to map to array to remove the duplicate mapping code
itamar-smirra-port May 5, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 19 additions & 3 deletions pkg/jq/parser.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,12 @@ package jq

import (
"fmt"
"github.com/itchyny/gojq"
"k8s.io/klog/v2"
"os"
"strings"
"sync"

"github.com/itchyny/gojq"
"k8s.io/klog/v2"
)

var mutex = &sync.Mutex{}
Expand Down Expand Up @@ -66,7 +67,7 @@ func ParseString(jqQuery string, obj interface{}) (string, error) {

str, ok := queryRes.(string)
if !ok {
return "", fmt.Errorf("failed to parse string: %#v", queryRes)
return "", fmt.Errorf("failed to parse string with jq '%#v': %#v", jqQuery, queryRes)
}

return strings.Trim(str, "\""), nil
Expand All @@ -81,6 +82,21 @@ func ParseInterface(jqQuery string, obj interface{}) (interface{}, error) {
return queryRes, nil
}

func ParseArray(jqQuery string, obj interface{}) ([]interface{}, error) {
queryRes, err := runJQQuery(jqQuery, obj)

if err != nil {
return nil, err
}

items, ok := queryRes.([]interface{})
if !ok {
return nil, fmt.Errorf("failed to parse array with jq '%#v': %#v", jqQuery, queryRes)
}

return items, nil
}

func ParseMapInterface(jqQueries map[string]string, obj interface{}) (map[string]interface{}, error) {
mapInterface := make(map[string]interface{}, len(jqQueries))

Expand Down
85 changes: 67 additions & 18 deletions pkg/k8s/controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@ package k8s
import (
"context"
"fmt"
"time"

"github.com/port-labs/port-k8s-exporter/pkg/config"
"github.com/port-labs/port-k8s-exporter/pkg/jq"
"github.com/port-labs/port-k8s-exporter/pkg/port"
Expand All @@ -12,7 +14,6 @@ import (
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/informers"
"time"

utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/wait"
Expand Down Expand Up @@ -184,7 +185,7 @@ func (c *Controller) objectHandler(obj interface{}, item EventItem) error {

errors := make([]error, 0)
for _, kindConfig := range c.resource.KindConfigs {
portEntities, err := c.getObjectEntities(obj, kindConfig.Selector, kindConfig.Port.Entity.Mappings)
portEntities, err := c.getObjectEntities(obj, kindConfig.Selector, kindConfig.Port.Entity.Mappings, kindConfig.Port.ItemsToParse)
if err != nil {
utilruntime.HandleError(fmt.Errorf("error getting entities for object key '%s': %v", item.Key, err))
continue
Expand All @@ -205,7 +206,33 @@ func (c *Controller) objectHandler(obj interface{}, item EventItem) error {
return nil
}

func (c *Controller) getObjectEntities(obj interface{}, selector port.Selector, mappings []port.EntityMapping) ([]port.Entity, error) {
func isPassSelector(obj interface{}, selector port.Selector) (bool, error) {
if selector.Query == "" {
return true, nil
}

selectorResult, err := jq.ParseBool(selector.Query, obj)
if err != nil {
return false, fmt.Errorf("invalid selector query '%s': %v", selector.Query, err)
}

return selectorResult, err
}

func mapEntities(obj interface{}, mappings []port.EntityMapping) ([]port.Entity, error) {
entities := make([]port.Entity, 0, len(mappings))
for _, entityMapping := range mappings {
portEntity, err := mapping.NewEntity(obj, entityMapping)
if err != nil {
return nil, fmt.Errorf("invalid entity mapping '%#v': %v", entityMapping, err)
}
entities = append(entities, *portEntity)
}

return entities, nil
}

func (c *Controller) getObjectEntities(obj interface{}, selector port.Selector, mappings []port.EntityMapping, itemsToParse string) ([]port.Entity, error) {
unstructuredObj, ok := obj.(*unstructured.Unstructured)
if !ok {
return nil, fmt.Errorf("error casting to unstructured")
Expand All @@ -216,25 +243,47 @@ func (c *Controller) getObjectEntities(obj interface{}, selector port.Selector,
return nil, fmt.Errorf("error converting from unstructured: %v", err)
}

var selectorResult = true
if selector.Query != "" {
selectorResult, err = jq.ParseBool(selector.Query, structuredObj)
if err != nil {
return nil, fmt.Errorf("invalid selector query '%s': %v", selector.Query, err)
entities := make([]port.Entity, 0, len(mappings))
objectsToMap := make([]interface{}, 0)

if (itemsToParse == "") {
objectsToMap = append(objectsToMap, structuredObj)
} else {
items, parseItemsError := jq.ParseArray(itemsToParse, structuredObj)
if parseItemsError != nil {
return nil, parseItemsError
}

mappedObject, ok := structuredObj.(map[string]interface{})
if !ok {
return nil, fmt.Errorf("error parsing object '%#v'", structuredObj)
}

for _, item := range items {
copiedObject := make(map[string]interface{})
for key, value := range mappedObject {
copiedObject[key] = value
}
copiedObject["item"] = item
objectsToMap = append(objectsToMap, copiedObject)
}
}
if !selectorResult {
return nil, nil
}

for _, objectToMap := range objectsToMap {
selectorResult, err := isPassSelector(objectToMap, selector)

entities := make([]port.Entity, 0, len(mappings))
for _, entityMapping := range mappings {
var portEntity *port.Entity
portEntity, err = mapping.NewEntity(structuredObj, entityMapping)
if err != nil {
return nil, fmt.Errorf("invalid entity mapping '%#v': %v", entityMapping, err)
return nil, err
}

if selectorResult {
currentEntities, err := mapEntities(objectToMap, mappings)
if err != nil {
return nil, err
}

entities = append(entities, currentEntities...)
}
entities = append(entities, *portEntity)
}

return entities, nil
Expand Down Expand Up @@ -323,7 +372,7 @@ func (c *Controller) GetEntitiesSet() (map[string]interface{}, error) {
Blueprint: m.Blueprint,
})
}
entities, err := c.getObjectEntities(obj, kindConfig.Selector, mappings)
entities, err := c.getObjectEntities(obj, kindConfig.Selector, mappings, kindConfig.Port.ItemsToParse)
if err != nil {
return nil, fmt.Errorf("error getting entities of object: %v", err)
}
Expand Down
9 changes: 5 additions & 4 deletions pkg/k8s/controller_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,11 @@ package k8s
import (
"context"
"fmt"
"reflect"
"strings"
"testing"
"time"

"github.com/port-labs/port-k8s-exporter/pkg/config"
"github.com/port-labs/port-k8s-exporter/pkg/port"
"github.com/port-labs/port-k8s-exporter/pkg/port/cli"
Expand All @@ -11,10 +16,6 @@ import (
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
k8sfake "k8s.io/client-go/dynamic/fake"
"reflect"
"strings"
"testing"
"time"

appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
Expand Down
25 changes: 13 additions & 12 deletions pkg/port/models.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -179,31 +179,32 @@ type ResponseBody struct {
}

type EntityMapping struct {
Identifier string `json:"identifier"`
Title string `json:"title"`
Blueprint string `json:"blueprint"`
Icon string `json:"icon,omitempty"`
Team string `json:"team,omitempty"`
Properties map[string]string `json:"properties,omitempty"`
Relations map[string]string `json:"relations,omitempty"`
Identifier string `json:"identifier" yaml:"identifier"`
Title string `json:"title" yaml:"title"`
Blueprint string `json:"blueprint" yaml:"blueprint"`
Icon string `json:"icon,omitempty" yaml:"icon,omitempty"`
Team string `json:"team,omitempty" yaml:"team,omitempty"`
Properties map[string]string `json:"properties,omitempty" yaml:"properties,omitempty"`
Relations map[string]string `json:"relations,omitempty" yaml:"relations,omitempty"`
}

type EntityMappings struct {
Mappings []EntityMapping `json:"mappings"`
Mappings []EntityMapping `json:"mappings" yaml:"mappings"`
}

type Port struct {
Entity EntityMappings `json:"entity"`
Entity EntityMappings `json:"entity" yaml:"entity"`
ItemsToParse string `json:"itemsToParse" yaml:"itemsToParse"`
}

type Selector struct {
Query string
}

type Resource struct {
Kind string `json:"kind"`
Selector Selector `json:"selector,omitempty"`
Port Port `json:"port"`
Kind string `json:"kind" yaml:"kind"`
Selector Selector `json:"selector,omitempty" yaml:"selector,omitempty"`
Port Port `json:"port" yaml:"port"`
}

type EventListenerSettings struct {
Expand Down