From 9c2efe318e05dee46a1f2255867d82bc20cda701 Mon Sep 17 00:00:00 2001 From: yair Date: Wed, 13 Dec 2023 11:15:03 +0200 Subject: [PATCH 01/32] default resources --- assets/defaults/appConfig.yaml | 119 ++++++++++++++++ assets/defaults/blueprints.json | 151 ++++++++++++++++++++ assets/defaults/scorecards.json | 139 ++++++++++++++++++ main.go | 8 +- pkg/config/config.go | 10 +- pkg/defaults/defaults.go | 229 ++++++++++++++++++++++++++++++ pkg/port/blueprint/blueprint.go | 51 +++++++ pkg/port/models.go | 8 ++ pkg/port/scorecards/scorecards.go | 22 +++ 9 files changed, 735 insertions(+), 2 deletions(-) create mode 100644 assets/defaults/appConfig.yaml create mode 100644 assets/defaults/blueprints.json create mode 100644 assets/defaults/scorecards.json create mode 100644 pkg/defaults/defaults.go create mode 100644 pkg/port/blueprint/blueprint.go create mode 100644 pkg/port/scorecards/scorecards.go diff --git a/assets/defaults/appConfig.yaml b/assets/defaults/appConfig.yaml new file mode 100644 index 0000000..2d6e358 --- /dev/null +++ b/assets/defaults/appConfig.yaml @@ -0,0 +1,119 @@ +createMissingRelatedEntities: true +resources: + - kind: v1/namespaces + port: + entity: + mappings: + - blueprint: '"namespace"' + identifier: .metadata.name + "-" + env.CLUSTER_NAME + properties: + creationTimestamp: .metadata.creationTimestamp + labels: .metadata.labels + relations: + Cluster: env.CLUSTER_NAME + title: .metadata.name + selector: + query: .metadata.name | startswith("kube") | not + - kind: v1/namespaces + port: + entity: + mappings: + - blueprint: '"cluster"' + identifier: env.CLUSTER_NAME + title: env.CLUSTER_NAME + selector: + query: .metadata.name | contains("kube-system") + - kind: apps/v1/deployments + port: + entity: + mappings: + - blueprint: '"workload"' + icon: '"Deployment"' + identifier: .metadata.name + "-Deployment-" + .metadata.namespace + "-" + env.CLUSTER_NAME + properties: + images: '(.spec.template.spec.containers | map({name, image, resources})) | map("\(.name): \(.image)")' + availableReplicas: .status.availableReplicas + containers: (.spec.template.spec.containers | map({name, image, resources})) + creationTimestamp: .metadata.creationTimestamp + hasLatest: .spec.template.spec.containers[].image | contains(":latest") + hasLimits: .spec.template.spec.containers | all(has("resources") and (.resources.limits.memory + and .resources.limits.cpu)) + hasPrivileged: .spec.template.spec.containers | [.[].securityContext.privileged] + | any + isHealthy: if .spec.replicas == .status.availableReplicas then "Healthy" + else "Unhealthy" end + kind: '"Deployment"' + labels: .metadata.labels + replicas: .spec.replicas + strategy: .spec.strategy.type + strategyConfig: .spec.strategy // {} + relations: + Namespace: .metadata.namespace + "-" + env.CLUSTER_NAME + title: .metadata.name + selector: + query: .metadata.namespace | startswith("kube") | not + - kind: apps/v1/daemonsets + port: + entity: + mappings: + - blueprint: '"workload"' + identifier: .metadata.name + "-DaemonSet-" + .metadata.namespace + "-" + env.CLUSTER_NAME + properties: + availableReplicas: .status.availableReplicas + containers: (.spec.template.spec.containers | map({name, image, resources})) + creationTimestamp: .metadata.creationTimestamp + hasLatest: .spec.template.spec.containers[].image | contains(":latest") + hasLimits: .spec.template.spec.containers | all(has("resources") and (.resources.limits.memory + and .resources.limits.cpu)) + hasPrivileged: .spec.template.spec.containers | [.[].securityContext.privileged] + | any + isHealthy: if .spec.replicas == .status.availableReplicas then "Healthy" + else "Unhealthy" end + kind: '"DaemonSet"' + labels: .metadata.labels + replicas: .spec.replicas + strategyConfig: .spec.strategy // {} + relations: + Namespace: .metadata.namespace + "-" + env.CLUSTER_NAME + title: .metadata.name + selector: + query: .metadata.namespace | startswith("kube") | not + - kind: apps/v1/statefulsets + port: + entity: + mappings: + - blueprint: '"workload"' + identifier: .metadata.name + "-StatefulSet-" + .metadata.namespace + "-" + env.CLUSTER_NAME + properties: + availableReplicas: .status.availableReplicas + containers: (.spec.template.spec.containers | map({name, image, resources})) + creationTimestamp: .metadata.creationTimestamp + hasLatest: .spec.template.spec.containers[].image | contains(":latest") + hasLimits: .spec.template.spec.containers | all(has("resources") and (.resources.limits.memory + and .resources.limits.cpu)) + hasPrivileged: .spec.template.spec.containers | [.[].securityContext.privileged] + | any + isHealthy: if .spec.replicas == .status.availableReplicas then "Healthy" + else "Unhealthy" end + kind: '"StatefulSet"' + labels: .metadata.labels + replicas: .spec.replicas + strategyConfig: .spec.strategy // {} + relations: + Namespace: .metadata.namespace + "-" + env.CLUSTER_NAME + title: .metadata.name + selector: + query: .metadata.namespace | startswith("kube") | not + - kind: apps/v1/deployments + port: + entity: + mappings: + - blueprint: '"service"' + icon: '"Deployment"' + identifier: .metadata.labels.portService + properties: {} + relations: + prod_runtime: .metadata.name + "-Deployment-" + .metadata.namespace + "-" + env.CLUSTER_NAME + title: .metadata.name + selector: + query: .metadata.namespace | startswith("kube") | not \ No newline at end of file diff --git a/assets/defaults/blueprints.json b/assets/defaults/blueprints.json new file mode 100644 index 0000000..3e7fca3 --- /dev/null +++ b/assets/defaults/blueprints.json @@ -0,0 +1,151 @@ +[ + { + "identifier": "cluster", + "description": "This blueprint represents a Kubernetes Cluster", + "title": "Cluster", + "icon": "Cluster", + "schema": { + "properties": {}, + "required": [] + }, + "mirrorProperties": {}, + "calculationProperties": {}, + "relations": {} + }, + { + "identifier": "namespace", + "description": "This blueprint represents a k8s Namespace", + "title": "Namespace", + "icon": "Environment", + "schema": { + "properties": { + "creationTimestamp": { + "type": "string", + "title": "Created", + "format": "date-time", + "description": "When the Namespace was created" + }, + "labels": { + "type": "object", + "title": "Labels", + "description": "Labels of the Namespace" + } + }, + "required": [] + }, + "mirrorProperties": {}, + "calculationProperties": {}, + "relations": { + "Cluster": { + "title": "Cluster", + "description": "The namespace's Kubernetes cluster", + "target": "cluster", + "required": false, + "many": false + } + } + }, + { + "identifier": "workload", + "description": "This blueprint represents a k8s Workload. This includes all k8s objects which can create pods (deployments[replicasets], daemonsets, statefulsets...)", + "title": "Workload", + "icon": "Deployment", + "schema": { + "properties": { + "availableReplicas": { + "type": "number", + "title": "Running Replicas", + "description": "Current running replica count" + }, + "containers": { + "type": "array", + "title": "Containers", + "default": [], + "description": "The containers for each pod instance of the Workload" + }, + "creationTimestamp": { + "type": "string", + "title": "Created", + "format": "date-time", + "description": "When the Workload was created" + }, + "labels": { + "type": "object", + "title": "Labels", + "description": "Labels of the Workload" + }, + "replicas": { + "type": "number", + "title": "Wanted Replicas", + "description": "Wanted replica count" + }, + "strategy": { + "type": "string", + "title": "Strategy", + "description": "Rollout Strategy" + }, + "hasPrivileged": { + "type": "boolean", + "title": "Has Privileged Container" + }, + "hasLatest": { + "type": "boolean", + "title": "Has 'latest' tag", + "description": "Has Container with 'latest' as image tag" + }, + "hasLimits": { + "type": "boolean", + "title": "All containers have limits" + }, + "isHealthy": { + "type": "string", + "enum": [ + "Healthy", + "Unhealthy" + ], + "enumColors": { + "Healthy": "green", + "Unhealthy": "red" + }, + "title": "Workload Health" + }, + "kind": { + "title": "Workload Kind", + "description": "The kind of Workload", + "type": "string", + "enum": [ + "StatefulSet", + "DaemonSet", + "Deployment", + "Rollout" + ] + }, + "strategyConfig": { + "type": "object", + "title": "Strategy Config", + "description": "The workloads rollout strategy" + } + }, + "required": [] + }, + "mirrorProperties": { + "Cluster": { + "title": "Cluster", + "path": "Namespace.Cluster.$title" + }, + "namespace": { + "title": "Namespace", + "path": "Namespace.$title" + } + }, + "calculationProperties": {}, + "relations": { + "Namespace": { + "title": "Namespace", + "target": "namespace", + "required": false, + "many": false + } + } + } +] \ No newline at end of file diff --git a/assets/defaults/scorecards.json b/assets/defaults/scorecards.json new file mode 100644 index 0000000..f5bc2da --- /dev/null +++ b/assets/defaults/scorecards.json @@ -0,0 +1,139 @@ +[ + { + "blueprint": "workload", + "data": [ + { + "identifier": "configuration", + "title": "Configuration Checks", + "rules": [ + { + "identifier": "notPrivileged", + "title": "No privilged containers", + "level": "Bronze", + "query": { + "combinator": "and", + "conditions": [ + { + "property": "hasPrivileged", + "operator": "!=", + "value": true + } + ] + } + }, + { + "identifier": "hasLimits", + "title": "All containers have CPU and Memory limits", + "level": "Bronze", + "query": { + "combinator": "and", + "conditions": [ + { + "property": "hasLimits", + "operator": "=", + "value": true + } + ] + } + }, + { + "identifier": "notDefaultNamespace", + "title": "Not in 'default' namespace", + "level": "Bronze", + "query": { + "combinator": "and", + "conditions": [ + { + "property": "namespace", + "operator": "!=", + "value": "default" + } + ] + } + }, + { + "identifier": "rolloutStrategy", + "title": "Using Rolling update strategy", + "level": "Silver", + "query": { + "combinator": "and", + "conditions": [ + { + "property": "strategy", + "operator": "=", + "value": "RollingUpdate" + } + ] + } + }, + { + "identifier": "imageTag", + "title": "Doesn't have a container with image tag 'latest'", + "level": "Gold", + "query": { + "combinator": "and", + "conditions": [ + { + "property": "hasLatest", + "operator": "!=", + "value": "false" + } + ] + } + } + ] + }, + { + "identifier": "highAvailability", + "title": "High Availability", + "rules": [ + { + "identifier": "highAvalabilityB", + "title": "Highly Available", + "level": "Bronze", + "query": { + "combinator": "and", + "conditions": [ + { + "property": "replicas", + "operator": ">=", + "value": 1 + } + ] + } + }, + { + "identifier": "highAvalabilityS", + "title": "Highly Available", + "level": "Silver", + "query": { + "combinator": "and", + "conditions": [ + { + "property": "replicas", + "operator": ">=", + "value": 2 + } + ] + } + }, + { + "identifier": "highAvalabilityG", + "title": "Highly Available", + "level": "Gold", + "query": { + "combinator": "and", + "conditions": [ + { + "property": "replicas", + "operator": ">=", + "value": 3 + } + ] + } + } + ] + } + ] + } +] \ No newline at end of file diff --git a/main.go b/main.go index a99962c..834aaec 100644 --- a/main.go +++ b/main.go @@ -67,7 +67,13 @@ func main() { } exporterConfig, err := config.New(configFilePath, resyncInterval, stateKey, eventListenerType) - if err != nil { + var fileNotFoundError *config.FileNotFoundError + if errors.As(err, &fileNotFoundError) { + err := defaults.InitializeDefaults(portClient, exporterConfig) + if err != nil { + klog.Warningf("Error initializing defaults: %s", err.Error()) + } + } else if err != nil { klog.Fatalf("Error building Port K8s Exporter config: %s", err.Error()) } diff --git a/pkg/config/config.go b/pkg/config/config.go index 5f0356b..0cd03a3 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -7,6 +7,14 @@ import ( "gopkg.in/yaml.v2" ) +type FileNotFoundError struct { + s string +} + +func (e *FileNotFoundError) Error() string { + return e.s +} + func New(filepath string, resyncInterval uint, stateKey string, eventListenerType string) (*port.Config, error) { c := &port.Config{ ResyncInterval: resyncInterval, @@ -15,7 +23,7 @@ func New(filepath string, resyncInterval uint, stateKey string, eventListenerTyp } config, err := os.ReadFile(filepath) if err != nil { - return nil, err + return c, &FileNotFoundError{err.Error()} } err = yaml.Unmarshal(config, c) diff --git a/pkg/defaults/defaults.go b/pkg/defaults/defaults.go new file mode 100644 index 0000000..d5e9a47 --- /dev/null +++ b/pkg/defaults/defaults.go @@ -0,0 +1,229 @@ +package defaults + +import ( + "encoding/json" + "fmt" + "github.com/port-labs/port-k8s-exporter/pkg/port" + "github.com/port-labs/port-k8s-exporter/pkg/port/blueprint" + "github.com/port-labs/port-k8s-exporter/pkg/port/cli" + "github.com/port-labs/port-k8s-exporter/pkg/port/integration" + "github.com/port-labs/port-k8s-exporter/pkg/port/scorecards" + "gopkg.in/yaml.v3" + "k8s.io/klog/v2" + "log" + "os" + "sync" +) + +type ScorecardDefault struct { + Blueprint string `json:"blueprint"` + Scorecards []port.Scorecard `json:"data"` +} + +type Defaults struct { + Blueprints []port.Blueprint + Scorecards []ScorecardDefault + AppConfig *port.AppConfig +} + +func getDefaults() (*Defaults, error) { + var bp []port.Blueprint + file, err := os.ReadFile("assets/defaults/blueprints.json") + if err != nil { + klog.Infof("No default blueprints found. Skipping...") + } else { + err = json.Unmarshal(file, &bp) + if err != nil { + return nil, err + } + } + + var sc []ScorecardDefault + file, err = os.ReadFile("./assets/defaults/scorecards.json") + if err != nil { + klog.Infof("No default scorecards found. Skipping...") + } else { + err = json.Unmarshal(file, &sc) + if err != nil { + return nil, err + } + } + + var appConfig *port.AppConfig + file, err = os.ReadFile("./assets/defaults/appConfig.yaml") + if err != nil { + klog.Infof("No default appConfig found. Skipping...") + } else { + err = yaml.Unmarshal(file, &appConfig) + if err != nil { + return nil, err + } + } + + return &Defaults{ + Blueprints: bp, + Scorecards: sc, + AppConfig: appConfig, + }, nil +} + +func deconstructBlueprintsToCreationSteps(rawBlueprints []port.Blueprint) ([]port.Blueprint, [][]port.Blueprint) { + var ( + bareBlueprints []port.Blueprint + withRelations []port.Blueprint + fullBlueprints []port.Blueprint + ) + + for _, bp := range append([]port.Blueprint{}, rawBlueprints...) { + bareBlueprint := port.Blueprint{ + Identifier: bp.Identifier, + Title: bp.Title, + Icon: bp.Icon, + Description: bp.Description, + Schema: bp.Schema, + } + bareBlueprints = append(bareBlueprints, bareBlueprint) + + withRelation := bareBlueprint + withRelation.Relations = bp.Relations + withRelations = append(withRelations, withRelation) + + fullBlueprint := withRelation + fullBlueprint.FormulaProperties = bp.FormulaProperties + fullBlueprint.MirrorProperties = bp.MirrorProperties + fullBlueprints = append(fullBlueprints, fullBlueprint) + } + + return bareBlueprints, [][]port.Blueprint{withRelations, fullBlueprints} +} + +type AbortDefaultCreationError struct { + BlueprintsToRollback []string + Errors []error +} + +func (e *AbortDefaultCreationError) Error() string { + return "AbortDefaultCreationError" +} + +func validateBlueprintErrors(createdBlueprints []string, blueprintErrors []error) *AbortDefaultCreationError { + if len(blueprintErrors) > 0 { + for _, err := range blueprintErrors { + log.Printf("Failed to create resources: %v.", err.Error()) + } + return &AbortDefaultCreationError{BlueprintsToRollback: createdBlueprints, Errors: blueprintErrors} + } + return nil +} + +func CreateResources(portClient *cli.PortClient, defaults *Defaults, config *port.Config) *AbortDefaultCreationError { + if _, err := integration.GetIntegrationConfig(portClient, config.StateKey); err == nil { + log.Println("Integration already exists. Skipping...") + return nil + } + + bareBlueprints, patchStages := deconstructBlueprintsToCreationSteps(defaults.Blueprints) + + waitGroup := sync.WaitGroup{} + + var blueprintErrors []error + var createdBlueprints []string + + for _, bp := range bareBlueprints { + waitGroup.Add(1) + go func(bp port.Blueprint) { + defer waitGroup.Done() + result, err := blueprint.NewBlueprint(portClient, bp) + + if err != nil { + blueprintErrors = append(blueprintErrors, err) + } else { + createdBlueprints = append(createdBlueprints, result.Identifier) + } + }(bp) + } + waitGroup.Wait() + + if err := validateBlueprintErrors(createdBlueprints, blueprintErrors); err != nil { + return err + } + + for _, patchStage := range patchStages { + for _, bp := range patchStage { + waitGroup.Add(1) + go func(bp port.Blueprint) { + defer waitGroup.Done() + if _, err := blueprint.PatchBlueprint(portClient, bp); err != nil { + blueprintErrors = append(blueprintErrors, err) + } + }(bp) + } + waitGroup.Wait() + } + + if err := validateBlueprintErrors(createdBlueprints, blueprintErrors); err != nil { + return err + } + + for _, blueprintScorecards := range defaults.Scorecards { + for _, scorecard := range blueprintScorecards.Scorecards { + waitGroup.Add(1) + go func(blueprintIdentifier string, scorecard port.Scorecard) { + defer waitGroup.Done() + if _, err := scorecards.NewScorecard(portClient, blueprintIdentifier, scorecard); err != nil { + blueprintErrors = append(blueprintErrors, err) + } + }(blueprintScorecards.Blueprint, scorecard) + } + } + waitGroup.Wait() + + if err := validateBlueprintErrors(createdBlueprints, blueprintErrors); err != nil { + return err + } + if defaults.AppConfig != nil { + if err := integration.NewIntegration(portClient, config, defaults.AppConfig.Resources); err != nil { + log.Printf("Failed to create resources: %v.", err.Error()) + return &AbortDefaultCreationError{BlueprintsToRollback: createdBlueprints, Errors: []error{err}} + } + } else { + log.Println("No appConfig found. Skipping...") + } + + return nil +} + +func InitializeDefaults(portClient *cli.PortClient, config *port.Config) error { + defaults, err := getDefaults() + if err != nil { + return err + } + + if err := CreateResources(portClient, defaults, config); err != nil { + if err != nil { + log.Printf("Failed to create resources. Rolling back blueprints: %v", err.BlueprintsToRollback) + var rollbackWg sync.WaitGroup + for _, identifier := range err.BlueprintsToRollback { + rollbackWg.Add(1) + go func(identifier string) { + defer rollbackWg.Done() + _ = blueprint.DeleteBlueprint(portClient, identifier) + }(identifier) + } + rollbackWg.Wait() + return &ExceptionGroup{Message: err.Error(), Errors: err.Errors} + } + return fmt.Errorf("unknown error during resource creation") + } + + return nil +} + +type ExceptionGroup struct { + Message string + Errors []error +} + +func (e *ExceptionGroup) Error() string { + return e.Message +} diff --git a/pkg/port/blueprint/blueprint.go b/pkg/port/blueprint/blueprint.go new file mode 100644 index 0000000..6a1e56c --- /dev/null +++ b/pkg/port/blueprint/blueprint.go @@ -0,0 +1,51 @@ +package blueprint + +import ( + "fmt" + "github.com/port-labs/port-k8s-exporter/pkg/port" + "github.com/port-labs/port-k8s-exporter/pkg/port/cli" +) + +func NewBlueprint(portClient *cli.PortClient, blueprint port.Blueprint) (*port.Blueprint, error) { + pb := &port.ResponseBody{} + resp, err := portClient.Client.R(). + SetResult(&pb). + SetBody(blueprint). + Post("v1/blueprints") + if err != nil { + return nil, err + } + if !pb.OK { + return nil, fmt.Errorf("failed to create blueprint, got: %s", resp.Body()) + } + return &pb.Blueprint, nil +} + +func PatchBlueprint(portClient *cli.PortClient, blueprint port.Blueprint) (*port.Blueprint, error) { + pb := &port.ResponseBody{} + resp, err := portClient.Client.R(). + SetResult(&pb). + SetBody(blueprint). + Patch(fmt.Sprintf("v1/blueprints/%s", blueprint.Identifier)) + if err != nil { + return nil, err + } + if !pb.OK { + return nil, fmt.Errorf("failed to patch blueprint, got: %s", resp.Body()) + } + return &pb.Blueprint, nil +} + +func DeleteBlueprint(portClient *cli.PortClient, blueprintIdentifier string) error { + pb := &port.ResponseBody{} + resp, err := portClient.Client.R(). + SetResult(&pb). + Delete(fmt.Sprintf("v1/blueprints/%s", blueprintIdentifier)) + if err != nil { + return err + } + if !pb.OK { + return fmt.Errorf("failed to delete blueprint, got: %s", resp.Body()) + } + return nil +} diff --git a/pkg/port/models.go b/pkg/port/models.go index e88adab..f97e13e 100644 --- a/pkg/port/models.go +++ b/pkg/port/models.go @@ -103,6 +103,13 @@ type ( InvocationMethod *InvocationMethod `json:"invocationMethod,omitempty"` } + Scorecard struct { + Identifier string `json:"identifier,omitempty"` + Title string `json:"title,omitempty"` + Filter interface{} `json:"filter,omitempty"` + Rules []interface{} `json:"rules,omitempty"` + } + Relation struct { Identifier string `json:"identifier,omitempty"` Title string `json:"title,omitempty"` @@ -141,6 +148,7 @@ type ResponseBody struct { Integration Integration `json:"integration"` KafkaCredentials OrgKafkaCredentials `json:"credentials"` OrgDetails OrgDetails `json:"organization"` + Scorecard Scorecard `json:"scorecard"` } type EntityMapping struct { diff --git a/pkg/port/scorecards/scorecards.go b/pkg/port/scorecards/scorecards.go new file mode 100644 index 0000000..a703ea3 --- /dev/null +++ b/pkg/port/scorecards/scorecards.go @@ -0,0 +1,22 @@ +package scorecards + +import ( + "fmt" + "github.com/port-labs/port-k8s-exporter/pkg/port" + "github.com/port-labs/port-k8s-exporter/pkg/port/cli" +) + +func NewScorecard(portClient *cli.PortClient, blueprintIdentifier string, scorecard port.Scorecard) (*port.Scorecard, error) { + pb := &port.ResponseBody{} + resp, err := portClient.Client.R(). + SetResult(&pb). + SetBody(scorecard). + Post("v1/blueprints/" + blueprintIdentifier + "/scorecards") + if err != nil { + return nil, err + } + if !pb.OK { + return nil, fmt.Errorf("failed to create scorecard, got: %s", resp.Body()) + } + return &pb.Scorecard, nil +} From b5d671807d49ee205db19828c442c17d7dd74014 Mon Sep 17 00:00:00 2001 From: yair Date: Wed, 13 Dec 2023 11:53:22 +0200 Subject: [PATCH 02/32] imports --- main.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/main.go b/main.go index 834aaec..b17fb74 100644 --- a/main.go +++ b/main.go @@ -1,8 +1,10 @@ package main import ( + "errors" "flag" "fmt" + "github.com/port-labs/port-k8s-exporter/pkg/defaults" "github.com/port-labs/port-k8s-exporter/pkg/goutils" "github.com/port-labs/port-k8s-exporter/pkg/port" "os" From 205cfb32f9a882b53925a38e4661d3d30d4e8eca Mon Sep 17 00:00:00 2001 From: yair Date: Wed, 13 Dec 2023 16:22:06 +0200 Subject: [PATCH 03/32] fixed imports --- main.go | 1 + 1 file changed, 1 insertion(+) diff --git a/main.go b/main.go index 34c97b5..d17b832 100644 --- a/main.go +++ b/main.go @@ -1,6 +1,7 @@ package main import ( + "errors" "flag" "fmt" "github.com/port-labs/port-k8s-exporter/pkg/config" From 6ff6dad65ca2d5aed7807cb14a2bd5f108c603f4 Mon Sep 17 00:00:00 2001 From: yair Date: Thu, 14 Dec 2023 13:04:51 +0200 Subject: [PATCH 04/32] fixed defaults behavior --- pkg/defaults/defaults.go | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/pkg/defaults/defaults.go b/pkg/defaults/defaults.go index d5e9a47..86bd707 100644 --- a/pkg/defaults/defaults.go +++ b/pkg/defaults/defaults.go @@ -181,13 +181,10 @@ func CreateResources(portClient *cli.PortClient, defaults *Defaults, config *por if err := validateBlueprintErrors(createdBlueprints, blueprintErrors); err != nil { return err } - if defaults.AppConfig != nil { - if err := integration.NewIntegration(portClient, config, defaults.AppConfig.Resources); err != nil { - log.Printf("Failed to create resources: %v.", err.Error()) - return &AbortDefaultCreationError{BlueprintsToRollback: createdBlueprints, Errors: []error{err}} - } - } else { - log.Println("No appConfig found. Skipping...") + + if err := integration.NewIntegration(portClient, config, defaults.AppConfig.Resources); err != nil { + log.Printf("Failed to create resources: %v.", err.Error()) + return &AbortDefaultCreationError{BlueprintsToRollback: createdBlueprints, Errors: []error{err}} } return nil From 34b8a32598896481c27e4536a3345c3696403a3b Mon Sep 17 00:00:00 2001 From: yair Date: Wed, 20 Dec 2023 15:15:52 +0200 Subject: [PATCH 05/32] fixed init integration.go --- main.go | 54 ++++++++++----------- pkg/config/utils.go | 8 +--- pkg/defaults/defaults.go | 6 +-- pkg/event_handler/polling/polling_test.go | 10 ++-- pkg/handlers/controllers.go | 2 +- pkg/port/cli/integration.go | 13 ++---- pkg/port/integration/integration.go | 57 +++++++++++++++++++---- pkg/port/models.go | 8 ++-- 8 files changed, 95 insertions(+), 63 deletions(-) diff --git a/main.go b/main.go index 000c4db..b7e64e5 100644 --- a/main.go +++ b/main.go @@ -5,7 +5,6 @@ import ( "flag" "fmt" "github.com/port-labs/port-k8s-exporter/pkg/config" - "github.com/port-labs/port-k8s-exporter/pkg/defaults" "github.com/port-labs/port-k8s-exporter/pkg/event_handler" "github.com/port-labs/port-k8s-exporter/pkg/event_handler/consumer" "github.com/port-labs/port-k8s-exporter/pkg/event_handler/polling" @@ -45,16 +44,30 @@ func createEventListener(stateKey string, eventListenerType string, portClient * } +func getApplicationConfig() *port.Config { + appConfig, err := config.GetConfigFile(config.ApplicationConfig.ConfigFilePath) + var fileNotFoundError *config.FileNotFoundError + if errors.As(err, &fileNotFoundError) { + appConfig = &port.Config{ + StateKey: config.ApplicationConfig.StateKey, + EventListenerType: config.ApplicationConfig.EventListenerType, + } + } + + if config.ApplicationConfig.ResyncInterval != 0 { + appConfig.ResyncInterval = config.ApplicationConfig.ResyncInterval + } + + return appConfig +} + func main() { klog.InitFlags(nil) k8sConfig := k8s.NewKubeConfig() - - fileConfig, err := config.GetConfigFile(config.ApplicationConfig.ConfigFilePath) - var fileNotFoundError *config.FileNotFoundError - // TODO: Handle file not found ffor the configuration - if errors.As(err, &fileNotFoundError) {} - + + applicationConfig := getApplicationConfig() + clientConfig, err := k8sConfig.ClientConfig() if err != nil { klog.Fatalf("Error getting K8s client config: %s", err.Error()) @@ -67,42 +80,25 @@ func main() { portClient, err := cli.New(config.ApplicationConfig.PortBaseURL, cli.WithClientID(config.ApplicationConfig.PortClientId), cli.WithClientSecret(config.ApplicationConfig.PortClientSecret), - cli.WithHeader("User-Agent", fmt.Sprintf("port-k8s-exporter/0.1 (statekey/%s)", exporterConfig.StateKey)), + cli.WithHeader("User-Agent", fmt.Sprintf("port-k8s-exporter/0.1 (statekey/%s)", applicationConfig.StateKey)), ) if err != nil { klog.Fatalf("Error building Port client: %s", err.Error()) } - if fileConfig == nil { - // Todo: pass specific needed values - err := defaults.InitializeDefaults(portClient, exporterConfig) - if err != nil { - klog.Warningf("Error initializing defaults: %s", err.Error()) - } - } else if err != nil { - klog.Fatalf("Error building Port K8s Exporter config: %s", err.Error()) - } - - _, err = integration.GetIntegrationConfig(portClient, config.ApplicationConfig.StateKey) - if err != nil { - if exporterConfig == nil { - klog.Fatalf("The integration does not exist and no config file was provided") - } - err = integration.NewIntegration(portClient, exporterConfig.StateKey, exporterConfig.EventListenerType, exporterConfig.Resources) - if err != nil { - klog.Fatalf("Error creating K8s integration: %s", err.Error()) - } + if err := integration.InitIntegration(portClient, applicationConfig); err != nil { + klog.Fatalf("Error initializing Port integration: %s", err.Error()) } - eventListener, err := createEventListener(exporterConfig.StateKey, exporterConfig.EventListenerType, portClient) + eventListener, err := createEventListener(applicationConfig.StateKey, applicationConfig.EventListenerType, portClient) if err != nil { klog.Fatalf("Error creating event listener: %s", err.Error()) } klog.Info("Starting controllers handler") err = event_handler.StartEventHandler(eventListener, func() (event_handler.IStoppableRsync, error) { - return initiateHandler(exporterConfig, k8sClient, portClient) + return initiateHandler(applicationConfig, k8sClient, portClient) }) if err != nil { diff --git a/pkg/config/utils.go b/pkg/config/utils.go index ba61c39..db39f84 100644 --- a/pkg/config/utils.go +++ b/pkg/config/utils.go @@ -42,12 +42,8 @@ func (e *FileNotFoundError) Error() string { return e.s } -func GetConfigFile(filepath string, resyncInterval uint, stateKey string, eventListenerType string) (*port.Config, error) { - c := &port.Config{ - ResyncInterval: resyncInterval, - StateKey: stateKey, - EventListenerType: eventListenerType, - } +func GetConfigFile(filepath string) (*port.Config, error) { + c := &port.Config{} config, err := os.ReadFile(filepath) if err != nil { return c, &FileNotFoundError{err.Error()} diff --git a/pkg/defaults/defaults.go b/pkg/defaults/defaults.go index 86bd707..9711243 100644 --- a/pkg/defaults/defaults.go +++ b/pkg/defaults/defaults.go @@ -23,7 +23,7 @@ type ScorecardDefault struct { type Defaults struct { Blueprints []port.Blueprint Scorecards []ScorecardDefault - AppConfig *port.AppConfig + AppConfig *port.IntegrationConfig } func getDefaults() (*Defaults, error) { @@ -49,7 +49,7 @@ func getDefaults() (*Defaults, error) { } } - var appConfig *port.AppConfig + var appConfig *port.IntegrationConfig file, err = os.ReadFile("./assets/defaults/appConfig.yaml") if err != nil { klog.Infof("No default appConfig found. Skipping...") @@ -182,7 +182,7 @@ func CreateResources(portClient *cli.PortClient, defaults *Defaults, config *por return err } - if err := integration.NewIntegration(portClient, config, defaults.AppConfig.Resources); err != nil { + if err := integration.NewIntegration(portClient, config.StateKey, config.EventListenerType, defaults.AppConfig); err != nil { log.Printf("Failed to create resources: %v.", err.Error()) return &AbortDefaultCreationError{BlueprintsToRollback: createdBlueprints, Errors: []error{err}} } diff --git a/pkg/event_handler/polling/polling_test.go b/pkg/event_handler/polling/polling_test.go index 759e84d..aed0311 100644 --- a/pkg/event_handler/polling/polling_test.go +++ b/pkg/event_handler/polling/polling_test.go @@ -39,7 +39,9 @@ func NewFixture(t *testing.T, c chan time.Time) *Fixture { } _ = integration.DeleteIntegration(portClient, stateKey) - err = integration.NewIntegration(portClient, stateKey, "", []port.Resource{}) + err = integration.NewIntegration(portClient, stateKey, "", &port.IntegrationConfig{ + Resources: []port.Resource{}, + }) if err != nil { t.Errorf("Error creating Port integration: %s", err.Error()) } @@ -69,8 +71,10 @@ func TestPolling_DifferentConfiguration(t *testing.T) { time.Sleep(time.Millisecond * 500) assert.False(t, called) - _ = integration.UpdateIntegrationConfig(fixture.portClient, fixture.stateKey, &port.AppConfig{ - Resources: []port.Resource{}, + _ = integration.PatchIntegration(fixture.portClient, fixture.stateKey, &port.Integration{ + Config: &port.IntegrationConfig{ + Resources: []port.Resource{}, + }, }) c <- time.Now() diff --git a/pkg/handlers/controllers.go b/pkg/handlers/controllers.go index 385aff3..78a7a8c 100644 --- a/pkg/handlers/controllers.go +++ b/pkg/handlers/controllers.go @@ -21,7 +21,7 @@ type ControllersHandler struct { stopCh chan struct{} } -func NewControllersHandler(exporterConfig *port.Config, portConfig *port.AppConfig, k8sClient *k8s.Client, portClient *cli.PortClient) *ControllersHandler { +func NewControllersHandler(exporterConfig *port.Config, portConfig *port.IntegrationConfig, k8sClient *k8s.Client, portClient *cli.PortClient) *ControllersHandler { resync := time.Minute * time.Duration(exporterConfig.ResyncInterval) informersFactory := dynamicinformer.NewDynamicSharedInformerFactory(k8sClient.DynamicClient, resync) diff --git a/pkg/port/cli/integration.go b/pkg/port/cli/integration.go index df38d9a..6059a28 100644 --- a/pkg/port/cli/integration.go +++ b/pkg/port/cli/integration.go @@ -52,7 +52,7 @@ func (c *PortClient) GetIntegration(stateKey string) (*port.Integration, error) return &pb.Integration, nil } -func (c *PortClient) GetIntegrationConfig(stateKey string) (*port.AppConfig, error) { +func (c *PortClient) GetIntegrationConfig(stateKey string) (*port.IntegrationConfig, error) { pb := &port.ResponseBody{} resp, err := c.Client.R(). SetResult(&pb). @@ -78,17 +78,12 @@ func (c *PortClient) DeleteIntegration(stateKey string) error { return nil } -func (c *PortClient) UpdateConfig(stateKey string, config *port.AppConfig) error { - type Config struct { - Config *port.AppConfig `json:"config"` - } +func (c *PortClient) PatchConfig(stateKey string, integration *port.Integration) error { pb := &port.ResponseBody{} resp, err := c.Client.R(). - SetBody(&Config{ - Config: config, - }). + SetBody(integration). SetResult(&pb). - Patch(fmt.Sprintf("v1/integration/%s/config", stateKey)) + Patch(fmt.Sprintf("v1/integration/%s", stateKey)) if err != nil { return err } diff --git a/pkg/port/integration/integration.go b/pkg/port/integration/integration.go index 1e92d18..cddb82e 100644 --- a/pkg/port/integration/integration.go +++ b/pkg/port/integration/integration.go @@ -3,11 +3,13 @@ package integration import ( "context" "fmt" + "github.com/port-labs/port-k8s-exporter/pkg/defaults" "github.com/port-labs/port-k8s-exporter/pkg/port" "github.com/port-labs/port-k8s-exporter/pkg/port/cli" + "k8s.io/klog/v2" ) -func NewIntegration(portClient *cli.PortClient, stateKey string, eventListenerType string, resources []port.Resource) error { +func NewIntegration(portClient *cli.PortClient, stateKey string, eventListenerType string, appConfig *port.IntegrationConfig) error { integration := &port.Integration{ Title: stateKey, InstallationAppType: "K8S EXPORTER", @@ -15,9 +17,7 @@ func NewIntegration(portClient *cli.PortClient, stateKey string, eventListenerTy EventListener: port.EventListenerSettings{ Type: eventListenerType, }, - Config: &port.AppConfig{ - Resources: resources, - }, + Config: appConfig, } _, err := portClient.Authenticate(context.Background(), portClient.ClientID, portClient.ClientSecret) if err != nil { @@ -31,7 +31,7 @@ func NewIntegration(portClient *cli.PortClient, stateKey string, eventListenerTy return nil } -func GetIntegrationConfig(portClient *cli.PortClient, stateKey string) (*port.AppConfig, error) { +func GetIntegrationConfig(portClient *cli.PortClient, stateKey string) (*port.IntegrationConfig, error) { _, err := portClient.Authenticate(context.Background(), portClient.ClientID, portClient.ClientSecret) if err != nil { return nil, fmt.Errorf("error authenticating with Port: %v", err) @@ -72,15 +72,56 @@ func DeleteIntegration(portClient *cli.PortClient, stateKey string) error { return nil } -func UpdateIntegrationConfig(portClient *cli.PortClient, stateKey string, config *port.AppConfig) error { +func PatchIntegration(portClient *cli.PortClient, stateKey string, integration *port.Integration) error { _, err := portClient.Authenticate(context.Background(), portClient.ClientID, portClient.ClientSecret) if err != nil { return fmt.Errorf("error authenticating with Port: %v", err) } - err = portClient.UpdateConfig(stateKey, config) + err = portClient.PatchConfig(stateKey, integration) if err != nil { - return fmt.Errorf("error updating Port integration config: %v", err) + return fmt.Errorf("error updating Port integration: %v", err) + } + return nil +} + +func InitIntegration(portClient *cli.PortClient, applicationConfig *port.Config) error { + existingIntegration, err := GetIntegration(portClient, applicationConfig.StateKey) + defaultIntegrationConfig := &port.IntegrationConfig{} + + if applicationConfig.Resources != nil { + defaultIntegrationConfig.Resources = applicationConfig.Resources } + + if err != nil { + var defaultsInitializationError error + if applicationConfig.Resources == nil { + defaultsInitializationError = defaults.InitializeDefaults(portClient, applicationConfig) + if err != nil { + klog.Warningf("Error initializing defaults: %s", err.Error()) + } + } + if defaultsInitializationError != nil || applicationConfig.Resources != nil { + // Handle a deprecated case where resources are provided in config file + err = NewIntegration(portClient, applicationConfig.StateKey, applicationConfig.EventListenerType, defaultIntegrationConfig) + if err != nil { + return fmt.Errorf("error creating Port integration: %v", err) + } + } + } else { + integrationPatch := &port.Integration{ + EventListener: port.EventListenerSettings{ + Type: applicationConfig.EventListenerType, + }, + } + if existingIntegration.Config == nil { + integrationPatch.Config = defaultIntegrationConfig + } + + if err := PatchIntegration(portClient, applicationConfig.StateKey, integrationPatch); err != nil { + return fmt.Errorf("error updating Port integration: %v", err) + } + } + return nil } diff --git a/pkg/port/models.go b/pkg/port/models.go index 8b55d64..d65f050 100644 --- a/pkg/port/models.go +++ b/pkg/port/models.go @@ -33,7 +33,7 @@ type ( Version string `json:"version,omitempty"` InstallationAppType string `json:"installationAppType,omitempty"` EventListener EventListenerSettings `json:"changelogDestination,omitempty"` - Config *AppConfig `json:"config,omitempty"` + Config *IntegrationConfig `json:"config,omitempty"` UpdatedAt *time.Time `json:"updatedAt,omitempty"` } @@ -193,16 +193,16 @@ type AggregatedResource struct { KindConfigs []KindConfig } -type AppConfig struct { +type IntegrationConfig struct { DeleteDependents bool `json:"deleteDependents,omitempty"` CreateMissingRelatedEntities bool `json:"createMissingRelatedEntities,omitempty"` - Resources []Resource `json:"resources"` + Resources []Resource `json:"resources,omitempty"` } type Config struct { ResyncInterval uint StateKey string EventListenerType string - // Deprecated: use AppConfig instead. Used for updating the Port integration config on startup. + // Deprecated: use IntegrationConfig instead. Used for updating the Port integration config on startup. Resources []Resource } From 6194ac8a80b707bb58daaa25e24d04efcf3eddf6 Mon Sep 17 00:00:00 2001 From: yair Date: Wed, 20 Dec 2023 15:16:06 +0200 Subject: [PATCH 06/32] removed whitespace --- main.go | 1 - 1 file changed, 1 deletion(-) diff --git a/main.go b/main.go index b7e64e5..a455629 100644 --- a/main.go +++ b/main.go @@ -41,7 +41,6 @@ func createEventListener(stateKey string, eventListenerType string, portClient * default: return nil, fmt.Errorf("unknown event listener type: %s", eventListenerType) } - } func getApplicationConfig() *port.Config { From 926be7ebf07e44ab48b26b3083b6fee3b4c70e14 Mon Sep 17 00:00:00 2001 From: yair Date: Wed, 20 Dec 2023 16:13:43 +0200 Subject: [PATCH 07/32] cr fixes --- main.go | 14 +++++++++----- pkg/defaults/defaults.go | 2 +- pkg/event_handler/polling/polling.go | 4 ++-- pkg/port/integration/integration.go | 15 --------------- 4 files changed, 12 insertions(+), 23 deletions(-) diff --git a/main.go b/main.go index 57f3812..66b2376 100644 --- a/main.go +++ b/main.go @@ -17,15 +17,19 @@ import ( ) func initiateHandler(exporterConfig *port.Config, k8sClient *k8s.Client, portClient *cli.PortClient) (*handlers.ControllersHandler, error) { - apiConfig, err := integration.GetIntegrationConfig(portClient, exporterConfig.StateKey) + i, err := integration.GetIntegration(portClient, exporterConfig.StateKey) if err != nil { - klog.Fatalf("Error getting K8s integration config: %s", err.Error()) + return nil, fmt.Errorf("error getting Port integration: %v", err) } + if i.Config == nil { + return nil, errors.New("integration config is nil") - cli.WithDeleteDependents(apiConfig.DeleteDependents)(portClient) - cli.WithCreateMissingRelatedEntities(apiConfig.CreateMissingRelatedEntities)(portClient) + } + + cli.WithDeleteDependents(i.Config.DeleteDependents)(portClient) + cli.WithCreateMissingRelatedEntities(i.Config.CreateMissingRelatedEntities)(portClient) - newHandler := handlers.NewControllersHandler(exporterConfig, apiConfig, k8sClient, portClient) + newHandler := handlers.NewControllersHandler(exporterConfig, i.Config, k8sClient, portClient) newHandler.Handle() return newHandler, nil diff --git a/pkg/defaults/defaults.go b/pkg/defaults/defaults.go index 9711243..bbb4551 100644 --- a/pkg/defaults/defaults.go +++ b/pkg/defaults/defaults.go @@ -117,7 +117,7 @@ func validateBlueprintErrors(createdBlueprints []string, blueprintErrors []error } func CreateResources(portClient *cli.PortClient, defaults *Defaults, config *port.Config) *AbortDefaultCreationError { - if _, err := integration.GetIntegrationConfig(portClient, config.StateKey); err == nil { + if _, err := integration.GetIntegration(portClient, config.StateKey); err == nil { log.Println("Integration already exists. Skipping...") return nil } diff --git a/pkg/event_handler/polling/polling.go b/pkg/event_handler/polling/polling.go index 8a7bdc6..99194a9 100644 --- a/pkg/event_handler/polling/polling.go +++ b/pkg/event_handler/polling/polling.go @@ -58,7 +58,7 @@ func (h *Handler) Run(resync func()) { klog.Infof("Starting polling handler") currentState, err := integration.GetIntegration(h.portClient, h.stateKey) if err != nil { - klog.Errorf("Error fetching the first AppConfig state: %s", err.Error()) + klog.Warningf("Error fetching the first AppConfig state: %s", err.Error()) } sigChan := make(chan os.Signal, 1) @@ -75,7 +75,7 @@ func (h *Handler) Run(resync func()) { klog.Infof("Polling event listener iteration after %d seconds. Checking for changes...", h.pollingRate) configuration, err := integration.GetIntegration(h.portClient, h.stateKey) if err != nil { - klog.Errorf("error resyncing: %s", err.Error()) + klog.Warningf("error resyncing: %s", err.Error()) } if reflect.DeepEqual(currentState, configuration) != true { diff --git a/pkg/port/integration/integration.go b/pkg/port/integration/integration.go index 377817b..16376db 100644 --- a/pkg/port/integration/integration.go +++ b/pkg/port/integration/integration.go @@ -31,21 +31,6 @@ func NewIntegration(portClient *cli.PortClient, stateKey string, eventListenerTy return nil } -// ToDo: remove this function -func GetIntegrationConfig(portClient *cli.PortClient, stateKey string) (*port.IntegrationConfig, error) { - _, err := portClient.Authenticate(context.Background(), portClient.ClientID, portClient.ClientSecret) - if err != nil { - return nil, fmt.Errorf("error authenticating with Port: %v", err) - } - - apiConfig, err := portClient.GetIntegrationConfig(stateKey) - if err != nil { - return nil, fmt.Errorf("error getting Port integration config: %v", err) - } - - return apiConfig, nil -} - func GetIntegration(portClient *cli.PortClient, stateKey string) (*port.Integration, error) { _, err := portClient.Authenticate(context.Background(), portClient.ClientID, portClient.ClientSecret) if err != nil { From 48d529584a985cbf15dba10db9e5d4ef76d5c0a2 Mon Sep 17 00:00:00 2001 From: yair Date: Wed, 20 Dec 2023 16:15:06 +0200 Subject: [PATCH 08/32] update naming --- pkg/port/cli/integration.go | 2 +- pkg/port/integration/integration.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/port/cli/integration.go b/pkg/port/cli/integration.go index 6059a28..530d317 100644 --- a/pkg/port/cli/integration.go +++ b/pkg/port/cli/integration.go @@ -78,7 +78,7 @@ func (c *PortClient) DeleteIntegration(stateKey string) error { return nil } -func (c *PortClient) PatchConfig(stateKey string, integration *port.Integration) error { +func (c *PortClient) PatchIntegration(stateKey string, integration *port.Integration) error { pb := &port.ResponseBody{} resp, err := c.Client.R(). SetBody(integration). diff --git a/pkg/port/integration/integration.go b/pkg/port/integration/integration.go index 16376db..ae5777d 100644 --- a/pkg/port/integration/integration.go +++ b/pkg/port/integration/integration.go @@ -64,7 +64,7 @@ func PatchIntegration(portClient *cli.PortClient, stateKey string, integration * return fmt.Errorf("error authenticating with Port: %v", err) } - err = portClient.PatchConfig(stateKey, integration) + err = portClient.PatchIntegration(stateKey, integration) if err != nil { return fmt.Errorf("error updating Port integration: %v", err) } From f34bf6b1eb20d80ed818f73432e5e2a62befa67b Mon Sep 17 00:00:00 2001 From: yair Date: Fri, 22 Dec 2023 15:43:57 +0200 Subject: [PATCH 09/32] fixed pr conflicts --- assets/defaults/appConfig.yaml | 4 +-- main.go | 3 +- pkg/defaults/init.go | 51 +++++++++++++++++++++++++++++ pkg/port/cli/integration.go | 2 +- pkg/port/integration/integration.go | 45 +------------------------ pkg/port/models.go | 14 ++++---- 6 files changed, 63 insertions(+), 56 deletions(-) create mode 100644 pkg/defaults/init.go diff --git a/assets/defaults/appConfig.yaml b/assets/defaults/appConfig.yaml index 2d6e358..9dec380 100644 --- a/assets/defaults/appConfig.yaml +++ b/assets/defaults/appConfig.yaml @@ -110,10 +110,8 @@ resources: mappings: - blueprint: '"service"' icon: '"Deployment"' - identifier: .metadata.labels.portService + identifier: .metadata.name + "-Deployment-" + .metadata.namespace + "-" + env.CLUSTER_NAME properties: {} - relations: - prod_runtime: .metadata.name + "-Deployment-" + .metadata.namespace + "-" + env.CLUSTER_NAME title: .metadata.name selector: query: .metadata.namespace | startswith("kube") | not \ No newline at end of file diff --git a/main.go b/main.go index 66b2376..fda8e32 100644 --- a/main.go +++ b/main.go @@ -5,6 +5,7 @@ import ( "flag" "fmt" "github.com/port-labs/port-k8s-exporter/pkg/config" + "github.com/port-labs/port-k8s-exporter/pkg/defaults" "github.com/port-labs/port-k8s-exporter/pkg/event_handler" "github.com/port-labs/port-k8s-exporter/pkg/event_handler/consumer" "github.com/port-labs/port-k8s-exporter/pkg/event_handler/polling" @@ -90,7 +91,7 @@ func main() { klog.Fatalf("Error building Port client: %s", err.Error()) } - if err := integration.InitIntegration(portClient, applicationConfig); err != nil { + if err := defaults.InitIntegration(portClient, applicationConfig); err != nil { klog.Fatalf("Error initializing Port integration: %s", err.Error()) } diff --git a/pkg/defaults/init.go b/pkg/defaults/init.go new file mode 100644 index 0000000..f83ffba --- /dev/null +++ b/pkg/defaults/init.go @@ -0,0 +1,51 @@ +package defaults + +import ( + "github.com/port-labs/port-k8s-exporter/pkg/port" + "github.com/port-labs/port-k8s-exporter/pkg/port/cli" + "github.com/port-labs/port-k8s-exporter/pkg/port/integration" + "k8s.io/klog/v2" +) + +func getEventListenerConfig(eventListenerType string) *port.EventListenerSettings { + if eventListenerType == "Kafka" { + return &port.EventListenerSettings{ + Type: eventListenerType, + } + } + return nil +} + +func InitIntegration(portClient *cli.PortClient, applicationConfig *port.Config) error { + existingIntegration, err := integration.GetIntegration(portClient, applicationConfig.StateKey) + defaultIntegrationConfig := &port.IntegrationConfig{} + + if applicationConfig.Resources != nil { + defaultIntegrationConfig.Resources = applicationConfig.Resources + } + + if err != nil { + var defaultsInitializationError error + if applicationConfig.Resources == nil { + defaultsInitializationError = InitializeDefaults(portClient, applicationConfig) + if err != nil { + klog.Warningf("Error initializing defaults: %s", err.Error()) + } + } + if defaultsInitializationError != nil || applicationConfig.Resources != nil { + // Handle a deprecated case where resources are provided in config file + return integration.NewIntegration(portClient, applicationConfig.StateKey, applicationConfig.EventListenerType, defaultIntegrationConfig) + } + } else { + integrationPatch := &port.Integration{ + EventListener: getEventListenerConfig(applicationConfig.EventListenerType), + } + if existingIntegration.Config == nil { + integrationPatch.Config = defaultIntegrationConfig + } + + return integration.PatchIntegration(portClient, applicationConfig.StateKey, integrationPatch) + } + + return nil +} diff --git a/pkg/port/cli/integration.go b/pkg/port/cli/integration.go index 530d317..7eea37c 100644 --- a/pkg/port/cli/integration.go +++ b/pkg/port/cli/integration.go @@ -14,7 +14,7 @@ func parseIntegration(i *port.Integration) *port.Integration { } if i.EventListener.Type == "KAFKA" { - x.EventListener = port.EventListenerSettings{ + x.EventListener = &port.EventListenerSettings{ Type: i.EventListener.Type, } } diff --git a/pkg/port/integration/integration.go b/pkg/port/integration/integration.go index ae5777d..50e6cdb 100644 --- a/pkg/port/integration/integration.go +++ b/pkg/port/integration/integration.go @@ -3,10 +3,8 @@ package integration import ( "context" "fmt" - "github.com/port-labs/port-k8s-exporter/pkg/defaults" "github.com/port-labs/port-k8s-exporter/pkg/port" "github.com/port-labs/port-k8s-exporter/pkg/port/cli" - "k8s.io/klog/v2" ) func NewIntegration(portClient *cli.PortClient, stateKey string, eventListenerType string, appConfig *port.IntegrationConfig) error { @@ -14,7 +12,7 @@ func NewIntegration(portClient *cli.PortClient, stateKey string, eventListenerTy Title: stateKey, InstallationAppType: "K8S EXPORTER", InstallationId: stateKey, - EventListener: port.EventListenerSettings{ + EventListener: &port.EventListenerSettings{ Type: eventListenerType, }, Config: appConfig, @@ -70,44 +68,3 @@ func PatchIntegration(portClient *cli.PortClient, stateKey string, integration * } return nil } - -func InitIntegration(portClient *cli.PortClient, applicationConfig *port.Config) error { - existingIntegration, err := GetIntegration(portClient, applicationConfig.StateKey) - defaultIntegrationConfig := &port.IntegrationConfig{} - - if applicationConfig.Resources != nil { - defaultIntegrationConfig.Resources = applicationConfig.Resources - } - - if err != nil { - var defaultsInitializationError error - if applicationConfig.Resources == nil { - defaultsInitializationError = defaults.InitializeDefaults(portClient, applicationConfig) - if err != nil { - klog.Warningf("Error initializing defaults: %s", err.Error()) - } - } - if defaultsInitializationError != nil || applicationConfig.Resources != nil { - // Handle a deprecated case where resources are provided in config file - err = NewIntegration(portClient, applicationConfig.StateKey, applicationConfig.EventListenerType, defaultIntegrationConfig) - if err != nil { - return fmt.Errorf("error creating Port integration: %v", err) - } - } - } else { - integrationPatch := &port.Integration{ - EventListener: port.EventListenerSettings{ - Type: applicationConfig.EventListenerType, - }, - } - if existingIntegration.Config == nil { - integrationPatch.Config = defaultIntegrationConfig - } - - if err := PatchIntegration(portClient, applicationConfig.StateKey, integrationPatch); err != nil { - return fmt.Errorf("error updating Port integration: %v", err) - } - } - - return nil -} diff --git a/pkg/port/models.go b/pkg/port/models.go index d65f050..651030a 100644 --- a/pkg/port/models.go +++ b/pkg/port/models.go @@ -28,13 +28,13 @@ type ( } Integration struct { - InstallationId string `json:"installationId,omitempty"` - Title string `json:"title,omitempty"` - Version string `json:"version,omitempty"` - InstallationAppType string `json:"installationAppType,omitempty"` - EventListener EventListenerSettings `json:"changelogDestination,omitempty"` - Config *IntegrationConfig `json:"config,omitempty"` - UpdatedAt *time.Time `json:"updatedAt,omitempty"` + InstallationId string `json:"installationId,omitempty"` + Title string `json:"title,omitempty"` + Version string `json:"version,omitempty"` + InstallationAppType string `json:"installationAppType,omitempty"` + EventListener *EventListenerSettings `json:"changelogDestination,omitempty"` + Config *IntegrationConfig `json:"config,omitempty"` + UpdatedAt *time.Time `json:"updatedAt,omitempty"` } BlueprintProperty struct { From 281c440cb20b1068d82ad4c50ac124b6310fd8c6 Mon Sep 17 00:00:00 2001 From: yair Date: Fri, 22 Dec 2023 15:44:47 +0200 Subject: [PATCH 10/32] comment --- pkg/defaults/init.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/defaults/init.go b/pkg/defaults/init.go index f83ffba..cbc4fe0 100644 --- a/pkg/defaults/init.go +++ b/pkg/defaults/init.go @@ -37,6 +37,7 @@ func InitIntegration(portClient *cli.PortClient, applicationConfig *port.Config) return integration.NewIntegration(portClient, applicationConfig.StateKey, applicationConfig.EventListenerType, defaultIntegrationConfig) } } else { + // Handle a deprecated case where resources are provided in config file and integration already exists in port with no resources integrationPatch := &port.Integration{ EventListener: getEventListenerConfig(applicationConfig.EventListenerType), } From 8be7fc7b04190a63b830b2d43cbe0cab32565205 Mon Sep 17 00:00:00 2001 From: yair Date: Fri, 22 Dec 2023 15:51:58 +0200 Subject: [PATCH 11/32] renaming --- pkg/defaults/defaults.go | 6 +++--- pkg/defaults/init.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/defaults/defaults.go b/pkg/defaults/defaults.go index bbb4551..88aa104 100644 --- a/pkg/defaults/defaults.go +++ b/pkg/defaults/defaults.go @@ -116,7 +116,7 @@ func validateBlueprintErrors(createdBlueprints []string, blueprintErrors []error return nil } -func CreateResources(portClient *cli.PortClient, defaults *Defaults, config *port.Config) *AbortDefaultCreationError { +func createResources(portClient *cli.PortClient, defaults *Defaults, config *port.Config) *AbortDefaultCreationError { if _, err := integration.GetIntegration(portClient, config.StateKey); err == nil { log.Println("Integration already exists. Skipping...") return nil @@ -190,13 +190,13 @@ func CreateResources(portClient *cli.PortClient, defaults *Defaults, config *por return nil } -func InitializeDefaults(portClient *cli.PortClient, config *port.Config) error { +func initializeDefaults(portClient *cli.PortClient, config *port.Config) error { defaults, err := getDefaults() if err != nil { return err } - if err := CreateResources(portClient, defaults, config); err != nil { + if err := createResources(portClient, defaults, config); err != nil { if err != nil { log.Printf("Failed to create resources. Rolling back blueprints: %v", err.BlueprintsToRollback) var rollbackWg sync.WaitGroup diff --git a/pkg/defaults/init.go b/pkg/defaults/init.go index cbc4fe0..e0579c3 100644 --- a/pkg/defaults/init.go +++ b/pkg/defaults/init.go @@ -27,7 +27,7 @@ func InitIntegration(portClient *cli.PortClient, applicationConfig *port.Config) if err != nil { var defaultsInitializationError error if applicationConfig.Resources == nil { - defaultsInitializationError = InitializeDefaults(portClient, applicationConfig) + defaultsInitializationError = initializeDefaults(portClient, applicationConfig) if err != nil { klog.Warningf("Error initializing defaults: %s", err.Error()) } From f763647f1952565b95559fa3c74765698adcc933 Mon Sep 17 00:00:00 2001 From: yair Date: Sun, 24 Dec 2023 02:26:18 +0200 Subject: [PATCH 12/32] defaults tests --- .gitignore | 2 + go.mod | 1 + go.sum | 2 + main.go | 2 - pkg/config/config.go | 12 ++ pkg/defaults/defaults.go | 9 +- pkg/defaults/defaults_test.go | 207 ++++++++++++++++++++++ pkg/defaults/init.go | 34 ++-- pkg/event_handler/polling/polling_test.go | 8 +- pkg/goutils/env.go | 1 + pkg/handlers/controllers.go | 2 +- pkg/port/blueprint/blueprint.go | 14 ++ pkg/port/cli/integration.go | 2 +- pkg/port/integration/integration.go | 2 +- pkg/port/models.go | 8 +- 15 files changed, 269 insertions(+), 37 deletions(-) create mode 100644 pkg/defaults/defaults_test.go diff --git a/.gitignore b/.gitignore index d51d5c5..790622b 100644 --- a/.gitignore +++ b/.gitignore @@ -23,3 +23,5 @@ __debug_bin config.yaml deployments/k8s + +.env \ No newline at end of file diff --git a/go.mod b/go.mod index 2ce19e9..a23c46c 100644 --- a/go.mod +++ b/go.mod @@ -33,6 +33,7 @@ require ( github.com/google/gofuzz v1.1.0 // indirect github.com/imdario/mergo v0.3.6 // indirect github.com/itchyny/timefmt-go v0.1.4 // indirect + github.com/joho/godotenv v1.5.1 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/mailru/easyjson v0.7.6 // indirect diff --git a/go.sum b/go.sum index 399958e..922bf79 100644 --- a/go.sum +++ b/go.sum @@ -181,6 +181,8 @@ github.com/jhump/gopoet v0.1.0/go.mod h1:me9yfT6IJSlOL3FCfrg+L6yzUEZ+5jW6WHt4Sk+ github.com/jhump/goprotoc v0.5.0/go.mod h1:VrbvcYrQOrTi3i0Vf+m+oqQWk9l72mjkJCYo7UvLHRQ= github.com/jhump/protoreflect v1.11.0/go.mod h1:U7aMIjN0NWq9swDP7xDdoMfRHb35uiuTd3Z9nFXJf5E= github.com/jhump/protoreflect v1.12.0/go.mod h1:JytZfP5d0r8pVNLZvai7U/MCuTWITgrI4tTg7puQFKI= +github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= +github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= diff --git a/main.go b/main.go index fda8e32..9ac83fc 100644 --- a/main.go +++ b/main.go @@ -2,7 +2,6 @@ package main import ( "errors" - "flag" "fmt" "github.com/port-labs/port-k8s-exporter/pkg/config" "github.com/port-labs/port-k8s-exporter/pkg/defaults" @@ -112,5 +111,4 @@ func main() { func init() { config.Init() - flag.Parse() } diff --git a/pkg/config/config.go b/pkg/config/config.go index fe8ea14..1362cec 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -1,11 +1,21 @@ package config +import ( + "flag" + "github.com/joho/godotenv" + "k8s.io/klog/v2" +) + var KafkaConfig = &KafkaConfiguration{} var PollingListenerRate uint var ApplicationConfig = &ApplicationConfiguration{} func Init() { + e := godotenv.Load() + if e != nil { + klog.Warningf("Error loading .env file: %s", e.Error()) + } // Kafka listener Configuration NewString(&KafkaConfig.Brokers, "event-listener-brokers", "localhost:9092", "Kafka event listener brokers") NewString(&KafkaConfig.SecurityProtocol, "event-listener-security-protocol", "plaintext", "Kafka event listener security protocol") @@ -22,4 +32,6 @@ func Init() { NewString(&ApplicationConfig.PortClientId, "port-client-id", "", "Port client id. Required.") NewString(&ApplicationConfig.PortClientSecret, "port-client-secret", "", "Port client secret. Required.") NewString(&ApplicationConfig.EventListenerType, "event-listener-type", "POLLING", "Event listener type, can be either POLLING or KAFKA. Optional.") + + flag.Parse() } diff --git a/pkg/defaults/defaults.go b/pkg/defaults/defaults.go index 88aa104..a0aae6b 100644 --- a/pkg/defaults/defaults.go +++ b/pkg/defaults/defaults.go @@ -23,7 +23,7 @@ type ScorecardDefault struct { type Defaults struct { Blueprints []port.Blueprint Scorecards []ScorecardDefault - AppConfig *port.IntegrationConfig + AppConfig *port.IntegrationAppConfig } func getDefaults() (*Defaults, error) { @@ -49,7 +49,7 @@ func getDefaults() (*Defaults, error) { } } - var appConfig *port.IntegrationConfig + var appConfig *port.IntegrationAppConfig file, err = os.ReadFile("./assets/defaults/appConfig.yaml") if err != nil { klog.Infof("No default appConfig found. Skipping...") @@ -118,8 +118,9 @@ func validateBlueprintErrors(createdBlueprints []string, blueprintErrors []error func createResources(portClient *cli.PortClient, defaults *Defaults, config *port.Config) *AbortDefaultCreationError { if _, err := integration.GetIntegration(portClient, config.StateKey); err == nil { - log.Println("Integration already exists. Skipping...") - return nil + return &AbortDefaultCreationError{Errors: []error{ + fmt.Errorf("integration with state key %s already exists", config.StateKey), + }} } bareBlueprints, patchStages := deconstructBlueprintsToCreationSteps(defaults.Blueprints) diff --git a/pkg/defaults/defaults_test.go b/pkg/defaults/defaults_test.go new file mode 100644 index 0000000..7ef85d1 --- /dev/null +++ b/pkg/defaults/defaults_test.go @@ -0,0 +1,207 @@ +package defaults + +import ( + "fmt" + guuid "github.com/google/uuid" + "github.com/port-labs/port-k8s-exporter/pkg/config" + "github.com/port-labs/port-k8s-exporter/pkg/port" + "github.com/port-labs/port-k8s-exporter/pkg/port/blueprint" + "github.com/port-labs/port-k8s-exporter/pkg/port/cli" + "github.com/port-labs/port-k8s-exporter/pkg/port/integration" + "github.com/stretchr/testify/assert" + "testing" +) + +type Fixture struct { + t *testing.T + portClient *cli.PortClient + stateKey string +} + +func NewFixture(t *testing.T) *Fixture { + config.Init() + stateKey := guuid.NewString() + portClient, err := cli.New(config.ApplicationConfig.PortBaseURL, cli.WithHeader("User-Agent", fmt.Sprintf("port-k8s-exporter/0.1 (statekey/%s)", stateKey)), + cli.WithClientID(config.ApplicationConfig.PortClientId), cli.WithClientSecret(config.ApplicationConfig.PortClientSecret)) + if err != nil { + t.Errorf("Error building Port client: %s", err.Error()) + } + + _ = integration.DeleteIntegration(portClient, stateKey) + return &Fixture{ + t: t, + portClient: portClient, + stateKey: stateKey, + } +} + +func (f *Fixture) CreateIntegration() { + err := integration.NewIntegration(f.portClient, f.stateKey, "", &port.IntegrationAppConfig{ + Resources: []port.Resource{}, + }) + + if err != nil { + f.t.Errorf("Error creating Port integration: %s", err.Error()) + } +} + +func (f *Fixture) CleanIntegration() { + _ = integration.DeleteIntegration(f.portClient, f.stateKey) +} + +func deleteDefaultResources(t *testing.T, portClient *cli.PortClient, stateKey string) { + _ = integration.DeleteIntegration(portClient, stateKey) + _ = blueprint.DeleteBlueprint(portClient, "workload") + _ = blueprint.DeleteBlueprint(portClient, "namespace") + _ = blueprint.DeleteBlueprint(portClient, "cluster") +} + +func Test_InitIntegration_InitDefaults(t *testing.T) { + f := NewFixture(t) + defer deleteDefaultResources(t, f.portClient, f.stateKey) + e := InitIntegration(f.portClient, &port.Config{ + StateKey: f.stateKey, + EventListenerType: "POLLING", + }) + assert.Nil(t, e) + + _, err := integration.GetIntegration(f.portClient, f.stateKey) + assert.Nil(t, err) + + _, err = blueprint.GetBlueprint(f.portClient, "workload") + assert.Nil(t, err) + + _, err = blueprint.GetBlueprint(f.portClient, "namespace") + assert.Nil(t, err) + + _, err = blueprint.GetBlueprint(f.portClient, "cluster") + assert.Nil(t, err) +} + +func Test_InitIntegration_FailingInitDefaults(t *testing.T) { + f := NewFixture(t) + defer deleteDefaultResources(t, f.portClient, f.stateKey) + if _, err := blueprint.NewBlueprint(f.portClient, port.Blueprint{ + Identifier: "workload", + Title: "Workload", + Schema: port.BlueprintSchema{ + Properties: map[string]port.BlueprintProperty{}, + }, + }); err != nil { + t.Errorf("Error creating Port blueprint: %s", err.Error()) + } + e := InitIntegration(f.portClient, &port.Config{ + StateKey: f.stateKey, + EventListenerType: "POLLING", + }) + assert.Nil(t, e) + + i, err := integration.GetIntegration(f.portClient, f.stateKey) + assert.True(t, nil == i.Config.Resources) + assert.Nil(t, err) + + _, err = blueprint.GetBlueprint(f.portClient, "namespace") + if err != nil { + _ = blueprint.DeleteBlueprint(f.portClient, "namespace") + } + assert.NotNil(t, err) + + _, err = blueprint.GetBlueprint(f.portClient, "cluster") + if err != nil { + _ = blueprint.DeleteBlueprint(f.portClient, "cluster") + } + assert.NotNil(t, err) +} + +func Test_InitIntegration_DeprecatedResourcesConfiguration(t *testing.T) { + f := NewFixture(t) + defer deleteDefaultResources(t, f.portClient, f.stateKey) + err := integration.NewIntegration(f.portClient, f.stateKey, "", nil) + if err != nil { + t.Errorf("Error creating Port integration: %s", err.Error()) + } + expectedResources := []port.Resource{ + { + Kind: "workload", + Port: port.Port{ + Entity: port.EntityMappings{ + Mappings: []port.EntityMapping{ + { + Identifier: "workload", + Title: "Workload", + Blueprint: "workload", + Properties: map[string]string{ + "namespace": "default", + }, + }, + }, + }, + }, + }, + } + e := InitIntegration(f.portClient, &port.Config{ + StateKey: f.stateKey, + EventListenerType: "POLLING", + Resources: expectedResources, + }) + assert.Nil(t, e) + + i, err := integration.GetIntegration(f.portClient, f.stateKey) + assert.Equal(t, expectedResources, i.Config.Resources) + assert.Nil(t, err) + + _, err = blueprint.GetBlueprint(f.portClient, "workload") + if err != nil { + _ = blueprint.DeleteBlueprint(f.portClient, "namespace") + } + assert.NotNil(t, err) + + _, err = blueprint.GetBlueprint(f.portClient, "namespace") + if err != nil { + _ = blueprint.DeleteBlueprint(f.portClient, "namespace") + } + assert.NotNil(t, err) + + _, err = blueprint.GetBlueprint(f.portClient, "cluster") + if err != nil { + _ = blueprint.DeleteBlueprint(f.portClient, "cluster") + } + assert.NotNil(t, err) +} + +func Test_InitIntegration_DeprecatedResourcesConfiguration_ExistingIntegration_EmptyConfiguration(t *testing.T) { + f := NewFixture(t) + defer deleteDefaultResources(t, f.portClient, f.stateKey) + err := integration.NewIntegration(f.portClient, f.stateKey, "POLLING", nil) + if err != nil { + t.Errorf("Error creating Port integration: %s", err.Error()) + } + e := InitIntegration(f.portClient, &port.Config{ + StateKey: f.stateKey, + EventListenerType: "KAFKA", + Resources: nil, + }) + assert.Nil(t, e) + + i, err := integration.GetIntegration(f.portClient, f.stateKey) + assert.Nil(t, err) + assert.Equal(t, "KAFKA", i.EventListener.Type) + + _, err = blueprint.GetBlueprint(f.portClient, "workload") + if err != nil { + _ = blueprint.DeleteBlueprint(f.portClient, "namespace") + } + assert.NotNil(t, err) + + _, err = blueprint.GetBlueprint(f.portClient, "namespace") + if err != nil { + _ = blueprint.DeleteBlueprint(f.portClient, "namespace") + } + assert.NotNil(t, err) + + _, err = blueprint.GetBlueprint(f.portClient, "cluster") + if err != nil { + _ = blueprint.DeleteBlueprint(f.portClient, "cluster") + } + assert.NotNil(t, err) +} diff --git a/pkg/defaults/init.go b/pkg/defaults/init.go index e0579c3..eb9f4ad 100644 --- a/pkg/defaults/init.go +++ b/pkg/defaults/init.go @@ -8,7 +8,7 @@ import ( ) func getEventListenerConfig(eventListenerType string) *port.EventListenerSettings { - if eventListenerType == "Kafka" { + if eventListenerType == "KAFKA" { return &port.EventListenerSettings{ Type: eventListenerType, } @@ -18,35 +18,31 @@ func getEventListenerConfig(eventListenerType string) *port.EventListenerSetting func InitIntegration(portClient *cli.PortClient, applicationConfig *port.Config) error { existingIntegration, err := integration.GetIntegration(portClient, applicationConfig.StateKey) - defaultIntegrationConfig := &port.IntegrationConfig{} - - if applicationConfig.Resources != nil { - defaultIntegrationConfig.Resources = applicationConfig.Resources - } + defaultIntegrationConfig := &port.IntegrationAppConfig{} if err != nil { - var defaultsInitializationError error - if applicationConfig.Resources == nil { - defaultsInitializationError = initializeDefaults(portClient, applicationConfig) - if err != nil { + if defaultIntegrationConfig.Resources == nil { + if err := initializeDefaults(portClient, applicationConfig); err != nil { klog.Warningf("Error initializing defaults: %s", err.Error()) + } else { + return nil } } - if defaultsInitializationError != nil || applicationConfig.Resources != nil { - // Handle a deprecated case where resources are provided in config file - return integration.NewIntegration(portClient, applicationConfig.StateKey, applicationConfig.EventListenerType, defaultIntegrationConfig) - } + + // Handle a deprecated case where resources are provided in config file + return integration.NewIntegration(portClient, applicationConfig.StateKey, applicationConfig.EventListenerType, defaultIntegrationConfig) } else { - // Handle a deprecated case where resources are provided in config file and integration already exists in port with no resources integrationPatch := &port.Integration{ EventListener: getEventListenerConfig(applicationConfig.EventListenerType), } - if existingIntegration.Config == nil { - integrationPatch.Config = defaultIntegrationConfig + if existingIntegration.Config == nil && applicationConfig.Resources != nil { + integrationPatch.Config = &port.IntegrationAppConfig{ + DeleteDependents: defaultIntegrationConfig.DeleteDependents, + CreateMissingRelatedEntities: defaultIntegrationConfig.CreateMissingRelatedEntities, + Resources: applicationConfig.Resources, + } } return integration.PatchIntegration(portClient, applicationConfig.StateKey, integrationPatch) } - - return nil } diff --git a/pkg/event_handler/polling/polling_test.go b/pkg/event_handler/polling/polling_test.go index aed0311..97098d9 100644 --- a/pkg/event_handler/polling/polling_test.go +++ b/pkg/event_handler/polling/polling_test.go @@ -1,7 +1,6 @@ package polling import ( - "flag" "fmt" guuid "github.com/google/uuid" "github.com/port-labs/port-k8s-exporter/pkg/config" @@ -30,16 +29,15 @@ func (m *MockTicker) GetC() <-chan time.Time { func NewFixture(t *testing.T, c chan time.Time) *Fixture { config.Init() - flag.Parse() stateKey := guuid.NewString() - portClient, err := cli.New("https://api.getport.io", cli.WithHeader("User-Agent", fmt.Sprintf("port-k8s-exporter/0.1 (statekey/%s)", stateKey)), + portClient, err := cli.New(config.ApplicationConfig.PortBaseURL, cli.WithHeader("User-Agent", fmt.Sprintf("port-k8s-exporter/0.1 (statekey/%s)", stateKey)), cli.WithClientID(config.ApplicationConfig.PortClientId), cli.WithClientSecret(config.ApplicationConfig.PortClientSecret)) if err != nil { t.Errorf("Error building Port client: %s", err.Error()) } _ = integration.DeleteIntegration(portClient, stateKey) - err = integration.NewIntegration(portClient, stateKey, "", &port.IntegrationConfig{ + err = integration.NewIntegration(portClient, stateKey, "", &port.IntegrationAppConfig{ Resources: []port.Resource{}, }) if err != nil { @@ -72,7 +70,7 @@ func TestPolling_DifferentConfiguration(t *testing.T) { assert.False(t, called) _ = integration.PatchIntegration(fixture.portClient, fixture.stateKey, &port.Integration{ - Config: &port.IntegrationConfig{ + Config: &port.IntegrationAppConfig{ Resources: []port.Resource{}, }, }) diff --git a/pkg/goutils/env.go b/pkg/goutils/env.go index 3526492..9468b1e 100644 --- a/pkg/goutils/env.go +++ b/pkg/goutils/env.go @@ -7,6 +7,7 @@ import ( ) func GetStringEnvOrDefault(key string, defaultValue string) string { + // Try to get from .env file value := os.Getenv(key) if value == "" { return defaultValue diff --git a/pkg/handlers/controllers.go b/pkg/handlers/controllers.go index 78a7a8c..83437e4 100644 --- a/pkg/handlers/controllers.go +++ b/pkg/handlers/controllers.go @@ -21,7 +21,7 @@ type ControllersHandler struct { stopCh chan struct{} } -func NewControllersHandler(exporterConfig *port.Config, portConfig *port.IntegrationConfig, k8sClient *k8s.Client, portClient *cli.PortClient) *ControllersHandler { +func NewControllersHandler(exporterConfig *port.Config, portConfig *port.IntegrationAppConfig, k8sClient *k8s.Client, portClient *cli.PortClient) *ControllersHandler { resync := time.Minute * time.Duration(exporterConfig.ResyncInterval) informersFactory := dynamicinformer.NewDynamicSharedInformerFactory(k8sClient.DynamicClient, resync) diff --git a/pkg/port/blueprint/blueprint.go b/pkg/port/blueprint/blueprint.go index 6a1e56c..fd96c48 100644 --- a/pkg/port/blueprint/blueprint.go +++ b/pkg/port/blueprint/blueprint.go @@ -49,3 +49,17 @@ func DeleteBlueprint(portClient *cli.PortClient, blueprintIdentifier string) err } return nil } + +func GetBlueprint(portClient *cli.PortClient, blueprintIdentifier string) (*port.Blueprint, error) { + pb := &port.ResponseBody{} + resp, err := portClient.Client.R(). + SetResult(&pb). + Get(fmt.Sprintf("v1/blueprints/%s", blueprintIdentifier)) + if err != nil { + return nil, err + } + if !pb.OK { + return nil, fmt.Errorf("failed to get blueprint, got: %s", resp.Body()) + } + return &pb.Blueprint, nil +} diff --git a/pkg/port/cli/integration.go b/pkg/port/cli/integration.go index 7eea37c..0094855 100644 --- a/pkg/port/cli/integration.go +++ b/pkg/port/cli/integration.go @@ -52,7 +52,7 @@ func (c *PortClient) GetIntegration(stateKey string) (*port.Integration, error) return &pb.Integration, nil } -func (c *PortClient) GetIntegrationConfig(stateKey string) (*port.IntegrationConfig, error) { +func (c *PortClient) GetIntegrationConfig(stateKey string) (*port.IntegrationAppConfig, error) { pb := &port.ResponseBody{} resp, err := c.Client.R(). SetResult(&pb). diff --git a/pkg/port/integration/integration.go b/pkg/port/integration/integration.go index 50e6cdb..855b114 100644 --- a/pkg/port/integration/integration.go +++ b/pkg/port/integration/integration.go @@ -7,7 +7,7 @@ import ( "github.com/port-labs/port-k8s-exporter/pkg/port/cli" ) -func NewIntegration(portClient *cli.PortClient, stateKey string, eventListenerType string, appConfig *port.IntegrationConfig) error { +func NewIntegration(portClient *cli.PortClient, stateKey string, eventListenerType string, appConfig *port.IntegrationAppConfig) error { integration := &port.Integration{ Title: stateKey, InstallationAppType: "K8S EXPORTER", diff --git a/pkg/port/models.go b/pkg/port/models.go index 651030a..bb6b97e 100644 --- a/pkg/port/models.go +++ b/pkg/port/models.go @@ -33,7 +33,7 @@ type ( Version string `json:"version,omitempty"` InstallationAppType string `json:"installationAppType,omitempty"` EventListener *EventListenerSettings `json:"changelogDestination,omitempty"` - Config *IntegrationConfig `json:"config,omitempty"` + Config *IntegrationAppConfig `json:"config,omitempty"` UpdatedAt *time.Time `json:"updatedAt,omitempty"` } @@ -83,7 +83,7 @@ type ( Blueprint struct { Meta Identifier string `json:"identifier,omitempty"` - Title string `json:"title"` + Title string `json:"title,omitempty"` Icon string `json:"icon"` Description string `json:"description"` Schema BlueprintSchema `json:"schema"` @@ -193,7 +193,7 @@ type AggregatedResource struct { KindConfigs []KindConfig } -type IntegrationConfig struct { +type IntegrationAppConfig struct { DeleteDependents bool `json:"deleteDependents,omitempty"` CreateMissingRelatedEntities bool `json:"createMissingRelatedEntities,omitempty"` Resources []Resource `json:"resources,omitempty"` @@ -203,6 +203,6 @@ type Config struct { ResyncInterval uint StateKey string EventListenerType string - // Deprecated: use IntegrationConfig instead. Used for updating the Port integration config on startup. + // Deprecated: use IntegrationAppConfig instead. Used for updating the Port integration config on startup. Resources []Resource } From bc5ceb617eae4fe48d20ba93b8d637d035b047a9 Mon Sep 17 00:00:00 2001 From: yair Date: Sun, 24 Dec 2023 10:00:01 +0200 Subject: [PATCH 13/32] removed error printing for the .env file --- pkg/config/config.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/pkg/config/config.go b/pkg/config/config.go index 1362cec..67291eb 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -3,7 +3,6 @@ package config import ( "flag" "github.com/joho/godotenv" - "k8s.io/klog/v2" ) var KafkaConfig = &KafkaConfiguration{} @@ -12,10 +11,7 @@ var PollingListenerRate uint var ApplicationConfig = &ApplicationConfiguration{} func Init() { - e := godotenv.Load() - if e != nil { - klog.Warningf("Error loading .env file: %s", e.Error()) - } + _ = godotenv.Load() // Kafka listener Configuration NewString(&KafkaConfig.Brokers, "event-listener-brokers", "localhost:9092", "Kafka event listener brokers") NewString(&KafkaConfig.SecurityProtocol, "event-listener-security-protocol", "plaintext", "Kafka event listener security protocol") From ef56c38e979c8134c1fddd3b236adba1255c4144 Mon Sep 17 00:00:00 2001 From: yair Date: Sun, 24 Dec 2023 10:01:30 +0200 Subject: [PATCH 14/32] removed a comment --- pkg/goutils/env.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/goutils/env.go b/pkg/goutils/env.go index 9468b1e..3526492 100644 --- a/pkg/goutils/env.go +++ b/pkg/goutils/env.go @@ -7,7 +7,6 @@ import ( ) func GetStringEnvOrDefault(key string, defaultValue string) string { - // Try to get from .env file value := os.Getenv(key) if value == "" { return defaultValue From c7d2801288c956f28ed2313dd1095fbba073c0a8 Mon Sep 17 00:00:00 2001 From: yair Date: Sun, 24 Dec 2023 12:36:46 +0200 Subject: [PATCH 15/32] fix tests --- pkg/config/utils.go | 3 +- pkg/defaults/defaults_test.go | 2 +- pkg/event_handler/polling/polling_test.go | 3 +- pkg/k8s/controller_test.go | 9 ++-- pkg/port/blueprint/blueprint.go | 61 ++++++++++----------- pkg/port/cli/blueprint.go | 64 +++++++++++++++++++++++ test_utils/testing_init.go | 20 +++++++ 7 files changed, 121 insertions(+), 41 deletions(-) create mode 100644 pkg/port/cli/blueprint.go create mode 100644 test_utils/testing_init.go diff --git a/pkg/config/utils.go b/pkg/config/utils.go index db39f84..8a889ed 100644 --- a/pkg/config/utils.go +++ b/pkg/config/utils.go @@ -5,7 +5,6 @@ import ( "github.com/port-labs/port-k8s-exporter/pkg/goutils" "github.com/port-labs/port-k8s-exporter/pkg/port" "gopkg.in/yaml.v2" - "k8s.io/klog/v2" "k8s.io/utils/strings/slices" "os" "strings" @@ -17,7 +16,7 @@ func prepareEnvKey(key string) string { newKey := strings.ToUpper(strings.ReplaceAll(key, "-", "_")) if slices.Contains(keys, newKey) { - klog.Fatalf("Application Error : Found duplicate config key: %s", newKey) + panic("Application Error : Found duplicate config key: " + newKey) } keys = append(keys, newKey) diff --git a/pkg/defaults/defaults_test.go b/pkg/defaults/defaults_test.go index 7ef85d1..90cb2d3 100644 --- a/pkg/defaults/defaults_test.go +++ b/pkg/defaults/defaults_test.go @@ -8,6 +8,7 @@ import ( "github.com/port-labs/port-k8s-exporter/pkg/port/blueprint" "github.com/port-labs/port-k8s-exporter/pkg/port/cli" "github.com/port-labs/port-k8s-exporter/pkg/port/integration" + _ "github.com/port-labs/port-k8s-exporter/test_utils" "github.com/stretchr/testify/assert" "testing" ) @@ -19,7 +20,6 @@ type Fixture struct { } func NewFixture(t *testing.T) *Fixture { - config.Init() stateKey := guuid.NewString() portClient, err := cli.New(config.ApplicationConfig.PortBaseURL, cli.WithHeader("User-Agent", fmt.Sprintf("port-k8s-exporter/0.1 (statekey/%s)", stateKey)), cli.WithClientID(config.ApplicationConfig.PortClientId), cli.WithClientSecret(config.ApplicationConfig.PortClientSecret)) diff --git a/pkg/event_handler/polling/polling_test.go b/pkg/event_handler/polling/polling_test.go index 97098d9..6cba72f 100644 --- a/pkg/event_handler/polling/polling_test.go +++ b/pkg/event_handler/polling/polling_test.go @@ -2,6 +2,8 @@ package polling import ( "fmt" + _ "github.com/port-labs/port-k8s-exporter/test_utils" + guuid "github.com/google/uuid" "github.com/port-labs/port-k8s-exporter/pkg/config" "github.com/port-labs/port-k8s-exporter/pkg/port" @@ -28,7 +30,6 @@ func (m *MockTicker) GetC() <-chan time.Time { } func NewFixture(t *testing.T, c chan time.Time) *Fixture { - config.Init() stateKey := guuid.NewString() portClient, err := cli.New(config.ApplicationConfig.PortBaseURL, cli.WithHeader("User-Agent", fmt.Sprintf("port-k8s-exporter/0.1 (statekey/%s)", stateKey)), cli.WithClientID(config.ApplicationConfig.PortClientId), cli.WithClientSecret(config.ApplicationConfig.PortClientSecret)) diff --git a/pkg/k8s/controller_test.go b/pkg/k8s/controller_test.go index f6dac62..32ec26e 100644 --- a/pkg/k8s/controller_test.go +++ b/pkg/k8s/controller_test.go @@ -1,13 +1,14 @@ package k8s import ( + "github.com/port-labs/port-k8s-exporter/pkg/config" "github.com/port-labs/port-k8s-exporter/pkg/port" "github.com/port-labs/port-k8s-exporter/pkg/port/cli" + _ "github.com/port-labs/port-k8s-exporter/test_utils" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" k8sfake "k8s.io/client-go/dynamic/fake" - "os" "reflect" "strings" "testing" @@ -33,12 +34,12 @@ func newFixture(t *testing.T, portClientId string, portClientSecret string, reso kubeclient := k8sfake.NewSimpleDynamicClient(runtime.NewScheme()) if portClientId == "" { - portClientId = os.Getenv("PORT_CLIENT_ID") + portClientId = config.ApplicationConfig.PortClientId } if portClientSecret == "" { - portClientSecret = os.Getenv("PORT_CLIENT_SECRET") + portClientSecret = config.ApplicationConfig.PortClientSecret } - portClient, err := cli.New("https://api.getport.io", cli.WithHeader("User-Agent", "port-k8s-exporter/0.1"), + portClient, err := cli.New(config.ApplicationConfig.PortBaseURL, cli.WithHeader("User-Agent", "port-k8s-exporter/0.1"), cli.WithClientID(portClientId), cli.WithClientSecret(portClientSecret)) if err != nil { t.Errorf("Error building Port client: %s", err.Error()) diff --git a/pkg/port/blueprint/blueprint.go b/pkg/port/blueprint/blueprint.go index fd96c48..df51b5f 100644 --- a/pkg/port/blueprint/blueprint.go +++ b/pkg/port/blueprint/blueprint.go @@ -1,65 +1,60 @@ package blueprint import ( + "context" "fmt" "github.com/port-labs/port-k8s-exporter/pkg/port" "github.com/port-labs/port-k8s-exporter/pkg/port/cli" ) func NewBlueprint(portClient *cli.PortClient, blueprint port.Blueprint) (*port.Blueprint, error) { - pb := &port.ResponseBody{} - resp, err := portClient.Client.R(). - SetResult(&pb). - SetBody(blueprint). - Post("v1/blueprints") + _, err := portClient.Authenticate(context.Background(), portClient.ClientID, portClient.ClientSecret) if err != nil { - return nil, err + return nil, fmt.Errorf("error authenticating with Port: %v", err) } - if !pb.OK { - return nil, fmt.Errorf("failed to create blueprint, got: %s", resp.Body()) + + bp, err := cli.NewBlueprint(portClient, blueprint) + if err != nil { + return nil, fmt.Errorf("error creating Port blueprint: %v", err) } - return &pb.Blueprint, nil + return bp, nil } func PatchBlueprint(portClient *cli.PortClient, blueprint port.Blueprint) (*port.Blueprint, error) { - pb := &port.ResponseBody{} - resp, err := portClient.Client.R(). - SetResult(&pb). - SetBody(blueprint). - Patch(fmt.Sprintf("v1/blueprints/%s", blueprint.Identifier)) + _, err := portClient.Authenticate(context.Background(), portClient.ClientID, portClient.ClientSecret) if err != nil { - return nil, err + return nil, fmt.Errorf("error authenticating with Port: %v", err) } - if !pb.OK { - return nil, fmt.Errorf("failed to patch blueprint, got: %s", resp.Body()) + + bp, err := cli.PatchBlueprint(portClient, blueprint) + if err != nil { + return nil, fmt.Errorf("error patching Port blueprint: %v", err) } - return &pb.Blueprint, nil + return bp, nil } func DeleteBlueprint(portClient *cli.PortClient, blueprintIdentifier string) error { - pb := &port.ResponseBody{} - resp, err := portClient.Client.R(). - SetResult(&pb). - Delete(fmt.Sprintf("v1/blueprints/%s", blueprintIdentifier)) + _, err := portClient.Authenticate(context.Background(), portClient.ClientID, portClient.ClientSecret) if err != nil { - return err + return fmt.Errorf("error authenticating with Port: %v", err) } - if !pb.OK { - return fmt.Errorf("failed to delete blueprint, got: %s", resp.Body()) + + err = cli.DeleteBlueprint(portClient, blueprintIdentifier) + if err != nil { + return fmt.Errorf("error deleting Port blueprint: %v", err) } return nil } func GetBlueprint(portClient *cli.PortClient, blueprintIdentifier string) (*port.Blueprint, error) { - pb := &port.ResponseBody{} - resp, err := portClient.Client.R(). - SetResult(&pb). - Get(fmt.Sprintf("v1/blueprints/%s", blueprintIdentifier)) + _, err := portClient.Authenticate(context.Background(), portClient.ClientID, portClient.ClientSecret) if err != nil { - return nil, err + return nil, fmt.Errorf("error authenticating with Port: %v", err) } - if !pb.OK { - return nil, fmt.Errorf("failed to get blueprint, got: %s", resp.Body()) + + bp, err := cli.GetBlueprint(portClient, blueprintIdentifier) + if err != nil { + return nil, fmt.Errorf("error getting Port blueprint: %v", err) } - return &pb.Blueprint, nil + return bp, nil } diff --git a/pkg/port/cli/blueprint.go b/pkg/port/cli/blueprint.go new file mode 100644 index 0000000..280897b --- /dev/null +++ b/pkg/port/cli/blueprint.go @@ -0,0 +1,64 @@ +package cli + +import ( + "fmt" + "github.com/port-labs/port-k8s-exporter/pkg/port" +) + +func NewBlueprint(portClient *PortClient, blueprint port.Blueprint) (*port.Blueprint, error) { + pb := &port.ResponseBody{} + resp, err := portClient.Client.R(). + SetResult(&pb). + SetBody(blueprint). + Post("v1/blueprints") + if err != nil { + return nil, err + } + if !pb.OK { + return nil, fmt.Errorf("failed to create blueprint, got: %s", resp.Body()) + } + return &pb.Blueprint, nil +} + +func PatchBlueprint(portClient *PortClient, blueprint port.Blueprint) (*port.Blueprint, error) { + pb := &port.ResponseBody{} + resp, err := portClient.Client.R(). + SetResult(&pb). + SetBody(blueprint). + Patch(fmt.Sprintf("v1/blueprints/%s", blueprint.Identifier)) + if err != nil { + return nil, err + } + if !pb.OK { + return nil, fmt.Errorf("failed to patch blueprint, got: %s", resp.Body()) + } + return &pb.Blueprint, nil +} + +func DeleteBlueprint(portClient *PortClient, blueprintIdentifier string) error { + pb := &port.ResponseBody{} + resp, err := portClient.Client.R(). + SetResult(&pb). + Delete(fmt.Sprintf("v1/blueprints/%s", blueprintIdentifier)) + if err != nil { + return err + } + if !pb.OK { + return fmt.Errorf("failed to delete blueprint, got: %s", resp.Body()) + } + return nil +} + +func GetBlueprint(portClient *PortClient, blueprintIdentifier string) (*port.Blueprint, error) { + pb := &port.ResponseBody{} + resp, err := portClient.Client.R(). + SetResult(&pb). + Get(fmt.Sprintf("v1/blueprints/%s", blueprintIdentifier)) + if err != nil { + return nil, err + } + if !pb.OK { + return nil, fmt.Errorf("failed to get blueprint, got: %s", resp.Body()) + } + return &pb.Blueprint, nil +} diff --git a/test_utils/testing_init.go b/test_utils/testing_init.go new file mode 100644 index 0000000..27873e0 --- /dev/null +++ b/test_utils/testing_init.go @@ -0,0 +1,20 @@ +package testing_init + +import ( + "github.com/port-labs/port-k8s-exporter/pkg/config" + "os" + "path" + "runtime" + "testing" +) + +func init() { + _, filename, _, _ := runtime.Caller(0) + dir := path.Join(path.Dir(filename), "..") + err := os.Chdir(dir) + if err != nil { + panic(err) + } + testing.Init() + config.Init() +} From b4a52704f61c672ca6f3a98f56e2872a25bf8af1 Mon Sep 17 00:00:00 2001 From: yair Date: Sun, 24 Dec 2023 16:48:18 +0200 Subject: [PATCH 16/32] allow configuring if resources should be created --- main.go | 5 +++-- pkg/config/config.go | 1 + pkg/config/models.go | 15 ++++++++------- pkg/config/utils.go | 5 +++++ pkg/defaults/init.go | 7 ++++--- pkg/goutils/env.go | 13 +++++++++++++ pkg/port/models.go | 7 ++++--- 7 files changed, 38 insertions(+), 15 deletions(-) diff --git a/main.go b/main.go index 9ac83fc..ad61185 100644 --- a/main.go +++ b/main.go @@ -52,8 +52,9 @@ func getApplicationConfig() *port.Config { var fileNotFoundError *config.FileNotFoundError if errors.As(err, &fileNotFoundError) { appConfig = &port.Config{ - StateKey: config.ApplicationConfig.StateKey, - EventListenerType: config.ApplicationConfig.EventListenerType, + StateKey: config.ApplicationConfig.StateKey, + EventListenerType: config.ApplicationConfig.EventListenerType, + CreateDefaultResources: config.ApplicationConfig.CreateDefaultResources, } } diff --git a/pkg/config/config.go b/pkg/config/config.go index 67291eb..f741abd 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -28,6 +28,7 @@ func Init() { NewString(&ApplicationConfig.PortClientId, "port-client-id", "", "Port client id. Required.") NewString(&ApplicationConfig.PortClientSecret, "port-client-secret", "", "Port client secret. Required.") NewString(&ApplicationConfig.EventListenerType, "event-listener-type", "POLLING", "Event listener type, can be either POLLING or KAFKA. Optional.") + NewBool(&ApplicationConfig.CreateDefaultResources, "create-default-resources", true, "Create default resources on installation. Optional.") flag.Parse() } diff --git a/pkg/config/models.go b/pkg/config/models.go index f57eef1..88d3e10 100644 --- a/pkg/config/models.go +++ b/pkg/config/models.go @@ -11,11 +11,12 @@ type KafkaConfiguration struct { } type ApplicationConfiguration struct { - ConfigFilePath string - StateKey string - ResyncInterval uint - PortBaseURL string - PortClientId string - PortClientSecret string - EventListenerType string + ConfigFilePath string + StateKey string + ResyncInterval uint + PortBaseURL string + PortClientId string + PortClientSecret string + EventListenerType string + CreateDefaultResources bool } diff --git a/pkg/config/utils.go b/pkg/config/utils.go index 8a889ed..bf1f163 100644 --- a/pkg/config/utils.go +++ b/pkg/config/utils.go @@ -33,6 +33,11 @@ func NewUInt(v *uint, key string, defaultValue uint, description string) { flag.UintVar(v, key, value, description) } +func NewBool(v *bool, key string, defaultValue bool, description string) { + value := goutils.GetBoolEnvOrDefault(prepareEnvKey(key), defaultValue) + flag.BoolVar(v, key, value, description) +} + type FileNotFoundError struct { s string } diff --git a/pkg/defaults/init.go b/pkg/defaults/init.go index eb9f4ad..5b27119 100644 --- a/pkg/defaults/init.go +++ b/pkg/defaults/init.go @@ -18,10 +18,12 @@ func getEventListenerConfig(eventListenerType string) *port.EventListenerSetting func InitIntegration(portClient *cli.PortClient, applicationConfig *port.Config) error { existingIntegration, err := integration.GetIntegration(portClient, applicationConfig.StateKey) - defaultIntegrationConfig := &port.IntegrationAppConfig{} + defaultIntegrationConfig := &port.IntegrationAppConfig{ + Resources: applicationConfig.Resources, + } if err != nil { - if defaultIntegrationConfig.Resources == nil { + if defaultIntegrationConfig.Resources == nil && applicationConfig.CreateDefaultResources { if err := initializeDefaults(portClient, applicationConfig); err != nil { klog.Warningf("Error initializing defaults: %s", err.Error()) } else { @@ -39,7 +41,6 @@ func InitIntegration(portClient *cli.PortClient, applicationConfig *port.Config) integrationPatch.Config = &port.IntegrationAppConfig{ DeleteDependents: defaultIntegrationConfig.DeleteDependents, CreateMissingRelatedEntities: defaultIntegrationConfig.CreateMissingRelatedEntities, - Resources: applicationConfig.Resources, } } diff --git a/pkg/goutils/env.go b/pkg/goutils/env.go index 3526492..be6c0fd 100644 --- a/pkg/goutils/env.go +++ b/pkg/goutils/env.go @@ -26,3 +26,16 @@ func GetUintEnvOrDefault(key string, defaultValue uint64) uint64 { } return result } + +func GetBoolEnvOrDefault(key string, defaultValue bool) bool { + value := os.Getenv(key) + if value == "" { + return defaultValue + } + result, err := strconv.ParseBool(value) + if err != nil { + fmt.Printf("Using default value "+strconv.FormatBool(defaultValue)+" for "+key+". error parsing env variable %s: %s", key, err.Error()) + return defaultValue + } + return result +} diff --git a/pkg/port/models.go b/pkg/port/models.go index bb6b97e..000a878 100644 --- a/pkg/port/models.go +++ b/pkg/port/models.go @@ -200,9 +200,10 @@ type IntegrationAppConfig struct { } type Config struct { - ResyncInterval uint - StateKey string - EventListenerType string + ResyncInterval uint + StateKey string + EventListenerType string + CreateDefaultResources bool // Deprecated: use IntegrationAppConfig instead. Used for updating the Port integration config on startup. Resources []Resource } From 2378a36419bb681b3558019a005670c72657eec0 Mon Sep 17 00:00:00 2001 From: yair Date: Mon, 25 Dec 2023 12:37:45 +0200 Subject: [PATCH 17/32] moved configuration --- pkg/config/config.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/config/config.go b/pkg/config/config.go index f741abd..7aae502 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -12,6 +12,9 @@ var ApplicationConfig = &ApplicationConfiguration{} func Init() { _ = godotenv.Load() + + NewString(&ApplicationConfig.EventListenerType, "event-listener-type", "POLLING", "Event listener type, can be either POLLING or KAFKA. Optional.") + // Kafka listener Configuration NewString(&KafkaConfig.Brokers, "event-listener-brokers", "localhost:9092", "Kafka event listener brokers") NewString(&KafkaConfig.SecurityProtocol, "event-listener-security-protocol", "plaintext", "Kafka event listener security protocol") @@ -27,7 +30,6 @@ func Init() { NewString(&ApplicationConfig.PortBaseURL, "port-base-url", "https://api.getport.io", "Port base URL. Optional.") NewString(&ApplicationConfig.PortClientId, "port-client-id", "", "Port client id. Required.") NewString(&ApplicationConfig.PortClientSecret, "port-client-secret", "", "Port client secret. Required.") - NewString(&ApplicationConfig.EventListenerType, "event-listener-type", "POLLING", "Event listener type, can be either POLLING or KAFKA. Optional.") NewBool(&ApplicationConfig.CreateDefaultResources, "create-default-resources", true, "Create default resources on installation. Optional.") flag.Parse() From e48d7826d64cb9d81d97bcab0ff15b89171b307a Mon Sep 17 00:00:00 2001 From: yair Date: Mon, 25 Dec 2023 12:41:25 +0200 Subject: [PATCH 18/32] fixed tests --- pkg/defaults/defaults_test.go | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/pkg/defaults/defaults_test.go b/pkg/defaults/defaults_test.go index 90cb2d3..fcae442 100644 --- a/pkg/defaults/defaults_test.go +++ b/pkg/defaults/defaults_test.go @@ -60,8 +60,9 @@ func Test_InitIntegration_InitDefaults(t *testing.T) { f := NewFixture(t) defer deleteDefaultResources(t, f.portClient, f.stateKey) e := InitIntegration(f.portClient, &port.Config{ - StateKey: f.stateKey, - EventListenerType: "POLLING", + StateKey: f.stateKey, + EventListenerType: "POLLING", + CreateDefaultResources: true, }) assert.Nil(t, e) @@ -91,8 +92,9 @@ func Test_InitIntegration_FailingInitDefaults(t *testing.T) { t.Errorf("Error creating Port blueprint: %s", err.Error()) } e := InitIntegration(f.portClient, &port.Config{ - StateKey: f.stateKey, - EventListenerType: "POLLING", + StateKey: f.stateKey, + EventListenerType: "POLLING", + CreateDefaultResources: true, }) assert.Nil(t, e) @@ -140,9 +142,10 @@ func Test_InitIntegration_DeprecatedResourcesConfiguration(t *testing.T) { }, } e := InitIntegration(f.portClient, &port.Config{ - StateKey: f.stateKey, - EventListenerType: "POLLING", - Resources: expectedResources, + StateKey: f.stateKey, + EventListenerType: "POLLING", + Resources: expectedResources, + CreateDefaultResources: true, }) assert.Nil(t, e) @@ -177,9 +180,10 @@ func Test_InitIntegration_DeprecatedResourcesConfiguration_ExistingIntegration_E t.Errorf("Error creating Port integration: %s", err.Error()) } e := InitIntegration(f.portClient, &port.Config{ - StateKey: f.stateKey, - EventListenerType: "KAFKA", - Resources: nil, + StateKey: f.stateKey, + EventListenerType: "KAFKA", + Resources: nil, + CreateDefaultResources: true, }) assert.Nil(t, e) From 9f685ed182609a3ca7f0a68d97c2e5ae29aaed67 Mon Sep 17 00:00:00 2001 From: yair Date: Mon, 25 Dec 2023 13:01:57 +0200 Subject: [PATCH 19/32] fixed tests --- pkg/defaults/init.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/defaults/init.go b/pkg/defaults/init.go index 5b27119..f02e76e 100644 --- a/pkg/defaults/init.go +++ b/pkg/defaults/init.go @@ -41,6 +41,7 @@ func InitIntegration(portClient *cli.PortClient, applicationConfig *port.Config) integrationPatch.Config = &port.IntegrationAppConfig{ DeleteDependents: defaultIntegrationConfig.DeleteDependents, CreateMissingRelatedEntities: defaultIntegrationConfig.CreateMissingRelatedEntities, + Resources: defaultIntegrationConfig.Resources, } } From b8c8cde9576d3240a1d94a19b488ad911e04ebdb Mon Sep 17 00:00:00 2001 From: yair Date: Mon, 25 Dec 2023 17:05:17 +0200 Subject: [PATCH 20/32] pr comments --- main.go | 37 +-------- pkg/config/config.go | 20 +++++ pkg/defaults/defaults.go | 50 ++++++----- pkg/defaults/defaults_test.go | 92 +++++++++------------ pkg/defaults/init.go | 5 +- pkg/event_handler/event_listener_factory.go | 21 +++++ pkg/event_handler/polling/polling_test.go | 2 +- pkg/port/blueprint/blueprint.go | 2 +- pkg/port/cli/blueprint.go | 2 +- pkg/port/cli/integration.go | 14 ---- pkg/port/cli/scorecards.go | 22 +++++ pkg/port/integration/integration.go | 2 +- pkg/port/models.go | 30 ++++--- pkg/port/scorecards/scorecards.go | 20 ++--- 14 files changed, 167 insertions(+), 152 deletions(-) create mode 100644 pkg/event_handler/event_listener_factory.go create mode 100644 pkg/port/cli/scorecards.go diff --git a/main.go b/main.go index ad61185..466f9d7 100644 --- a/main.go +++ b/main.go @@ -6,8 +6,6 @@ import ( "github.com/port-labs/port-k8s-exporter/pkg/config" "github.com/port-labs/port-k8s-exporter/pkg/defaults" "github.com/port-labs/port-k8s-exporter/pkg/event_handler" - "github.com/port-labs/port-k8s-exporter/pkg/event_handler/consumer" - "github.com/port-labs/port-k8s-exporter/pkg/event_handler/polling" "github.com/port-labs/port-k8s-exporter/pkg/handlers" "github.com/port-labs/port-k8s-exporter/pkg/k8s" "github.com/port-labs/port-k8s-exporter/pkg/port" @@ -35,42 +33,11 @@ func initiateHandler(exporterConfig *port.Config, k8sClient *k8s.Client, portCli return newHandler, nil } -func createEventListener(stateKey string, eventListenerType string, portClient *cli.PortClient) (event_handler.IListener, error) { - klog.Infof("Received event listener type: %s", eventListenerType) - switch eventListenerType { - case "KAFKA": - return consumer.NewEventListener(stateKey, portClient) - case "POLLING": - return polling.NewEventListener(stateKey, portClient), nil - default: - return nil, fmt.Errorf("unknown event listener type: %s", eventListenerType) - } -} - -func getApplicationConfig() *port.Config { - appConfig, err := config.GetConfigFile(config.ApplicationConfig.ConfigFilePath) - var fileNotFoundError *config.FileNotFoundError - if errors.As(err, &fileNotFoundError) { - appConfig = &port.Config{ - StateKey: config.ApplicationConfig.StateKey, - EventListenerType: config.ApplicationConfig.EventListenerType, - CreateDefaultResources: config.ApplicationConfig.CreateDefaultResources, - } - } - - if config.ApplicationConfig.ResyncInterval != 0 { - appConfig.ResyncInterval = config.ApplicationConfig.ResyncInterval - } - - return appConfig -} - func main() { klog.InitFlags(nil) k8sConfig := k8s.NewKubeConfig() - - applicationConfig := getApplicationConfig() + applicationConfig := config.NewConfiguration() clientConfig, err := k8sConfig.ClientConfig() if err != nil { @@ -95,7 +62,7 @@ func main() { klog.Fatalf("Error initializing Port integration: %s", err.Error()) } - eventListener, err := createEventListener(applicationConfig.StateKey, applicationConfig.EventListenerType, portClient) + eventListener, err := event_handler.CreateEventListener(applicationConfig.StateKey, applicationConfig.EventListenerType, portClient) if err != nil { klog.Fatalf("Error creating event listener: %s", err.Error()) } diff --git a/pkg/config/config.go b/pkg/config/config.go index 7aae502..d1e21da 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -1,8 +1,10 @@ package config import ( + "errors" "flag" "github.com/joho/godotenv" + "github.com/port-labs/port-k8s-exporter/pkg/port" ) var KafkaConfig = &KafkaConfiguration{} @@ -34,3 +36,21 @@ func Init() { flag.Parse() } + +func NewConfiguration() *port.Config { + c, err := GetConfigFile(ApplicationConfig.ConfigFilePath) + var fileNotFoundError *FileNotFoundError + if errors.As(err, &fileNotFoundError) { + c = &port.Config{ + StateKey: ApplicationConfig.StateKey, + EventListenerType: ApplicationConfig.EventListenerType, + CreateDefaultResources: ApplicationConfig.CreateDefaultResources, + } + } + + if ApplicationConfig.ResyncInterval != 0 { + c.ResyncInterval = ApplicationConfig.ResyncInterval + } + + return c +} diff --git a/pkg/defaults/defaults.go b/pkg/defaults/defaults.go index a0aae6b..4b7f370 100644 --- a/pkg/defaults/defaults.go +++ b/pkg/defaults/defaults.go @@ -26,9 +26,13 @@ type Defaults struct { AppConfig *port.IntegrationAppConfig } +var BlueprintsAsset = "assets/defaults/blueprints.json" +var ScorecardsAsset = "assets/defaults/scorecards.json" +var AppConfigAsset = "assets/defaults/appConfig.yaml" + func getDefaults() (*Defaults, error) { var bp []port.Blueprint - file, err := os.ReadFile("assets/defaults/blueprints.json") + file, err := os.ReadFile(BlueprintsAsset) if err != nil { klog.Infof("No default blueprints found. Skipping...") } else { @@ -39,7 +43,7 @@ func getDefaults() (*Defaults, error) { } var sc []ScorecardDefault - file, err = os.ReadFile("./assets/defaults/scorecards.json") + file, err = os.ReadFile(ScorecardsAsset) if err != nil { klog.Infof("No default scorecards found. Skipping...") } else { @@ -50,7 +54,7 @@ func getDefaults() (*Defaults, error) { } var appConfig *port.IntegrationAppConfig - file, err = os.ReadFile("./assets/defaults/appConfig.yaml") + file, err = os.ReadFile(AppConfigAsset) if err != nil { klog.Infof("No default appConfig found. Skipping...") } else { @@ -67,6 +71,11 @@ func getDefaults() (*Defaults, error) { }, nil } +// deconstructBlueprintsToCreationSteps takes a list of blueprints and returns a list of blueprints with only the +// required fields for creation, a list of blueprints with the required fields for creation and relations, and a list +// of blueprints with all fields for creation, relations, and calculation properties. +// This is done because there might be a case where a blueprint has a relation to another blueprint that +// hasn't been created yet. func deconstructBlueprintsToCreationSteps(rawBlueprints []port.Blueprint) ([]port.Blueprint, [][]port.Blueprint) { var ( bareBlueprints []port.Blueprint @@ -74,7 +83,7 @@ func deconstructBlueprintsToCreationSteps(rawBlueprints []port.Blueprint) ([]por fullBlueprints []port.Blueprint ) - for _, bp := range append([]port.Blueprint{}, rawBlueprints...) { + for _, bp := range rawBlueprints { bareBlueprint := port.Blueprint{ Identifier: bp.Identifier, Title: bp.Title, @@ -89,7 +98,7 @@ func deconstructBlueprintsToCreationSteps(rawBlueprints []port.Blueprint) ([]por withRelations = append(withRelations, withRelation) fullBlueprint := withRelation - fullBlueprint.FormulaProperties = bp.FormulaProperties + fullBlueprint.CalculationProperties = bp.CalculationProperties fullBlueprint.MirrorProperties = bp.MirrorProperties fullBlueprints = append(fullBlueprints, fullBlueprint) } @@ -109,7 +118,7 @@ func (e *AbortDefaultCreationError) Error() string { func validateBlueprintErrors(createdBlueprints []string, blueprintErrors []error) *AbortDefaultCreationError { if len(blueprintErrors) > 0 { for _, err := range blueprintErrors { - log.Printf("Failed to create resources: %v.", err.Error()) + klog.Infof("Failed to create resources: %v.", err.Error()) } return &AbortDefaultCreationError{BlueprintsToRollback: createdBlueprints, Errors: blueprintErrors} } @@ -171,7 +180,7 @@ func createResources(portClient *cli.PortClient, defaults *Defaults, config *por waitGroup.Add(1) go func(blueprintIdentifier string, scorecard port.Scorecard) { defer waitGroup.Done() - if _, err := scorecards.NewScorecard(portClient, blueprintIdentifier, scorecard); err != nil { + if err := scorecards.CreateScorecard(portClient, blueprintIdentifier, scorecard); err != nil { blueprintErrors = append(blueprintErrors, err) } }(blueprintScorecards.Blueprint, scorecard) @@ -183,7 +192,7 @@ func createResources(portClient *cli.PortClient, defaults *Defaults, config *por return err } - if err := integration.NewIntegration(portClient, config.StateKey, config.EventListenerType, defaults.AppConfig); err != nil { + if err := integration.CreateIntegration(portClient, config.StateKey, config.EventListenerType, defaults.AppConfig); err != nil { log.Printf("Failed to create resources: %v.", err.Error()) return &AbortDefaultCreationError{BlueprintsToRollback: createdBlueprints, Errors: []error{err}} } @@ -198,20 +207,19 @@ func initializeDefaults(portClient *cli.PortClient, config *port.Config) error { } if err := createResources(portClient, defaults, config); err != nil { - if err != nil { - log.Printf("Failed to create resources. Rolling back blueprints: %v", err.BlueprintsToRollback) - var rollbackWg sync.WaitGroup - for _, identifier := range err.BlueprintsToRollback { - rollbackWg.Add(1) - go func(identifier string) { - defer rollbackWg.Done() - _ = blueprint.DeleteBlueprint(portClient, identifier) - }(identifier) - } - rollbackWg.Wait() - return &ExceptionGroup{Message: err.Error(), Errors: err.Errors} + log.Printf("Failed to create resources. Rolling back blueprints: %v", err.BlueprintsToRollback) + var rollbackWg sync.WaitGroup + for _, identifier := range err.BlueprintsToRollback { + rollbackWg.Add(1) + go func(identifier string) { + defer rollbackWg.Done() + if err := blueprint.DeleteBlueprint(portClient, identifier); err != nil { + klog.Warningf("Failed to rollback blueprint %s creation: %v", identifier, err) + } + }(identifier) } - return fmt.Errorf("unknown error during resource creation") + rollbackWg.Wait() + return &ExceptionGroup{Message: err.Error(), Errors: err.Errors} } return nil diff --git a/pkg/defaults/defaults_test.go b/pkg/defaults/defaults_test.go index fcae442..e7f6fb7 100644 --- a/pkg/defaults/defaults_test.go +++ b/pkg/defaults/defaults_test.go @@ -19,6 +19,16 @@ type Fixture struct { stateKey string } +func checkBlueprintsDoesNotExist(f *Fixture, blueprints []string) { + for _, bp := range blueprints { + _, err := blueprint.GetBlueprint(f.portClient, bp) + if err != nil { + _ = blueprint.DeleteBlueprint(f.portClient, bp) + } + assert.NotNil(f.t, err) + } +} + func NewFixture(t *testing.T) *Fixture { stateKey := guuid.NewString() portClient, err := cli.New(config.ApplicationConfig.PortBaseURL, cli.WithHeader("User-Agent", fmt.Sprintf("port-k8s-exporter/0.1 (statekey/%s)", stateKey)), @@ -27,7 +37,7 @@ func NewFixture(t *testing.T) *Fixture { t.Errorf("Error building Port client: %s", err.Error()) } - _ = integration.DeleteIntegration(portClient, stateKey) + deleteDefaultResources(portClient, stateKey) return &Fixture{ t: t, portClient: portClient, @@ -36,7 +46,7 @@ func NewFixture(t *testing.T) *Fixture { } func (f *Fixture) CreateIntegration() { - err := integration.NewIntegration(f.portClient, f.stateKey, "", &port.IntegrationAppConfig{ + err := integration.CreateIntegration(f.portClient, f.stateKey, "", &port.IntegrationAppConfig{ Resources: []port.Resource{}, }) @@ -49,7 +59,7 @@ func (f *Fixture) CleanIntegration() { _ = integration.DeleteIntegration(f.portClient, f.stateKey) } -func deleteDefaultResources(t *testing.T, portClient *cli.PortClient, stateKey string) { +func deleteDefaultResources(portClient *cli.PortClient, stateKey string) { _ = integration.DeleteIntegration(portClient, stateKey) _ = blueprint.DeleteBlueprint(portClient, "workload") _ = blueprint.DeleteBlueprint(portClient, "namespace") @@ -58,7 +68,7 @@ func deleteDefaultResources(t *testing.T, portClient *cli.PortClient, stateKey s func Test_InitIntegration_InitDefaults(t *testing.T) { f := NewFixture(t) - defer deleteDefaultResources(t, f.portClient, f.stateKey) + defer deleteDefaultResources(f.portClient, f.stateKey) e := InitIntegration(f.portClient, &port.Config{ StateKey: f.stateKey, EventListenerType: "POLLING", @@ -79,9 +89,25 @@ func Test_InitIntegration_InitDefaults(t *testing.T) { assert.Nil(t, err) } +func Test_InitIntegration_InitDefaults_CreateDefaultResources_False(t *testing.T) { + f := NewFixture(t) + defer deleteDefaultResources(f.portClient, f.stateKey) + e := InitIntegration(f.portClient, &port.Config{ + StateKey: f.stateKey, + EventListenerType: "POLLING", + CreateDefaultResources: false, + }) + assert.Nil(t, e) + + _, err := integration.GetIntegration(f.portClient, f.stateKey) + assert.Nil(t, err) + + checkBlueprintsDoesNotExist(f, []string{"workload", "namespace", "cluster"}) +} + func Test_InitIntegration_FailingInitDefaults(t *testing.T) { f := NewFixture(t) - defer deleteDefaultResources(t, f.portClient, f.stateKey) + defer deleteDefaultResources(f.portClient, f.stateKey) if _, err := blueprint.NewBlueprint(f.portClient, port.Blueprint{ Identifier: "workload", Title: "Workload", @@ -102,23 +128,13 @@ func Test_InitIntegration_FailingInitDefaults(t *testing.T) { assert.True(t, nil == i.Config.Resources) assert.Nil(t, err) - _, err = blueprint.GetBlueprint(f.portClient, "namespace") - if err != nil { - _ = blueprint.DeleteBlueprint(f.portClient, "namespace") - } - assert.NotNil(t, err) - - _, err = blueprint.GetBlueprint(f.portClient, "cluster") - if err != nil { - _ = blueprint.DeleteBlueprint(f.portClient, "cluster") - } - assert.NotNil(t, err) + checkBlueprintsDoesNotExist(f, []string{"namespace", "cluster"}) } func Test_InitIntegration_DeprecatedResourcesConfiguration(t *testing.T) { f := NewFixture(t) - defer deleteDefaultResources(t, f.portClient, f.stateKey) - err := integration.NewIntegration(f.portClient, f.stateKey, "", nil) + defer deleteDefaultResources(f.portClient, f.stateKey) + err := integration.CreateIntegration(f.portClient, f.stateKey, "", nil) if err != nil { t.Errorf("Error creating Port integration: %s", err.Error()) } @@ -153,29 +169,13 @@ func Test_InitIntegration_DeprecatedResourcesConfiguration(t *testing.T) { assert.Equal(t, expectedResources, i.Config.Resources) assert.Nil(t, err) - _, err = blueprint.GetBlueprint(f.portClient, "workload") - if err != nil { - _ = blueprint.DeleteBlueprint(f.portClient, "namespace") - } - assert.NotNil(t, err) - - _, err = blueprint.GetBlueprint(f.portClient, "namespace") - if err != nil { - _ = blueprint.DeleteBlueprint(f.portClient, "namespace") - } - assert.NotNil(t, err) - - _, err = blueprint.GetBlueprint(f.portClient, "cluster") - if err != nil { - _ = blueprint.DeleteBlueprint(f.portClient, "cluster") - } - assert.NotNil(t, err) + checkBlueprintsDoesNotExist(f, []string{"workload", "namespace", "cluster"}) } func Test_InitIntegration_DeprecatedResourcesConfiguration_ExistingIntegration_EmptyConfiguration(t *testing.T) { f := NewFixture(t) - defer deleteDefaultResources(t, f.portClient, f.stateKey) - err := integration.NewIntegration(f.portClient, f.stateKey, "POLLING", nil) + defer deleteDefaultResources(f.portClient, f.stateKey) + err := integration.CreateIntegration(f.portClient, f.stateKey, "POLLING", nil) if err != nil { t.Errorf("Error creating Port integration: %s", err.Error()) } @@ -191,21 +191,5 @@ func Test_InitIntegration_DeprecatedResourcesConfiguration_ExistingIntegration_E assert.Nil(t, err) assert.Equal(t, "KAFKA", i.EventListener.Type) - _, err = blueprint.GetBlueprint(f.portClient, "workload") - if err != nil { - _ = blueprint.DeleteBlueprint(f.portClient, "namespace") - } - assert.NotNil(t, err) - - _, err = blueprint.GetBlueprint(f.portClient, "namespace") - if err != nil { - _ = blueprint.DeleteBlueprint(f.portClient, "namespace") - } - assert.NotNil(t, err) - - _, err = blueprint.GetBlueprint(f.portClient, "cluster") - if err != nil { - _ = blueprint.DeleteBlueprint(f.portClient, "cluster") - } - assert.NotNil(t, err) + checkBlueprintsDoesNotExist(f, []string{"workload", "namespace", "cluster"}) } diff --git a/pkg/defaults/init.go b/pkg/defaults/init.go index f02e76e..0cce0d7 100644 --- a/pkg/defaults/init.go +++ b/pkg/defaults/init.go @@ -32,11 +32,14 @@ func InitIntegration(portClient *cli.PortClient, applicationConfig *port.Config) } // Handle a deprecated case where resources are provided in config file - return integration.NewIntegration(portClient, applicationConfig.StateKey, applicationConfig.EventListenerType, defaultIntegrationConfig) + return integration.CreateIntegration(portClient, applicationConfig.StateKey, applicationConfig.EventListenerType, defaultIntegrationConfig) } else { integrationPatch := &port.Integration{ EventListener: getEventListenerConfig(applicationConfig.EventListenerType), } + + // Handle a deprecated case where resources are provided in config file and integration exists from previous + //versions without a config if existingIntegration.Config == nil && applicationConfig.Resources != nil { integrationPatch.Config = &port.IntegrationAppConfig{ DeleteDependents: defaultIntegrationConfig.DeleteDependents, diff --git a/pkg/event_handler/event_listener_factory.go b/pkg/event_handler/event_listener_factory.go new file mode 100644 index 0000000..e84fdb1 --- /dev/null +++ b/pkg/event_handler/event_listener_factory.go @@ -0,0 +1,21 @@ +package event_handler + +import ( + "fmt" + "github.com/port-labs/port-k8s-exporter/pkg/event_handler/consumer" + "github.com/port-labs/port-k8s-exporter/pkg/event_handler/polling" + "github.com/port-labs/port-k8s-exporter/pkg/port/cli" + "k8s.io/klog/v2" +) + +func CreateEventListener(stateKey string, eventListenerType string, portClient *cli.PortClient) (IListener, error) { + klog.Infof("Received event listener type: %s", eventListenerType) + switch eventListenerType { + case "KAFKA": + return consumer.NewEventListener(stateKey, portClient) + case "POLLING": + return polling.NewEventListener(stateKey, portClient), nil + default: + return nil, fmt.Errorf("unknown event listener type: %s", eventListenerType) + } +} diff --git a/pkg/event_handler/polling/polling_test.go b/pkg/event_handler/polling/polling_test.go index 6cba72f..468bfd8 100644 --- a/pkg/event_handler/polling/polling_test.go +++ b/pkg/event_handler/polling/polling_test.go @@ -38,7 +38,7 @@ func NewFixture(t *testing.T, c chan time.Time) *Fixture { } _ = integration.DeleteIntegration(portClient, stateKey) - err = integration.NewIntegration(portClient, stateKey, "", &port.IntegrationAppConfig{ + err = integration.CreateIntegration(portClient, stateKey, "", &port.IntegrationAppConfig{ Resources: []port.Resource{}, }) if err != nil { diff --git a/pkg/port/blueprint/blueprint.go b/pkg/port/blueprint/blueprint.go index df51b5f..6bb44ad 100644 --- a/pkg/port/blueprint/blueprint.go +++ b/pkg/port/blueprint/blueprint.go @@ -13,7 +13,7 @@ func NewBlueprint(portClient *cli.PortClient, blueprint port.Blueprint) (*port.B return nil, fmt.Errorf("error authenticating with Port: %v", err) } - bp, err := cli.NewBlueprint(portClient, blueprint) + bp, err := cli.CreateBlueprint(portClient, blueprint) if err != nil { return nil, fmt.Errorf("error creating Port blueprint: %v", err) } diff --git a/pkg/port/cli/blueprint.go b/pkg/port/cli/blueprint.go index 280897b..571ae2d 100644 --- a/pkg/port/cli/blueprint.go +++ b/pkg/port/cli/blueprint.go @@ -5,7 +5,7 @@ import ( "github.com/port-labs/port-k8s-exporter/pkg/port" ) -func NewBlueprint(portClient *PortClient, blueprint port.Blueprint) (*port.Blueprint, error) { +func CreateBlueprint(portClient *PortClient, blueprint port.Blueprint) (*port.Blueprint, error) { pb := &port.ResponseBody{} resp, err := portClient.Client.R(). SetResult(&pb). diff --git a/pkg/port/cli/integration.go b/pkg/port/cli/integration.go index 0094855..e158f6b 100644 --- a/pkg/port/cli/integration.go +++ b/pkg/port/cli/integration.go @@ -52,20 +52,6 @@ func (c *PortClient) GetIntegration(stateKey string) (*port.Integration, error) return &pb.Integration, nil } -func (c *PortClient) GetIntegrationConfig(stateKey string) (*port.IntegrationAppConfig, error) { - pb := &port.ResponseBody{} - resp, err := c.Client.R(). - SetResult(&pb). - Get(fmt.Sprintf("v1/integration/%s", stateKey)) - if err != nil { - return nil, err - } - if !pb.OK { - return nil, fmt.Errorf("failed to get integration config, got: %s", resp.Body()) - } - return pb.Integration.Config, nil -} - func (c *PortClient) DeleteIntegration(stateKey string) error { resp, err := c.Client.R(). Delete(fmt.Sprintf("v1/integration/%s", stateKey)) diff --git a/pkg/port/cli/scorecards.go b/pkg/port/cli/scorecards.go new file mode 100644 index 0000000..d740829 --- /dev/null +++ b/pkg/port/cli/scorecards.go @@ -0,0 +1,22 @@ +package cli + +import ( + "fmt" + "github.com/port-labs/port-k8s-exporter/pkg/port" +) + +func (c *PortClient) CreateScorecard(blueprintIdentifier string, scorecard port.Scorecard) (*port.Scorecard, error) { + pb := &port.ResponseBody{} + resp, err := c.Client.R(). + SetResult(&pb). + SetBody(scorecard). + SetPathParam("{blueprint}", blueprintIdentifier). + Post("v1/blueprints/{blueprint}/scorecards") + if err != nil { + return nil, err + } + if !pb.OK { + return nil, fmt.Errorf("failed to create scorecard, got: %s", resp.Body()) + } + return &pb.Scorecard, nil +} diff --git a/pkg/port/integration/integration.go b/pkg/port/integration/integration.go index 855b114..5af4ba1 100644 --- a/pkg/port/integration/integration.go +++ b/pkg/port/integration/integration.go @@ -7,7 +7,7 @@ import ( "github.com/port-labs/port-k8s-exporter/pkg/port/cli" ) -func NewIntegration(portClient *cli.PortClient, stateKey string, eventListenerType string, appConfig *port.IntegrationAppConfig) error { +func CreateIntegration(portClient *cli.PortClient, stateKey string, eventListenerType string, appConfig *port.IntegrationAppConfig) error { integration := &port.Integration{ Title: stateKey, InstallationAppType: "K8S EXPORTER", diff --git a/pkg/port/models.go b/pkg/port/models.go index 000a878..3757f96 100644 --- a/pkg/port/models.go +++ b/pkg/port/models.go @@ -57,10 +57,14 @@ type ( Path string `json:"path,omitempty"` } - BlueprintFormulaProperty struct { - Identifier string `json:"identifier,omitempty"` - Title string `json:"title,omitempty"` - Formula string `json:"formula,omitempty"` + BlueprintCalculationProperty struct { + Identifier string `json:"identifier,omitempty"` + Title string `json:"title,omitempty"` + Calculation string `json:"calculation,omitempty"` + Colors map[string]string `json:"colors,omitempty"` + Colorized bool `json:"colorized,omitempty"` + Format string `json:"format,omitempty"` + Type string `json:"type,omitempty"` } BlueprintSchema struct { @@ -82,15 +86,15 @@ type ( Blueprint struct { Meta - Identifier string `json:"identifier,omitempty"` - Title string `json:"title,omitempty"` - Icon string `json:"icon"` - Description string `json:"description"` - Schema BlueprintSchema `json:"schema"` - FormulaProperties map[string]BlueprintFormulaProperty `json:"formulaProperties"` - MirrorProperties map[string]BlueprintMirrorProperty `json:"mirrorProperties,omitempty"` - ChangelogDestination *ChangelogDestination `json:"changelogDestination,omitempty"` - Relations map[string]Relation `json:"relations,omitempty"` + Identifier string `json:"identifier,omitempty"` + Title string `json:"title,omitempty"` + Icon string `json:"icon"` + Description string `json:"description"` + Schema BlueprintSchema `json:"schema"` + CalculationProperties map[string]BlueprintCalculationProperty `json:"calculationProperties"` + MirrorProperties map[string]BlueprintMirrorProperty `json:"mirrorProperties,omitempty"` + ChangelogDestination *ChangelogDestination `json:"changelogDestination,omitempty"` + Relations map[string]Relation `json:"relations,omitempty"` } Action struct { diff --git a/pkg/port/scorecards/scorecards.go b/pkg/port/scorecards/scorecards.go index a703ea3..ddd12b6 100644 --- a/pkg/port/scorecards/scorecards.go +++ b/pkg/port/scorecards/scorecards.go @@ -1,22 +1,22 @@ package scorecards import ( + "context" "fmt" "github.com/port-labs/port-k8s-exporter/pkg/port" "github.com/port-labs/port-k8s-exporter/pkg/port/cli" ) -func NewScorecard(portClient *cli.PortClient, blueprintIdentifier string, scorecard port.Scorecard) (*port.Scorecard, error) { - pb := &port.ResponseBody{} - resp, err := portClient.Client.R(). - SetResult(&pb). - SetBody(scorecard). - Post("v1/blueprints/" + blueprintIdentifier + "/scorecards") +func CreateScorecard(portClient *cli.PortClient, blueprintIdentifier string, scorecard port.Scorecard) error { + _, err := portClient.Authenticate(context.Background(), portClient.ClientID, portClient.ClientSecret) if err != nil { - return nil, err + return fmt.Errorf("error authenticating with Port: %v", err) } - if !pb.OK { - return nil, fmt.Errorf("failed to create scorecard, got: %s", resp.Body()) + + _, err = portClient.CreateScorecard(blueprintIdentifier, scorecard) + if err != nil { + return fmt.Errorf("error creating Port integration: %v", err) } - return &pb.Scorecard, nil + + return nil } From 8a7a69ef9949dd4c76e6cdb05b166ed328841a77 Mon Sep 17 00:00:00 2001 From: yair Date: Mon, 25 Dec 2023 17:40:31 +0200 Subject: [PATCH 21/32] more comments --- pkg/defaults/defaults.go | 3 +++ pkg/defaults/init.go | 6 +++++- pkg/event_handler/polling/polling.go | 4 ++-- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/pkg/defaults/defaults.go b/pkg/defaults/defaults.go index 4b7f370..5432697 100644 --- a/pkg/defaults/defaults.go +++ b/pkg/defaults/defaults.go @@ -138,6 +138,7 @@ func createResources(portClient *cli.PortClient, defaults *Defaults, config *por var blueprintErrors []error var createdBlueprints []string + mutex := sync.Mutex{} for _, bp := range bareBlueprints { waitGroup.Add(1) @@ -145,11 +146,13 @@ func createResources(portClient *cli.PortClient, defaults *Defaults, config *por defer waitGroup.Done() result, err := blueprint.NewBlueprint(portClient, bp) + mutex.Lock() if err != nil { blueprintErrors = append(blueprintErrors, err) } else { createdBlueprints = append(createdBlueprints, result.Identifier) } + mutex.Unlock() }(bp) } waitGroup.Wait() diff --git a/pkg/defaults/init.go b/pkg/defaults/init.go index 0cce0d7..7320a3f 100644 --- a/pkg/defaults/init.go +++ b/pkg/defaults/init.go @@ -23,9 +23,13 @@ func InitIntegration(portClient *cli.PortClient, applicationConfig *port.Config) } if err != nil { + // The exporter supports a deprecated case where resources are provided in config file and integration does not + // exist. If this is not the case, we support the new way of creating the integration with the default resources. + // Only one of the two cases can be true. if defaultIntegrationConfig.Resources == nil && applicationConfig.CreateDefaultResources { if err := initializeDefaults(portClient, applicationConfig); err != nil { klog.Warningf("Error initializing defaults: %s", err.Error()) + klog.Warningf("The integration will start without default integration mapping and other default resources. Please create them manually in Port. ") } else { return nil } @@ -40,7 +44,7 @@ func InitIntegration(portClient *cli.PortClient, applicationConfig *port.Config) // Handle a deprecated case where resources are provided in config file and integration exists from previous //versions without a config - if existingIntegration.Config == nil && applicationConfig.Resources != nil { + if existingIntegration.Config == nil && defaultIntegrationConfig.Resources != nil { integrationPatch.Config = &port.IntegrationAppConfig{ DeleteDependents: defaultIntegrationConfig.DeleteDependents, CreateMissingRelatedEntities: defaultIntegrationConfig.CreateMissingRelatedEntities, diff --git a/pkg/event_handler/polling/polling.go b/pkg/event_handler/polling/polling.go index 99194a9..8a7bdc6 100644 --- a/pkg/event_handler/polling/polling.go +++ b/pkg/event_handler/polling/polling.go @@ -58,7 +58,7 @@ func (h *Handler) Run(resync func()) { klog.Infof("Starting polling handler") currentState, err := integration.GetIntegration(h.portClient, h.stateKey) if err != nil { - klog.Warningf("Error fetching the first AppConfig state: %s", err.Error()) + klog.Errorf("Error fetching the first AppConfig state: %s", err.Error()) } sigChan := make(chan os.Signal, 1) @@ -75,7 +75,7 @@ func (h *Handler) Run(resync func()) { klog.Infof("Polling event listener iteration after %d seconds. Checking for changes...", h.pollingRate) configuration, err := integration.GetIntegration(h.portClient, h.stateKey) if err != nil { - klog.Warningf("error resyncing: %s", err.Error()) + klog.Errorf("error resyncing: %s", err.Error()) } if reflect.DeepEqual(currentState, configuration) != true { From a298f08f2b89da1f52b27f2a7daacdd4568645a2 Mon Sep 17 00:00:00 2001 From: yair Date: Mon, 25 Dec 2023 17:43:00 +0200 Subject: [PATCH 22/32] omit calculation --- pkg/port/models.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/port/models.go b/pkg/port/models.go index 3757f96..42b672d 100644 --- a/pkg/port/models.go +++ b/pkg/port/models.go @@ -91,7 +91,7 @@ type ( Icon string `json:"icon"` Description string `json:"description"` Schema BlueprintSchema `json:"schema"` - CalculationProperties map[string]BlueprintCalculationProperty `json:"calculationProperties"` + CalculationProperties map[string]BlueprintCalculationProperty `json:"calculationProperties,omitempty"` MirrorProperties map[string]BlueprintMirrorProperty `json:"mirrorProperties,omitempty"` ChangelogDestination *ChangelogDestination `json:"changelogDestination,omitempty"` Relations map[string]Relation `json:"relations,omitempty"` From 9d63dcccea5f5a190616894406b00cd66d24b305 Mon Sep 17 00:00:00 2001 From: yair Date: Mon, 25 Dec 2023 17:48:17 +0200 Subject: [PATCH 23/32] aggregation properties --- pkg/defaults/defaults.go | 1 + pkg/port/models.go | 11 +++++++++++ 2 files changed, 12 insertions(+) diff --git a/pkg/defaults/defaults.go b/pkg/defaults/defaults.go index 5432697..69a8a73 100644 --- a/pkg/defaults/defaults.go +++ b/pkg/defaults/defaults.go @@ -98,6 +98,7 @@ func deconstructBlueprintsToCreationSteps(rawBlueprints []port.Blueprint) ([]por withRelations = append(withRelations, withRelation) fullBlueprint := withRelation + fullBlueprint.AggregationProperties = bp.AggregationProperties fullBlueprint.CalculationProperties = bp.CalculationProperties fullBlueprint.MirrorProperties = bp.MirrorProperties fullBlueprints = append(fullBlueprints, fullBlueprint) diff --git a/pkg/port/models.go b/pkg/port/models.go index 42b672d..87c1198 100644 --- a/pkg/port/models.go +++ b/pkg/port/models.go @@ -67,6 +67,16 @@ type ( Type string `json:"type,omitempty"` } + BlueprintAggregationProperty struct { + Title string `json:"title"` + Target string `json:"target"` + CalculationSpec interface{} `json:"calculationSpec"` + Query interface{} `json:"query,omitempty"` + Description string `json:"description,omitempty"` + Icon string `json:"icon,omitempty"` + Type string `json:"type,omitempty"` + } + BlueprintSchema struct { Properties map[string]BlueprintProperty `json:"properties"` Required []string `json:"required,omitempty"` @@ -92,6 +102,7 @@ type ( Description string `json:"description"` Schema BlueprintSchema `json:"schema"` CalculationProperties map[string]BlueprintCalculationProperty `json:"calculationProperties,omitempty"` + AggregationProperties map[string]BlueprintAggregationProperty `json:"aggregationProperties,omitempty"` MirrorProperties map[string]BlueprintMirrorProperty `json:"mirrorProperties,omitempty"` ChangelogDestination *ChangelogDestination `json:"changelogDestination,omitempty"` Relations map[string]Relation `json:"relations,omitempty"` From 22ec10f40790a20ad57116be0ef895efaa852ffa Mon Sep 17 00:00:00 2001 From: yair Date: Mon, 25 Dec 2023 17:59:30 +0200 Subject: [PATCH 24/32] deleting blueprints at the start --- pkg/defaults/defaults_test.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/pkg/defaults/defaults_test.go b/pkg/defaults/defaults_test.go index e7f6fb7..c220e7c 100644 --- a/pkg/defaults/defaults_test.go +++ b/pkg/defaults/defaults_test.go @@ -68,7 +68,6 @@ func deleteDefaultResources(portClient *cli.PortClient, stateKey string) { func Test_InitIntegration_InitDefaults(t *testing.T) { f := NewFixture(t) - defer deleteDefaultResources(f.portClient, f.stateKey) e := InitIntegration(f.portClient, &port.Config{ StateKey: f.stateKey, EventListenerType: "POLLING", @@ -91,7 +90,6 @@ func Test_InitIntegration_InitDefaults(t *testing.T) { func Test_InitIntegration_InitDefaults_CreateDefaultResources_False(t *testing.T) { f := NewFixture(t) - defer deleteDefaultResources(f.portClient, f.stateKey) e := InitIntegration(f.portClient, &port.Config{ StateKey: f.stateKey, EventListenerType: "POLLING", @@ -107,7 +105,6 @@ func Test_InitIntegration_InitDefaults_CreateDefaultResources_False(t *testing.T func Test_InitIntegration_FailingInitDefaults(t *testing.T) { f := NewFixture(t) - defer deleteDefaultResources(f.portClient, f.stateKey) if _, err := blueprint.NewBlueprint(f.portClient, port.Blueprint{ Identifier: "workload", Title: "Workload", @@ -133,7 +130,6 @@ func Test_InitIntegration_FailingInitDefaults(t *testing.T) { func Test_InitIntegration_DeprecatedResourcesConfiguration(t *testing.T) { f := NewFixture(t) - defer deleteDefaultResources(f.portClient, f.stateKey) err := integration.CreateIntegration(f.portClient, f.stateKey, "", nil) if err != nil { t.Errorf("Error creating Port integration: %s", err.Error()) @@ -174,7 +170,6 @@ func Test_InitIntegration_DeprecatedResourcesConfiguration(t *testing.T) { func Test_InitIntegration_DeprecatedResourcesConfiguration_ExistingIntegration_EmptyConfiguration(t *testing.T) { f := NewFixture(t) - defer deleteDefaultResources(f.portClient, f.stateKey) err := integration.CreateIntegration(f.portClient, f.stateKey, "POLLING", nil) if err != nil { t.Errorf("Error creating Port integration: %s", err.Error()) From 0599ab34087b196ecab05dbad31c0bb2aee099c3 Mon Sep 17 00:00:00 2001 From: yair Date: Mon, 25 Dec 2023 18:16:25 +0200 Subject: [PATCH 25/32] fixed tests --- pkg/port/cli/scorecards.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/port/cli/scorecards.go b/pkg/port/cli/scorecards.go index d740829..f0fbee8 100644 --- a/pkg/port/cli/scorecards.go +++ b/pkg/port/cli/scorecards.go @@ -10,7 +10,7 @@ func (c *PortClient) CreateScorecard(blueprintIdentifier string, scorecard port. resp, err := c.Client.R(). SetResult(&pb). SetBody(scorecard). - SetPathParam("{blueprint}", blueprintIdentifier). + SetPathParam("blueprint", blueprintIdentifier). Post("v1/blueprints/{blueprint}/scorecards") if err != nil { return nil, err From dc887c37824f26803dc847f5bfb91f7490d56677 Mon Sep 17 00:00:00 2001 From: yair Date: Mon, 25 Dec 2023 18:17:36 +0200 Subject: [PATCH 26/32] switch to klog --- pkg/defaults/defaults.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pkg/defaults/defaults.go b/pkg/defaults/defaults.go index 69a8a73..0e99eb8 100644 --- a/pkg/defaults/defaults.go +++ b/pkg/defaults/defaults.go @@ -10,7 +10,6 @@ import ( "github.com/port-labs/port-k8s-exporter/pkg/port/scorecards" "gopkg.in/yaml.v3" "k8s.io/klog/v2" - "log" "os" "sync" ) @@ -197,7 +196,7 @@ func createResources(portClient *cli.PortClient, defaults *Defaults, config *por } if err := integration.CreateIntegration(portClient, config.StateKey, config.EventListenerType, defaults.AppConfig); err != nil { - log.Printf("Failed to create resources: %v.", err.Error()) + klog.Infof("Failed to create resources: %v.", err.Error()) return &AbortDefaultCreationError{BlueprintsToRollback: createdBlueprints, Errors: []error{err}} } @@ -211,7 +210,7 @@ func initializeDefaults(portClient *cli.PortClient, config *port.Config) error { } if err := createResources(portClient, defaults, config); err != nil { - log.Printf("Failed to create resources. Rolling back blueprints: %v", err.BlueprintsToRollback) + klog.Infof("Failed to create resources. Rolling back blueprints: %v", err.BlueprintsToRollback) var rollbackWg sync.WaitGroup for _, identifier := range err.BlueprintsToRollback { rollbackWg.Add(1) From 0de6337fb70d7095dfb85f82db7f456772e86d84 Mon Sep 17 00:00:00 2001 From: yair Date: Mon, 25 Dec 2023 18:21:32 +0200 Subject: [PATCH 27/32] wrapped signal with mutex --- pkg/signal/signal.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/pkg/signal/signal.go b/pkg/signal/signal.go index 7b6ba80..ff2c225 100644 --- a/pkg/signal/signal.go +++ b/pkg/signal/signal.go @@ -4,31 +4,38 @@ import ( "fmt" "os" "os/signal" + "sync" "syscall" ) func SetupSignalHandler() (stopCh chan struct{}) { - + mutex := sync.Mutex{} stop := make(chan struct{}) gracefulStop := false shutdownCh := make(chan os.Signal, 2) signal.Notify(shutdownCh, os.Interrupt, syscall.SIGTERM) go func() { <-shutdownCh + mutex.Lock() if gracefulStop == false { fmt.Fprint(os.Stderr, "Received SIGTERM, exiting gracefully...\n") close(stop) } + mutex.Unlock() <-shutdownCh + mutex.Lock() if gracefulStop == false { fmt.Fprint(os.Stderr, "Received SIGTERM again, exiting forcefully...\n") os.Exit(1) } + mutex.Unlock() }() go func() { <-stop + mutex.Lock() gracefulStop = true + mutex.Unlock() close(shutdownCh) }() From 15668044304ad254273a118512fd7c07ad3d3eb4 Mon Sep 17 00:00:00 2001 From: yair Date: Mon, 25 Dec 2023 18:37:22 +0200 Subject: [PATCH 28/32] removed service from defaults --- assets/defaults/appConfig.yaml | 11 ----------- pkg/defaults/init.go | 3 ++- 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/assets/defaults/appConfig.yaml b/assets/defaults/appConfig.yaml index 9dec380..bf61c0d 100644 --- a/assets/defaults/appConfig.yaml +++ b/assets/defaults/appConfig.yaml @@ -102,16 +102,5 @@ resources: relations: Namespace: .metadata.namespace + "-" + env.CLUSTER_NAME title: .metadata.name - selector: - query: .metadata.namespace | startswith("kube") | not - - kind: apps/v1/deployments - port: - entity: - mappings: - - blueprint: '"service"' - icon: '"Deployment"' - identifier: .metadata.name + "-Deployment-" + .metadata.namespace + "-" + env.CLUSTER_NAME - properties: {} - title: .metadata.name selector: query: .metadata.namespace | startswith("kube") | not \ No newline at end of file diff --git a/pkg/defaults/init.go b/pkg/defaults/init.go index 7320a3f..36435df 100644 --- a/pkg/defaults/init.go +++ b/pkg/defaults/init.go @@ -19,7 +19,8 @@ func getEventListenerConfig(eventListenerType string) *port.EventListenerSetting func InitIntegration(portClient *cli.PortClient, applicationConfig *port.Config) error { existingIntegration, err := integration.GetIntegration(portClient, applicationConfig.StateKey) defaultIntegrationConfig := &port.IntegrationAppConfig{ - Resources: applicationConfig.Resources, + Resources: applicationConfig.Resources, + CreateMissingRelatedEntities: true, } if err != nil { From 5d78b8448a17889844cca19ec3615f0747ab5364 Mon Sep 17 00:00:00 2001 From: yair Date: Mon, 25 Dec 2023 18:42:43 +0200 Subject: [PATCH 29/32] removed service from defaults --- pkg/defaults/init.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/defaults/init.go b/pkg/defaults/init.go index 36435df..7320a3f 100644 --- a/pkg/defaults/init.go +++ b/pkg/defaults/init.go @@ -19,8 +19,7 @@ func getEventListenerConfig(eventListenerType string) *port.EventListenerSetting func InitIntegration(portClient *cli.PortClient, applicationConfig *port.Config) error { existingIntegration, err := integration.GetIntegration(portClient, applicationConfig.StateKey) defaultIntegrationConfig := &port.IntegrationAppConfig{ - Resources: applicationConfig.Resources, - CreateMissingRelatedEntities: true, + Resources: applicationConfig.Resources, } if err != nil { From 161e6ecfe21b9a934a210019a6ded94f26f524f4 Mon Sep 17 00:00:00 2001 From: yair Date: Tue, 26 Dec 2023 11:11:25 +0200 Subject: [PATCH 30/32] overriding configs --- main.go | 5 ++++- pkg/config/config.go | 28 +++++++++++++++++++--------- pkg/config/models.go | 4 ++++ 3 files changed, 27 insertions(+), 10 deletions(-) diff --git a/main.go b/main.go index 466f9d7..8478f89 100644 --- a/main.go +++ b/main.go @@ -37,7 +37,10 @@ func main() { klog.InitFlags(nil) k8sConfig := k8s.NewKubeConfig() - applicationConfig := config.NewConfiguration() + applicationConfig, err := config.NewConfiguration() + if err != nil { + klog.Fatalf("Error getting application config: %s", err.Error()) + } clientConfig, err := k8sConfig.ClientConfig() if err != nil { diff --git a/pkg/config/config.go b/pkg/config/config.go index d1e21da..fc011db 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -1,8 +1,10 @@ package config import ( + "encoding/json" "errors" "flag" + "fmt" "github.com/joho/godotenv" "github.com/port-labs/port-k8s-exporter/pkg/port" ) @@ -37,20 +39,28 @@ func Init() { flag.Parse() } -func NewConfiguration() *port.Config { +func NewConfiguration() (*port.Config, error) { + overrides := &port.Config{ + StateKey: ApplicationConfig.StateKey, + EventListenerType: ApplicationConfig.EventListenerType, + CreateDefaultResources: ApplicationConfig.CreateDefaultResources, + ResyncInterval: ApplicationConfig.ResyncInterval, + } + c, err := GetConfigFile(ApplicationConfig.ConfigFilePath) var fileNotFoundError *FileNotFoundError if errors.As(err, &fileNotFoundError) { - c = &port.Config{ - StateKey: ApplicationConfig.StateKey, - EventListenerType: ApplicationConfig.EventListenerType, - CreateDefaultResources: ApplicationConfig.CreateDefaultResources, - } + return overrides, nil + } + v, err := json.Marshal(overrides) + if err != nil { + return nil, fmt.Errorf("failed loading configuration: %w", err) } - if ApplicationConfig.ResyncInterval != 0 { - c.ResyncInterval = ApplicationConfig.ResyncInterval + err = json.Unmarshal(v, &c) + if err != nil { + return nil, fmt.Errorf("failed loading configuration: %w", err) } - return c + return c, nil } diff --git a/pkg/config/models.go b/pkg/config/models.go index 88d3e10..8722390 100644 --- a/pkg/config/models.go +++ b/pkg/config/models.go @@ -1,5 +1,7 @@ package config +import "github.com/port-labs/port-k8s-exporter/pkg/port" + type KafkaConfiguration struct { Brokers string SecurityProtocol string @@ -19,4 +21,6 @@ type ApplicationConfiguration struct { PortClientSecret string EventListenerType string CreateDefaultResources bool + // Deprecated: use IntegrationAppConfig instead. Used for updating the Port integration config on startup. + Resources []port.Resource } From 50f60c287859c75cbd49b0ab25bf4ebb9a2db188 Mon Sep 17 00:00:00 2001 From: yair Date: Tue, 26 Dec 2023 17:57:23 +0200 Subject: [PATCH 31/32] statekey to lower --- pkg/config/config.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/config/config.go b/pkg/config/config.go index fc011db..309a6d6 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -7,6 +7,7 @@ import ( "fmt" "github.com/joho/godotenv" "github.com/port-labs/port-k8s-exporter/pkg/port" + "strings" ) var KafkaConfig = &KafkaConfiguration{} @@ -62,5 +63,7 @@ func NewConfiguration() (*port.Config, error) { return nil, fmt.Errorf("failed loading configuration: %w", err) } + c.StateKey = strings.ToLower(c.StateKey) + return c, nil } From 7e18cab1ee3c7e0a72b0443c30aaf4997a93f18a Mon Sep 17 00:00:00 2001 From: yair Date: Wed, 27 Dec 2023 13:37:48 +0200 Subject: [PATCH 32/32] supporting delete dependents deprecated --- pkg/config/config.go | 14 ++++++++++---- pkg/config/models.go | 4 ++++ pkg/defaults/init.go | 12 +++++------- pkg/handlers/controllers.go | 4 ---- pkg/port/models.go | 6 +++++- 5 files changed, 24 insertions(+), 16 deletions(-) diff --git a/pkg/config/config.go b/pkg/config/config.go index 309a6d6..aa753be 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -37,15 +37,21 @@ func Init() { NewString(&ApplicationConfig.PortClientSecret, "port-client-secret", "", "Port client secret. Required.") NewBool(&ApplicationConfig.CreateDefaultResources, "create-default-resources", true, "Create default resources on installation. Optional.") + // Deprecated + NewBool(&ApplicationConfig.DeleteDependents, "delete-dependents", false, "Delete dependents. Optional.") + NewBool(&ApplicationConfig.CreateMissingRelatedEntities, "create-missing-related-entities", false, "Create missing related entities. Optional.") + flag.Parse() } func NewConfiguration() (*port.Config, error) { overrides := &port.Config{ - StateKey: ApplicationConfig.StateKey, - EventListenerType: ApplicationConfig.EventListenerType, - CreateDefaultResources: ApplicationConfig.CreateDefaultResources, - ResyncInterval: ApplicationConfig.ResyncInterval, + StateKey: ApplicationConfig.StateKey, + EventListenerType: ApplicationConfig.EventListenerType, + CreateDefaultResources: ApplicationConfig.CreateDefaultResources, + ResyncInterval: ApplicationConfig.ResyncInterval, + CreateMissingRelatedEntities: ApplicationConfig.CreateMissingRelatedEntities, + DeleteDependents: ApplicationConfig.DeleteDependents, } c, err := GetConfigFile(ApplicationConfig.ConfigFilePath) diff --git a/pkg/config/models.go b/pkg/config/models.go index 8722390..4626e05 100644 --- a/pkg/config/models.go +++ b/pkg/config/models.go @@ -23,4 +23,8 @@ type ApplicationConfiguration struct { CreateDefaultResources bool // Deprecated: use IntegrationAppConfig instead. Used for updating the Port integration config on startup. Resources []port.Resource + // Deprecated: use IntegrationAppConfig instead. Used for updating the Port integration config on startup. + DeleteDependents bool `json:"deleteDependents,omitempty"` + // Deprecated: use IntegrationAppConfig instead. Used for updating the Port integration config on startup. + CreateMissingRelatedEntities bool `json:"createMissingRelatedEntities,omitempty"` } diff --git a/pkg/defaults/init.go b/pkg/defaults/init.go index 7320a3f..2a04953 100644 --- a/pkg/defaults/init.go +++ b/pkg/defaults/init.go @@ -19,7 +19,9 @@ func getEventListenerConfig(eventListenerType string) *port.EventListenerSetting func InitIntegration(portClient *cli.PortClient, applicationConfig *port.Config) error { existingIntegration, err := integration.GetIntegration(portClient, applicationConfig.StateKey) defaultIntegrationConfig := &port.IntegrationAppConfig{ - Resources: applicationConfig.Resources, + Resources: applicationConfig.Resources, + DeleteDependents: applicationConfig.DeleteDependents, + CreateMissingRelatedEntities: applicationConfig.CreateMissingRelatedEntities, } if err != nil { @@ -44,12 +46,8 @@ func InitIntegration(portClient *cli.PortClient, applicationConfig *port.Config) // Handle a deprecated case where resources are provided in config file and integration exists from previous //versions without a config - if existingIntegration.Config == nil && defaultIntegrationConfig.Resources != nil { - integrationPatch.Config = &port.IntegrationAppConfig{ - DeleteDependents: defaultIntegrationConfig.DeleteDependents, - CreateMissingRelatedEntities: defaultIntegrationConfig.CreateMissingRelatedEntities, - Resources: defaultIntegrationConfig.Resources, - } + if existingIntegration.Config == nil { + integrationPatch.Config = defaultIntegrationConfig } return integration.PatchIntegration(portClient, applicationConfig.StateKey, integrationPatch) diff --git a/pkg/handlers/controllers.go b/pkg/handlers/controllers.go index 83437e4..4c14395 100644 --- a/pkg/handlers/controllers.go +++ b/pkg/handlers/controllers.go @@ -50,10 +50,6 @@ func NewControllersHandler(exporterConfig *port.Config, portConfig *port.Integra controllers = append(controllers, controller) } - if len(controllers) == 0 { - klog.Fatalf("Failed to initiate a controller for all resources, exiting...") - } - controllersHandler := &ControllersHandler{ controllers: controllers, informersFactory: informersFactory, diff --git a/pkg/port/models.go b/pkg/port/models.go index 87c1198..fa2c952 100644 --- a/pkg/port/models.go +++ b/pkg/port/models.go @@ -220,5 +220,9 @@ type Config struct { EventListenerType string CreateDefaultResources bool // Deprecated: use IntegrationAppConfig instead. Used for updating the Port integration config on startup. - Resources []Resource + Resources []Resource `json:"resources,omitempty"` + // Deprecated: use IntegrationAppConfig instead. Used for updating the Port integration config on startup. + DeleteDependents bool `json:"deleteDependents,omitempty"` + // Deprecated: use IntegrationAppConfig instead. Used for updating the Port integration config on startup. + CreateMissingRelatedEntities bool `json:"createMissingRelatedEntities,omitempty"` }