From d7e6140b99cd1d2fc09fa5f8cc9984794ae7dc5b Mon Sep 17 00:00:00 2001 From: Daniel Sinai Date: Sun, 19 May 2024 10:53:16 +0300 Subject: [PATCH] Crd autodiscovery nested schema support (#65) * feat: changed to API action v2 * feat: fixing edgecases with anyof type * feat: made tweaks for the schema so nested schemas can work * chore: added tests cases and beautified the code * chore: remove test CRD * tests: fixes * added state key * revert * chore: fix broken test * ci: remove test cache * fix: required shouldnt be hiden * fix: control the payload fixeS * chore: delete namepsace from payload after sending it * fix: spec payload * fix: fixed default jqQuery * typo Compatibility * fix: control the payload * chore: cr fixes * fix: added tests and handle required recursively * fix tests * fix: if spec not existws in custom resource avoid panic --- .github/workflows/go.yml | 5 +- go.mod | 2 +- pkg/crd/crd.go | 211 +++++++++++++++++--------------- pkg/crd/crd_test.go | 126 ++++++++++++------- pkg/crd/utils.go | 62 ++++++++++ pkg/crd/utils_test.go | 92 ++++++++++++++ pkg/defaults/defaults_test.go | 16 +-- pkg/goutils/slices.go | 11 ++ pkg/jq/parser.go | 6 +- pkg/k8s/controller_test.go | 2 +- pkg/port/blueprint/blueprint.go | 24 ---- pkg/port/cli/action.go | 26 +++- pkg/port/models.go | 46 ++++--- test_utils/cleanup.go | 14 ++- 14 files changed, 445 insertions(+), 198 deletions(-) create mode 100644 pkg/crd/utils.go create mode 100644 pkg/crd/utils_test.go create mode 100644 pkg/goutils/slices.go diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index edbc0fd..4fa3f8a 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -22,9 +22,12 @@ jobs: - name: Build run: go build -v ./... + - name: Clean Test Cache + run: go clean -testcache + - name: Test run: go test -v ./... env: PORT_CLIENT_ID: ${{ secrets.PORT_CLIENT_ID }} PORT_CLIENT_SECRET: ${{ secrets.PORT_CLIENT_SECRET }} - PORT_BASE_URL: https://api.stg-01.getport.io + PORT_BASE_URL: https://api.stg-01.getport.io \ No newline at end of file diff --git a/go.mod b/go.mod index 68610e6..de8383b 100644 --- a/go.mod +++ b/go.mod @@ -2,7 +2,7 @@ module github.com/port-labs/port-k8s-exporter go 1.22.0 -toolchain go1.22.2 +toolchain go1.22.3 require ( github.com/confluentinc/confluent-kafka-go/v2 v2.2.0 diff --git a/pkg/crd/crd.go b/pkg/crd/crd.go index 296946b..44a5d68 100644 --- a/pkg/crd/crd.go +++ b/pkg/crd/crd.go @@ -19,27 +19,12 @@ import ( ) const ( - KindCRD = "CustomResourceDefinition" - K8SIcon = "Cluster" - CrossplaneIcon = "Crossplane" + KindCRD = "CustomResourceDefinition" + K8SIcon = "Cluster" + CrossplaneIcon = "Crossplane" + NestedSchemaSeparator = "__" ) -var invisibleFields = []string{ - "writeConnectionSecretToRef", - "publishConnectionDetailsTo", - "resourceRefs", - "environmentConfigRefs", - "compositeDeletePolicy", - "resourceRef", - "claimRefs", - "compositionUpdatePolicy", - "compositionRevisionSelector", - "compositionRevisionRef", - "compositionSelector", - "compositionRef", - "claimRef", -} - func createKindConfigFromCRD(crd v1.CustomResourceDefinition) port.Resource { resource := crd.Spec.Names.Kind group := crd.Spec.Group @@ -82,100 +67,122 @@ func getIconFromCRD(crd v1.CustomResourceDefinition) string { return K8SIcon } -func buildCreateAction(crd v1.CustomResourceDefinition, as *port.ActionUserInputs, apiVersionProperty port.ActionProperty, kindProperty port.ActionProperty, nameProperty port.ActionProperty, namespaceProperty port.ActionProperty, invocation port.InvocationMethod) port.Action { +func buildCreateAction(crd v1.CustomResourceDefinition, as *port.ActionUserInputs, nameProperty port.ActionProperty, namespaceProperty port.ActionProperty, invocation port.InvocationMethod) port.Action { createActionProperties := goutils.MergeMaps( as.Properties, - map[string]port.ActionProperty{"apiVersion": apiVersionProperty, "kind": kindProperty, "name": nameProperty}, + map[string]port.ActionProperty{"name": nameProperty}, ) crtAct := port.Action{ Identifier: "create_" + crd.Spec.Names.Singular, Title: "Create " + strings.Title(crd.Spec.Names.Singular), Icon: getIconFromCRD(crd), - UserInputs: port.ActionUserInputs{ - Properties: createActionProperties, - Required: append(as.Required, "name"), + Trigger: &port.Trigger{ + Type: "self-service", + Operation: "CREATE", + BlueprintIdentifier: crd.Spec.Names.Singular, + UserInputs: &port.ActionUserInputs{ + Properties: createActionProperties, + Required: append(as.Required, "name"), + }, }, Description: getDescriptionFromCRD(crd), - Trigger: "CREATE", InvocationMethod: &invocation, } if isCRDNamespacedScoped(crd) { - crtAct.UserInputs.Properties["namespace"] = namespaceProperty - crtAct.UserInputs.Required = append(crtAct.UserInputs.Required, "namespace") + crtAct.Trigger.UserInputs.Properties["namespace"] = namespaceProperty + crtAct.Trigger.UserInputs.Required = append(crtAct.Trigger.UserInputs.Required, "namespace") } return crtAct } -func buildUpdateAction(crd v1.CustomResourceDefinition, as *port.ActionUserInputs, apiVersionProperty port.ActionProperty, kindProperty port.ActionProperty, namespaceProperty port.ActionProperty, invocation port.InvocationMethod) port.Action { - if isCRDNamespacedScoped(crd) { - as.Properties["namespace"] = namespaceProperty - as.Required = append(as.Required, "namespace") - } - +func buildUpdateAction(crd v1.CustomResourceDefinition, as *port.ActionUserInputs, invocation port.InvocationMethod) port.Action { for k, v := range as.Properties { updatedStruct := v defaultMap := make(map[string]string) - defaultMap["jqQuery"] = ".entity.properties." + k + // Blueprint schema differs from the action schema, as it not shallow - this JQ pattern assign the defaults from the entity nested schema to the action shallow one + defaultMap["jqQuery"] = ".entity.properties." + strings.Replace(k, NestedSchemaSeparator, ".", -1) updatedStruct.Default = defaultMap as.Properties[k] = updatedStruct } - updateProperties := goutils.MergeMaps( - as.Properties, - map[string]port.ActionProperty{"apiVersion": apiVersionProperty, "kind": kindProperty}, - ) - updtAct := port.Action{ Identifier: "update_" + crd.Spec.Names.Singular, Title: "Update " + strings.Title(crd.Spec.Names.Singular), Icon: getIconFromCRD(crd), Description: getDescriptionFromCRD(crd), - UserInputs: port.ActionUserInputs{ - Properties: updateProperties, - Required: as.Required, + Trigger: &port.Trigger{ + Type: "self-service", + Operation: "DAY-2", + BlueprintIdentifier: crd.Spec.Names.Singular, + UserInputs: &port.ActionUserInputs{ + Properties: as.Properties, + Required: as.Required, + }, }, - Trigger: "DAY-2", InvocationMethod: &invocation, } return updtAct } -func buildDeleteAction(crd v1.CustomResourceDefinition, apiVersionProperty port.ActionProperty, kindProperty port.ActionProperty, namespaceProperty port.ActionProperty, invocation port.InvocationMethod) port.Action { +func buildDeleteAction(crd v1.CustomResourceDefinition, invocation port.InvocationMethod) port.Action { dltAct := port.Action{ Identifier: "delete_" + crd.Spec.Names.Singular, Title: "Delete " + strings.Title(crd.Spec.Names.Singular), Icon: getIconFromCRD(crd), Description: getDescriptionFromCRD(crd), - Trigger: "DELETE", - UserInputs: port.ActionUserInputs{ - Properties: map[string]port.ActionProperty{ - "apiVersion": apiVersionProperty, - "kind": kindProperty, + Trigger: &port.Trigger{ + Type: "self-service", + BlueprintIdentifier: crd.Spec.Names.Singular, + UserInputs: &port.ActionUserInputs{ + Properties: map[string]port.ActionProperty{}, }, + Operation: "DELETE", }, - InvocationMethod: &invocation, - } - if isCRDNamespacedScoped(crd) { - visible := new(bool) // Using a pointer to bool to avoid the omitempty of false values - *visible = false - namespaceProperty.Visible = visible - dltAct.UserInputs.Properties["namespace"] = namespaceProperty - dltAct.UserInputs.Required = append(dltAct.UserInputs.Required, "namespace") + InvocationMethod: &invocation, } return dltAct } -func convertToPortSchema(crd v1.CustomResourceDefinition) ([]port.Action, *port.Blueprint, error) { +func adjustSchemaToPortSchemaCompatibilityLevel(spec *v1.JSONSchemaProps) { + for i, v := range spec.Properties { + switch v.Type { + case "object": + adjustSchemaToPortSchemaCompatibilityLevel(&v) + spec.Properties[i] = v + case "integer": + v.Type = "number" + v.Format = "" + spec.Properties[i] = v + case "": + if v.AnyOf != nil && len(v.AnyOf) > 0 { + possibleTypes := make([]string, 0) + for _, anyOf := range v.AnyOf { + possibleTypes = append(possibleTypes, anyOf.Type) + } + + // Prefer string over other types + if slices.Contains(possibleTypes, "string") { + v.Type = "string" + } else { + v.Type = possibleTypes[0] + } + } + spec.Properties[i] = v + } + } +} + +func convertToPortSchemas(crd v1.CustomResourceDefinition) ([]port.Action, *port.Blueprint, error) { latestCRDVersion := crd.Spec.Versions[0] - bs := &port.Schema{} + bs := &port.BlueprintSchema{} as := &port.ActionUserInputs{} notVisible := new(bool) // Using a pointer to bool to avoid the omitempty of false values *notVisible = false @@ -189,17 +196,11 @@ func convertToPortSchema(crd v1.CustomResourceDefinition) ([]port.Action, *port. spec = *latestCRDVersion.Schema.OpenAPIV3Schema } - // Convert integer types to number as Port does not yet support integers - for i, v := range spec.Properties { - if v.Type == "integer" { - v.Type = "number" - v.Format = "" - spec.Properties[i] = v - } - } + // Adjust schema to be compatible with Port schema + // Port's schema complexity is not rich as k8s, so we need to adjust some types and formats so we can bridge this gap + adjustSchemaToPortSchemaCompatibilityLevel(&spec) bytes, err := json.Marshal(&spec) - if err != nil { return nil, nil, fmt.Errorf("error marshaling schema: %v", err) } @@ -209,11 +210,31 @@ func convertToPortSchema(crd v1.CustomResourceDefinition) ([]port.Action, *port. return nil, nil, fmt.Errorf("error unmarshaling schema into blueprint schema: %v", err) } - err = json.Unmarshal(bytes, &as) + // Make nested schemas shallow with `NestedSchemaSeparator`(__) separator + shallowedSchema := ShallowJsonSchema(&spec, NestedSchemaSeparator) + bytesNested, err := json.Marshal(&shallowedSchema) + if err != nil { + return nil, nil, fmt.Errorf("error marshaling schema: %v", err) + } + + err = json.Unmarshal(bytesNested, &as) + if err != nil { return nil, nil, fmt.Errorf("error unmarshaling schema into action schema: %v", err) } + for k, v := range as.Properties { + if !slices.Contains(as.Required, k) { + v.Visible = new(bool) + // Not required fields should not be visible, and also shouldn't be applying default values in Port's side, instead we should let k8s apply the defaults + *v.Visible = false + v.Default = nil + as.Properties[k] = v + } + + as.Properties[k] = v + } + if isCRDNamespacedScoped(crd) { bs.Properties["namespace"] = port.Property{ Type: "string", @@ -228,26 +249,6 @@ func convertToPortSchema(crd v1.CustomResourceDefinition) ([]port.Action, *port. Schema: *bs, } - // Hide fields that are not commonly needed by default, with this approach we can still let the platform engineer show them afterwards if needed - for k, v := range as.Properties { - if slices.Contains(invisibleFields, k) { - v.Visible = notVisible - as.Properties[k] = v - } - } - - apiVersionProperty := port.ActionProperty{ - Type: "string", - Visible: notVisible, - Default: crd.Spec.Group + "/" + crd.Spec.Versions[0].Name, - } - - kindProperty := port.ActionProperty{ - Type: "string", - Visible: notVisible, - Default: crd.Spec.Names.Kind, - } - nameProperty := port.ActionProperty{ Type: "string", Title: crd.Spec.Names.Singular + " Name", @@ -265,15 +266,29 @@ func convertToPortSchema(crd v1.CustomResourceDefinition) ([]port.Action, *port. Organization: "", Repository: "", Workflow: "sync-control-plane-direct.yml", - OmitPayload: false, - OmitUserInputs: true, ReportWorkflowStatus: true, + WorkflowInputs: map[string]interface{}{ + "operation": "{{.trigger.operation}}", + "triggeringUser": "{{ .trigger.by.user.email }}", + "runId": "{{ .run.id }}", + "manifest": map[string]interface{}{ + "apiVersion": crd.Spec.Group + "/" + crd.Spec.Versions[0].Name, + "kind": crd.Spec.Names.Kind, + "metadata": map[string]interface{}{ + "{{if (.entity | has(\"identifier\")) then \"name\" else null end}}": "{{.entity.\"identifier\"}}", + "{{if (.inputs | has(\"name\")) then \"name\" else null end}}": "{{.inputs.\"name\"}}", + "{{if (.entity.properties | has(\"namespace\")) then \"namespace\" else null end}}": "{{.entity.properties.\"namespace\"}}", + "{{if (.inputs | has(\"namespace\")) then \"namespace\" else null end}}": "{{.inputs.\"namespace\"}}", + }, + "spec": "{{ .inputs | to_entries | map(if .key | contains(\"__\") then .key |= split(\"__\") else . end) | reduce .[] as $item ({}; if $item.key | type == \"array\" then setpath($item.key;$item.value) else setpath([$item.key];$item.value) end) | del(.name) | del (.namespace) }}", + }, + }, } actions := []port.Action{ - buildCreateAction(crd, as, apiVersionProperty, kindProperty, nameProperty, namespaceProperty, invocation), - buildUpdateAction(crd, as, apiVersionProperty, kindProperty, namespaceProperty, invocation), - buildDeleteAction(crd, apiVersionProperty, kindProperty, namespaceProperty, invocation), + buildCreateAction(crd, as, nameProperty, namespaceProperty, invocation), + buildUpdateAction(crd, as, invocation), + buildDeleteAction(crd, invocation), } return actions, &bp, nil @@ -309,7 +324,7 @@ func handleCRD(crds []v1.CustomResourceDefinition, portConfig *port.IntegrationA for _, crd := range matchedCRDs { portConfig.Resources = append(portConfig.Resources, createKindConfigFromCRD(crd)) - actions, bp, err := convertToPortSchema(crd) + actions, bp, err := convertToPortSchemas(crd) if err != nil { klog.Errorf("Error converting CRD to Port schemas: %s", err.Error()) continue @@ -330,11 +345,11 @@ func handleCRD(crds []v1.CustomResourceDefinition, portConfig *port.IntegrationA } for _, act := range actions { - _, err = blueprint.NewBlueprintAction(portClient, bp.Identifier, act) + _, err = cli.CreateAction(portClient, act) if err != nil { if strings.Contains(err.Error(), "taken") { - if portConfig.OverwriteCRDsActions == true { - _, err = blueprint.UpdateBlueprintAction(portClient, bp.Identifier, act) + if portConfig.OverwriteCRDsActions { + _, err = cli.UpdateAction(portClient, act) if err != nil { klog.Errorf("Error updating blueprint action: %s", err.Error()) } diff --git a/pkg/crd/crd_test.go b/pkg/crd/crd_test.go index 1db8b85..93c1c42 100644 --- a/pkg/crd/crd_test.go +++ b/pkg/crd/crd_test.go @@ -1,6 +1,7 @@ package crd import ( + "slices" "testing" "github.com/port-labs/port-k8s-exporter/pkg/config" @@ -58,7 +59,27 @@ func newFixture(t *testing.T, portClientId string, portClientSecret string, user "boolProperty": { Type: "boolean", }, + "nestedProperty": { + Type: "object", + Properties: map[string]v1.JSONSchemaProps{ + "nestedStringProperty": { + Type: "string", + }, + }, + Required: []string{"nestedStringProperty"}, + }, + "anyOfProperty": { + AnyOf: []v1.JSONSchemaProps{ + { + Type: "string", + }, + { + Type: "integer", + }, + }, + }, }, + Required: []string{"stringProperty", "nestedProperty"}, }, }, }, @@ -125,6 +146,12 @@ func checkBlueprintAndActionsProperties(t *testing.T, f *Fixture, namespaced boo if bp.Schema.Properties["boolProperty"].Type != "boolean" { t.Errorf("boolProperty type is not boolean") } + if bp.Schema.Properties["anyOfProperty"].Type != "string" { + t.Errorf("anyOfProperty type is not string") + } + if bp.Schema.Properties["nestedProperty"].Type != "object" { + t.Errorf("nestedProperty type is not object") + } if namespaced { if bp.Schema.Properties["namespace"].Type != "string" { t.Errorf("namespace type is not string") @@ -136,7 +163,7 @@ func checkBlueprintAndActionsProperties(t *testing.T, f *Fixture, namespaced boo } }) - createAction, err := cli.GetAction(f.portClient, "testkind", "create_testkind") + createAction, err := cli.GetAction(f.portClient, "create_testkind") if err != nil { t.Errorf("Error getting create action: %s", err.Error()) } @@ -144,33 +171,45 @@ func checkBlueprintAndActionsProperties(t *testing.T, f *Fixture, namespaced boo if createAction == nil { t.Errorf("Create action not found") } - if *createAction.UserInputs.Properties["apiVersion"].Visible != false { - t.Errorf("apiVersion should not be visible") - } - if *createAction.UserInputs.Properties["kind"].Visible != false { - t.Errorf("kind should not be visible") - } - if createAction.UserInputs.Properties["stringProperty"].Type != "string" { + if createAction.Trigger.UserInputs.Properties["stringProperty"].Type != "string" { t.Errorf("stringProperty type is not string") } - if createAction.UserInputs.Properties["intProperty"].Type != "number" { + if createAction.Trigger.UserInputs.Properties["intProperty"].Type != "number" { t.Errorf("intProperty type is not number") } - if createAction.UserInputs.Properties["boolProperty"].Type != "boolean" { + if createAction.Trigger.UserInputs.Properties["boolProperty"].Type != "boolean" { t.Errorf("boolProperty type is not boolean") } + if createAction.Trigger.UserInputs.Properties["anyOfProperty"].Type != "string" { + t.Errorf("anyOfProperty type is not string") + } + if _, ok := createAction.Trigger.UserInputs.Properties["nestedProperty"]; ok { + t.Errorf("nestedProperty should not be present") + } + if createAction.Trigger.UserInputs.Properties["nestedProperty__nestedStringProperty"].Type != "string" { + t.Errorf("nestedProperty__nestedStringProperty type is not string") + } if namespaced { - if createAction.UserInputs.Properties["namespace"].Type != "string" { + if createAction.Trigger.UserInputs.Properties["namespace"].Type != "string" { t.Errorf("namespace type is not string") } } else { - if _, ok := createAction.UserInputs.Properties["namespace"]; ok { + if _, ok := createAction.Trigger.UserInputs.Properties["namespace"]; ok { t.Errorf("namespace should not be present") } } + if slices.Contains(createAction.Trigger.UserInputs.Required, "stringProperty") == false { + t.Errorf("stringProperty should be required") + } + if slices.Contains(createAction.Trigger.UserInputs.Required, "nestedProperty__nestedStringProperty") == false { + t.Errorf("nestedProperty__nestedStringProperty should be required") + } + if slices.Contains(createAction.Trigger.UserInputs.Required, "nestedProperty") == true { + t.Errorf("nestedProperty should not be required") + } }) - updateAction, err := cli.GetAction(f.portClient, "testkind", "update_testkind") + updateAction, err := cli.GetAction(f.portClient, "update_testkind") if err != nil { t.Errorf("Error getting update action: %s", err.Error()) } @@ -178,33 +217,39 @@ func checkBlueprintAndActionsProperties(t *testing.T, f *Fixture, namespaced boo if updateAction == nil { t.Errorf("Update action not found") } - if *updateAction.UserInputs.Properties["apiVersion"].Visible != false { - t.Errorf("apiVersion should not be visible") - } - if *updateAction.UserInputs.Properties["kind"].Visible != false { - t.Errorf("kind should not be visible") - } - if updateAction.UserInputs.Properties["stringProperty"].Type != "string" { + if updateAction.Trigger.UserInputs.Properties["stringProperty"].Type != "string" { t.Errorf("stringProperty type is not string") } - if updateAction.UserInputs.Properties["intProperty"].Type != "number" { + if updateAction.Trigger.UserInputs.Properties["intProperty"].Type != "number" { t.Errorf("intProperty type is not number") } - if updateAction.UserInputs.Properties["boolProperty"].Type != "boolean" { + if updateAction.Trigger.UserInputs.Properties["boolProperty"].Type != "boolean" { t.Errorf("boolProperty type is not boolean") } - if namespaced { - if updateAction.UserInputs.Properties["namespace"].Type != "string" { - t.Errorf("namespace type is not string") - } - } else { - if _, ok := updateAction.UserInputs.Properties["namespace"]; ok { - t.Errorf("namespace should not be present") - } + if updateAction.Trigger.UserInputs.Properties["anyOfProperty"].Type != "string" { + t.Errorf("anyOfProperty type is not string") + } + if _, ok := createAction.Trigger.UserInputs.Properties["nestedProperty"]; ok { + t.Errorf("nestedProperty should not be present") + } + if createAction.Trigger.UserInputs.Properties["nestedProperty__nestedStringProperty"].Type != "string" { + t.Errorf("nestedProperty__nestedStringProperty type is not string") + } + if _, ok := updateAction.Trigger.UserInputs.Properties["namespace"]; ok { + t.Errorf("namespace should not be present") + } + if slices.Contains(createAction.Trigger.UserInputs.Required, "stringProperty") == false { + t.Errorf("stringProperty should be required") + } + if slices.Contains(createAction.Trigger.UserInputs.Required, "nestedProperty__nestedStringProperty") == false { + t.Errorf("nestedProperty__nestedStringProperty should be required") + } + if slices.Contains(createAction.Trigger.UserInputs.Required, "nestedProperty") == true { + t.Errorf("nestedProperty should not be required") } }) - deleteAction, err := cli.GetAction(f.portClient, "testkind", "delete_testkind") + deleteAction, err := cli.GetAction(f.portClient, "delete_testkind") if err != nil { t.Errorf("Error getting delete action: %s", err.Error()) } @@ -212,18 +257,13 @@ func checkBlueprintAndActionsProperties(t *testing.T, f *Fixture, namespaced boo if deleteAction == nil { t.Errorf("Delete action not found") } - if *deleteAction.UserInputs.Properties["apiVersion"].Visible != false { - t.Errorf("apiVersion should not be visible") - } - if *deleteAction.UserInputs.Properties["kind"].Visible != false { - t.Errorf("kind should not be visible") - } + // Delete action takes the namespace using control the payload feature if namespaced { - if deleteAction.UserInputs.Properties["namespace"].Type != "string" { - t.Errorf("namespace type is not string") + if _, ok := deleteAction.Trigger.UserInputs.Properties["namespace"]; ok { + t.Errorf("namespace should not be present") } } else { - if _, ok := deleteAction.UserInputs.Properties["namespace"]; ok { + if _, ok := deleteAction.Trigger.UserInputs.Properties["namespace"]; ok { t.Errorf("namespace should not be present") } } @@ -237,7 +277,7 @@ func TestCRD_crd_autoDiscoverCRDsToActionsClusterScoped(t *testing.T) { checkBlueprintAndActionsProperties(t, f, false) - testUtils.CheckResourcesExistence(true, f.portClient, t, []string{"testkind"}, []string{}) + testUtils.CheckResourcesExistence(true, f.portClient, t, []string{"testkind"}, []string{}, []string{"create_testkind", "update_testkind", "delete_testkind"}) } func TestCRD_crd_autoDiscoverCRDsToActionsNamespaced(t *testing.T) { @@ -247,7 +287,7 @@ func TestCRD_crd_autoDiscoverCRDsToActionsNamespaced(t *testing.T) { checkBlueprintAndActionsProperties(t, f, true) - testUtils.CheckResourcesExistence(true, f.portClient, t, []string{"testkind"}, []string{}) + testUtils.CheckResourcesExistence(true, f.portClient, t, []string{"testkind"}, []string{}, []string{"create_testkind", "update_testkind", "delete_testkind"}) } func TestCRD_crd_autoDiscoverCRDsToActionsNoCRDs(t *testing.T) { @@ -255,5 +295,5 @@ func TestCRD_crd_autoDiscoverCRDsToActionsNoCRDs(t *testing.T) { AutodiscoverCRDsToActions(f.portConfig, f.apiextensionClient, f.portClient) - testUtils.CheckResourcesExistence(false, f.portClient, t, []string{"testkind"}, []string{}) + testUtils.CheckResourcesExistence(false, f.portClient, t, []string{"testkind"}, []string{}, []string{"create_testkind", "update_testkind", "delete_testkind"}) } diff --git a/pkg/crd/utils.go b/pkg/crd/utils.go new file mode 100644 index 0000000..dee1a48 --- /dev/null +++ b/pkg/crd/utils.go @@ -0,0 +1,62 @@ +package crd + +import ( + v1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" +) + +func ShallowJsonSchema(schema *v1.JSONSchemaProps, separator string) *v1.JSONSchemaProps { + clonedSchema := schema.DeepCopy() + shallowProperties(clonedSchema, "", separator, clonedSchema) + + if schema.Type == "object" { + return &v1.JSONSchemaProps{ + Type: "object", + Properties: clonedSchema.Properties, + Required: shallowRequired(schema, "", separator), + } + } + + return schema +} + +func shallowProperties(schema *v1.JSONSchemaProps, parent string, seperator string, originalSchema *v1.JSONSchemaProps) { + for k, v := range schema.Properties { + shallowedKey := k + + if parent != "" { + shallowedKey = parent + seperator + k + } + + if v.Type != "object" { + originalSchema.Properties[shallowedKey] = v + } else { + shallowProperties(&v, shallowedKey, seperator, originalSchema) + delete(originalSchema.Properties, k) + } + } +} + +// shallowRequired recursively traverses the JSONSchemaProps and returns a list of required fields with nested field names concatenated by the provided separator. +func shallowRequired(schema *v1.JSONSchemaProps, prefix, separator string) []string { + var requiredFields []string + + for _, field := range schema.Required { + if propSchema, ok := schema.Properties[field]; ok { + fullFieldName := field + if prefix != "" { + fullFieldName = prefix + separator + field + } + + if propSchema.Type == "object" { + // Recursively process nested objects but don't add the object field itself + nestedRequiredFields := shallowRequired(&propSchema, fullFieldName, separator) + requiredFields = append(requiredFields, nestedRequiredFields...) + } else { + // Add non-object fields to the required list + requiredFields = append(requiredFields, fullFieldName) + } + } + } + + return requiredFields +} diff --git a/pkg/crd/utils_test.go b/pkg/crd/utils_test.go new file mode 100644 index 0000000..32323f0 --- /dev/null +++ b/pkg/crd/utils_test.go @@ -0,0 +1,92 @@ +package crd + +import ( + "reflect" + "testing" + + v1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" +) + +func TestCRD_crd_shallowNestedSchema(t *testing.T) { + originalSchema := &v1.JSONSchemaProps{ + Type: "object", + Properties: map[string]v1.JSONSchemaProps{ + "spec": { + Type: "object", + Properties: map[string]v1.JSONSchemaProps{ + "stringProperty": { + Type: "string", + }, + "intProperty": { + Type: "integer", + }, + "boolProperty": { + Type: "boolean", + }, + "nestedProperty": { + Type: "object", + Properties: map[string]v1.JSONSchemaProps{ + "nestedStringProperty": { + Type: "string", + }, + "nestedIntProperty": { + Type: "integer", + }, + }, + Required: []string{"nestedStringProperty"}, + }, + "multiNestedProperty": { + Type: "object", + Properties: map[string]v1.JSONSchemaProps{ + "nestedObjectProperty": { + Type: "object", + Properties: map[string]v1.JSONSchemaProps{ + "nestedStringProperty": { + Type: "string", + }, + }, + Required: []string{"nestedStringProperty"}, + }, + }, + Required: []string{}, + }, + }, + Required: []string{"stringProperty", "nestedProperty", "multiNestedProperty"}, + }, + }, + Required: []string{"spec"}, + } + + shallowedSchema := ShallowJsonSchema(originalSchema, "__") + + expectedSchema := &v1.JSONSchemaProps{ + Type: "object", + Properties: map[string]v1.JSONSchemaProps{ + "spec__stringProperty": { + Type: "string", + }, + "spec__intProperty": { + Type: "integer", + }, + "spec__boolProperty": { + Type: "boolean", + }, + "spec__nestedProperty__nestedStringProperty": { + Type: "string", + }, + "spec__nestedProperty__nestedIntProperty": { + Type: "integer", + }, + "spec__multiNestedProperty__nestedObjectProperty__nestedStringProperty": { + Type: "string", + }, + }, + Required: []string{"spec__stringProperty", "spec__nestedProperty__nestedStringProperty"}, + } + + if reflect.DeepEqual(shallowedSchema, expectedSchema) { + t.Logf("Shallowed schema is as expected") + } else { + t.Errorf("Shallowed schema is not as expected") + } +} diff --git a/pkg/defaults/defaults_test.go b/pkg/defaults/defaults_test.go index c262fee..2987212 100644 --- a/pkg/defaults/defaults_test.go +++ b/pkg/defaults/defaults_test.go @@ -100,7 +100,7 @@ func Test_InitIntegration_InitDefaults_CreateDefaultResources_False(t *testing.T _, err := integration.GetIntegration(f.portClient, f.stateKey) assert.Nil(t, err) - testUtils.CheckResourcesExistence(false, f.portClient, f.t, []string{"workload", "namespace", "cluster"}, []string{"workload_overview_dashboard", "availability_scorecard_dashboard"}) + testUtils.CheckResourcesExistence(false, f.portClient, f.t, []string{"workload", "namespace", "cluster"}, []string{"workload_overview_dashboard", "availability_scorecard_dashboard"}, []string{}) } func Test_InitIntegration_BlueprintExists(t *testing.T) { @@ -108,7 +108,7 @@ func Test_InitIntegration_BlueprintExists(t *testing.T) { if _, err := blueprint.NewBlueprint(f.portClient, port.Blueprint{ Identifier: "workload", Title: "Workload", - Schema: port.Schema{ + Schema: port.BlueprintSchema{ Properties: map[string]port.Property{}, }, }); err != nil { @@ -128,7 +128,7 @@ func Test_InitIntegration_BlueprintExists(t *testing.T) { _, err = blueprint.GetBlueprint(f.portClient, "workload") assert.Nil(t, err) - testUtils.CheckResourcesExistence(false, f.portClient, f.t, []string{"namespace", "cluster"}, []string{"workload_overview_dashboard", "availability_scorecard_dashboard"}) + testUtils.CheckResourcesExistence(false, f.portClient, f.t, []string{"namespace", "cluster"}, []string{"workload_overview_dashboard", "availability_scorecard_dashboard"}, []string{}) } func Test_InitIntegration_PageExists(t *testing.T) { @@ -153,7 +153,7 @@ func Test_InitIntegration_PageExists(t *testing.T) { _, err = page.GetPage(f.portClient, "workload_overview_dashboard") assert.Nil(t, err) - testUtils.CheckResourcesExistence(false, f.portClient, f.t, []string{"workload", "namespace", "cluster"}, []string{"availability_scorecard_dashboard"}) + testUtils.CheckResourcesExistence(false, f.portClient, f.t, []string{"workload", "namespace", "cluster"}, []string{"availability_scorecard_dashboard"}, []string{}) } func Test_InitIntegration_ExistingIntegration(t *testing.T) { @@ -172,7 +172,7 @@ func Test_InitIntegration_ExistingIntegration(t *testing.T) { _, err = integration.GetIntegration(f.portClient, f.stateKey) assert.Nil(t, err) - testUtils.CheckResourcesExistence(false, f.portClient, f.t, []string{"workload", "namespace", "cluster"}, []string{"workload_overview_dashboard", "availability_scorecard_dashboard"}) + testUtils.CheckResourcesExistence(false, f.portClient, f.t, []string{"workload", "namespace", "cluster"}, []string{"workload_overview_dashboard", "availability_scorecard_dashboard"}, []string{}) } func Test_InitIntegration_LocalResourcesConfiguration(t *testing.T) { @@ -213,7 +213,7 @@ func Test_InitIntegration_LocalResourcesConfiguration(t *testing.T) { assert.Equal(t, expectedResources, i.Config.Resources) assert.Nil(t, err) - testUtils.CheckResourcesExistence(false, f.portClient, f.t, []string{"workload", "namespace", "cluster"}, []string{"workload_overview_dashboard", "availability_scorecard_dashboard"}) + testUtils.CheckResourcesExistence(false, f.portClient, f.t, []string{"workload", "namespace", "cluster"}, []string{"workload_overview_dashboard", "availability_scorecard_dashboard"}, []string{}) } func Test_InitIntegration_LocalResourcesConfiguration_ExistingIntegration_EmptyConfiguration(t *testing.T) { @@ -234,7 +234,7 @@ func Test_InitIntegration_LocalResourcesConfiguration_ExistingIntegration_EmptyC assert.Nil(t, err) assert.Equal(t, "KAFKA", i.EventListener.Type) - testUtils.CheckResourcesExistence(false, f.portClient, f.t, []string{"workload", "namespace", "cluster"}, []string{"workload_overview_dashboard", "availability_scorecard_dashboard"}) + testUtils.CheckResourcesExistence(false, f.portClient, f.t, []string{"workload", "namespace", "cluster"}, []string{"workload_overview_dashboard", "availability_scorecard_dashboard"}, []string{}) } func Test_InitIntegration_LocalResourcesConfiguration_ExistingIntegration_WithConfiguration_WithOverwriteConfigurationOnRestartFlag(t *testing.T) { @@ -280,5 +280,5 @@ func Test_InitIntegration_LocalResourcesConfiguration_ExistingIntegration_WithCo assert.Nil(t, err) assert.Equal(t, expectedConfig.Resources, i.Config.Resources) - testUtils.CheckResourcesExistence(false, f.portClient, f.t, []string{"workload", "namespace", "cluster"}, []string{"workload_overview_dashboard", "availability_scorecard_dashboard"}) + testUtils.CheckResourcesExistence(false, f.portClient, f.t, []string{"workload", "namespace", "cluster"}, []string{"workload_overview_dashboard", "availability_scorecard_dashboard"}, []string{}) } diff --git a/pkg/goutils/slices.go b/pkg/goutils/slices.go new file mode 100644 index 0000000..aad0a60 --- /dev/null +++ b/pkg/goutils/slices.go @@ -0,0 +1,11 @@ +package goutils + +func Filter[T comparable](l []T, item T) []T { + out := make([]T, 0) + for _, element := range l { + if element != item { + out = append(out, element) + } + } + return out +} diff --git a/pkg/jq/parser.go b/pkg/jq/parser.go index 1d518f2..312d735 100644 --- a/pkg/jq/parser.go +++ b/pkg/jq/parser.go @@ -110,7 +110,11 @@ func ParseMapInterface(jqQueries map[string]string, obj interface{}) (map[string if key != "*" { mapInterface[key] = queryRes } else { - mapInterface = goutils.MergeMaps(mapInterface, queryRes.(map[string]interface{})) + if _, ok := queryRes.(map[string]interface{}); ok { + mapInterface = goutils.MergeMaps(mapInterface, queryRes.(map[string]interface{})) + } else { + mapInterface[key] = queryRes + } } } diff --git a/pkg/k8s/controller_test.go b/pkg/k8s/controller_test.go index 4440789..5f1737b 100644 --- a/pkg/k8s/controller_test.go +++ b/pkg/k8s/controller_test.go @@ -229,7 +229,7 @@ func TestDeleteDeploymentSameOwner(t *testing.T) { f.runControllerSyncHandler(item, false) _, err := f.controller.portClient.ReadEntity(context.Background(), "entityWithSameOwner", "k8s-export-test-bp") - if !strings.Contains(err.Error(), "was not found") { + if err != nil && !strings.Contains(err.Error(), "was not found") { t.Errorf("expected entity to be deleted") } } diff --git a/pkg/port/blueprint/blueprint.go b/pkg/port/blueprint/blueprint.go index cf4042a..2de0bbf 100644 --- a/pkg/port/blueprint/blueprint.go +++ b/pkg/port/blueprint/blueprint.go @@ -22,30 +22,6 @@ func NewBlueprint(portClient *cli.PortClient, blueprint port.Blueprint) (*port.B return bp, nil } -func NewBlueprintAction(portClient *cli.PortClient, blueprintIdentifier string, action port.Action) (*port.Action, error) { - _, err := portClient.Authenticate(context.Background(), portClient.ClientID, portClient.ClientSecret) - if err != nil { - return nil, fmt.Errorf("error authenticating with Port: %v", err) - } - act, err := cli.CreateAction(portClient, blueprintIdentifier, action) - if err != nil { - return nil, fmt.Errorf("error creating blueprint action: %v", err) - } - return act, nil -} - -func UpdateBlueprintAction(portClient *cli.PortClient, blueprintIdentifier string, action port.Action) (*port.Action, error) { - _, err := portClient.Authenticate(context.Background(), portClient.ClientID, portClient.ClientSecret) - if err != nil { - return nil, fmt.Errorf("error authenticating with Port: %v", err) - } - act, err := cli.UpdateAction(portClient, blueprintIdentifier, action) - if err != nil { - return nil, fmt.Errorf("error updating blueprint action: %v", err) - } - return act, nil -} - func PatchBlueprint(portClient *cli.PortClient, blueprint port.Blueprint) (*port.Blueprint, error) { _, err := portClient.Authenticate(context.Background(), portClient.ClientID, portClient.ClientSecret) if err != nil { diff --git a/pkg/port/cli/action.go b/pkg/port/cli/action.go index ddcf13e..fc6d8f6 100644 --- a/pkg/port/cli/action.go +++ b/pkg/port/cli/action.go @@ -6,12 +6,12 @@ import ( "github.com/port-labs/port-k8s-exporter/pkg/port" ) -func CreateAction(portClient *PortClient, blueprintIdentifier string, action port.Action) (*port.Action, error) { +func CreateAction(portClient *PortClient, action port.Action) (*port.Action, error) { pb := &port.ResponseBody{} resp, err := portClient.Client.R(). SetResult(&pb). SetBody(action). - Post(fmt.Sprintf("v1/blueprints/%s/actions/", blueprintIdentifier)) + Post("v1/actions") if err != nil { return nil, err } @@ -21,12 +21,12 @@ func CreateAction(portClient *PortClient, blueprintIdentifier string, action por return &pb.Action, nil } -func UpdateAction(portClient *PortClient, blueprintIdentifier string, action port.Action) (*port.Action, error) { +func UpdateAction(portClient *PortClient, action port.Action) (*port.Action, error) { pb := &port.ResponseBody{} resp, err := portClient.Client.R(). SetResult(&pb). SetBody(action). - Put(fmt.Sprintf("v1/blueprints/%s/actions/%s", blueprintIdentifier, action.Identifier)) + Put(fmt.Sprintf("v1/actions/%s", action.Identifier)) if err != nil { return nil, err } @@ -36,11 +36,11 @@ func UpdateAction(portClient *PortClient, blueprintIdentifier string, action por return &pb.Action, nil } -func GetAction(portClient *PortClient, blueprintIdentifier string, actionIdentifier string) (*port.Action, error) { +func GetAction(portClient *PortClient, actionIdentifier string) (*port.Action, error) { pb := &port.ResponseBody{} resp, err := portClient.Client.R(). SetResult(&pb). - Get(fmt.Sprintf("v1/blueprints/%s/actions/%s", blueprintIdentifier, actionIdentifier)) + Get(fmt.Sprintf("v1/actions/%s", actionIdentifier)) if err != nil { return nil, err } @@ -49,3 +49,17 @@ func GetAction(portClient *PortClient, blueprintIdentifier string, actionIdentif } return &pb.Action, nil } + +func DeleteAction(portClient *PortClient, actionIdentifier string) error { + pb := &port.ResponseBody{} + resp, err := portClient.Client.R(). + SetResult(&pb). + Delete(fmt.Sprintf("v1/actions/%s", actionIdentifier)) + if err != nil { + return err + } + if !pb.OK { + return fmt.Errorf("failed to delete action, got: %s", resp.Body()) + } + return nil +} diff --git a/pkg/port/models.go b/pkg/port/models.go index 32cbd5b..dc0e64e 100644 --- a/pkg/port/models.go +++ b/pkg/port/models.go @@ -93,20 +93,19 @@ type ( Type string `json:"type,omitempty"` } - Schema struct { + BlueprintSchema struct { Properties map[string]Property `json:"properties"` Required []string `json:"required,omitempty"` } InvocationMethod struct { - Type string `json:"type,omitempty"` - Url string `json:"url,omitempty"` - Organization string `json:"org,omitempty"` - Repository string `json:"repo,omitempty"` - Workflow string `json:"workflow,omitempty"` - OmitUserInputs bool `json:"omitUserInputs,omitempty"` - OmitPayload bool `json:"omitPayload,omitempty"` - ReportWorkflowStatus bool `json:"reportWorkflowStatus,omitempty"` + Type string `json:"type,omitempty"` + Url string `json:"url,omitempty"` + Organization string `json:"org,omitempty"` + Repository string `json:"repo,omitempty"` + Workflow string `json:"workflow,omitempty"` + WorkflowInputs map[string]interface{} `json:"workflowInputs,omitempty"` + ReportWorkflowStatus bool `json:"reportWorkflowStatus,omitempty"` } ChangelogDestination struct { @@ -125,7 +124,7 @@ type ( Title string `json:"title,omitempty"` Icon string `json:"icon"` Description string `json:"description"` - Schema Schema `json:"schema"` + Schema BlueprintSchema `json:"schema"` CalculationProperties map[string]BlueprintCalculationProperty `json:"calculationProperties,omitempty"` AggregationProperties map[string]BlueprintAggregationProperty `json:"aggregationProperties,omitempty"` MirrorProperties map[string]BlueprintMirrorProperty `json:"mirrorProperties,omitempty"` @@ -141,16 +140,35 @@ type ( Widgets interface{} `json:"widgets,omitempty"` Type string `json:"type,omitempty"` } + TriggerEvent struct { + Type string `json:"type"` + BlueprintIdentifier *string `json:"blueprintIdentifier,omitempty"` + PropertyIdentifier *string `json:"propertyIdentifier,omitempty"` + } + + TriggerCondition struct { + Type string `json:"type"` + Expressions []string `json:"expressions"` + Combinator *string `json:"combinator,omitempty"` + } + Trigger struct { + Type string `json:"type"` + BlueprintIdentifier string `json:"blueprintIdentifier,omitempty"` + Operation string `json:"operation,omitempty"` + UserInputs *ActionUserInputs `json:"userInputs,omitempty"` + Event *TriggerEvent `json:"event,omitempty"` + Condition *TriggerCondition `json:"condition,omitempty"` + } Action struct { ID string `json:"id,omitempty"` - Identifier string `json:"identifier,omitempty"` - Description string `json:"description,omitempty"` + Identifier string `json:"identifier"` Title string `json:"title,omitempty"` Icon string `json:"icon,omitempty"` - UserInputs ActionUserInputs `json:"userInputs"` - Trigger string `json:"trigger"` + Description string `json:"description,omitempty"` + Trigger *Trigger `json:"trigger"` InvocationMethod *InvocationMethod `json:"invocationMethod,omitempty"` + Publish bool `json:"publish,omitempty"` } Scorecard struct { diff --git a/test_utils/cleanup.go b/test_utils/cleanup.go index 39ff260..2f12c25 100644 --- a/test_utils/cleanup.go +++ b/test_utils/cleanup.go @@ -9,7 +9,19 @@ import ( "github.com/stretchr/testify/assert" ) -func CheckResourcesExistence(shouldExist bool, portClient *cli.PortClient, t *testing.T, blueprints []string, pages []string) { +func CheckResourcesExistence(shouldExist bool, portClient *cli.PortClient, t *testing.T, blueprints []string, pages []string, actions []string) { + for _, a := range actions { + _, err := cli.GetAction(portClient, a) + if err == nil { + _ = cli.DeleteAction(portClient, a) + } + if shouldExist { + assert.Nil(t, err) + } else { + assert.NotNil(t, err) + } + } + for _, bp := range blueprints { _, err := blueprint.GetBlueprint(portClient, bp) if err == nil {