[GEN-2081]: add new config properties to destination YAML files (#2051)

This pull request introduces several changes to enhance the
configuration and user interface for Elasticsearch and Jaeger, as well
as updates to the GraphQL schema and resolver functions. The most
important changes include the addition of new configuration options for
Elasticsearch and Jaeger, updates to the GraphQL schema, and refactoring
of the codebase for better organization.

### Configuration Enhancements:
* Added new configuration options for enabling HTTP Basic Authentication
(`ELASTICSEARCH_BASIC_AUTH_ENABLED`) and TLS
(`ELASTICSEARCH_TLS_ENABLED`) in Elasticsearch
(`common/config/elasticsearch.go`,
`destinations/data/elasticsearch.yaml`).
[[1]](diffhunk://#diff-a1750e735ff9a548285036b86fd1847bcb590d06d16578225b962913e13b9990R15-R18)
[[2]](diffhunk://#diff-6860db55e31925a6172248661127568446b2948272ad4a30bd928e168f2d94e8R39-R60)
[[3]](diffhunk://#diff-6860db55e31925a6172248661127568446b2948272ad4a30bd928e168f2d94e8R69-R94)
* Updated Jaeger configuration to include a TLS enable option with
additional properties (`destinations/data/jaeger.yaml`).
[[1]](diffhunk://#diff-3162158a78625da219b83ab5bbb2a00be52bc40da1111bc86caf72e6af34eef3L24-R36)
[[2]](diffhunk://#diff-3162158a78625da219b83ab5bbb2a00be52bc40da1111bc86caf72e6af34eef3R45-R46)

### GraphQL Schema and Resolvers:
* Introduced `CustomReadDataLabel` type and updated `Field` type to
include new properties such as `renderCondition`, `hideFromReadData`,
and `customReadDataLabels` in the GraphQL schema
(`frontend/graph/schema.graphqls`).
[[1]](diffhunk://#diff-bc07b91dedd1782d9ddbbb6374ad97c7604f9a267de5174645723d863c732f80R262-R277)
[[2]](diffhunk://#diff-bc07b91dedd1782d9ddbbb6374ad97c7604f9a267de5174645723d863c732f80L577-R597)
* Updated resolver functions to handle the new properties in `Field` and
added conversion functions (`frontend/graph/conversions.go`,
`frontend/graph/schema.resolvers.go`).
[[1]](diffhunk://#diff-019434fdf7b707424898bb20db99af0451a759ddf4678f82a3fbb2cee6bd3bd8L7-R33)
[[2]](diffhunk://#diff-8e6e95029056db2c0301fc338e0ca5a04356ce5d45ee9514bbd167f2d85bae70L735-L743)

### Code Refactoring:
* Refactored import paths and package names for better organization
(`frontend/graph/conversions.go`,
`frontend/services/describe/source_describe/source_describe.go`).
[[1]](diffhunk://#diff-019434fdf7b707424898bb20db99af0451a759ddf4678f82a3fbb2cee6bd3bd8L44-R54)
[[2]](diffhunk://#diff-db977601506771b4412474ebd6f78d0d071367497d0c7711be6b8a941da44120L1-R1)

These changes improve the flexibility and functionality of the
configuration options for Elasticsearch and Jaeger, enhance the GraphQL
schema for better data handling, and organize the codebase for
maintainability.

---------

Signed-off-by: Ben Elferink <[email protected]>

common/config/elasticsearch.go

@@ -12,8 +12,10 @@ const (
 	ElasticsearchUrlKey = "ELASTICSEARCH_URL"
 	esTracesIndexKey    = "ES_TRACES_INDEX"
 	esLogsIndexKey      = "ES_LOGS_INDEX"
+	esBasicAuthKey      = "ELASTICSEARCH_BASIC_AUTH_ENABLED" // unused in this file, currently UI only (we do not want to break existing setups by requiring this boolean)
 	esUsername          = "ELASTICSEARCH_USERNAME"
 	esPassword          = "ELASTICSEARCH_PASSWORD"
+	esTlsKey            = "ELASTICSEARCH_TLS_ENABLED" // unused in this file, currently UI only (we do not want to break existing setups by requiring this boolean)
 	esCaPem             = "ELASTICSEARCH_CA_PEM"
 )
 
@@ -46,21 +48,20 @@ func (e *Elasticsearch) ModifyConfig(dest ExporterConfigurer, currentConfig *Con
 		logIndexVal = "log_index"
 	}
 
-	basicAuthUsername := dest.GetConfig()[esUsername]
-	caPem := dest.GetConfig()[esCaPem]
-
 	exporterConfig := GenericMap{
 		"endpoints":    []string{parsedURL},
 		"traces_index": traceIndexVal,
 		"logs_index":   logIndexVal,
 	}
 
+	caPem := dest.GetConfig()[esCaPem]
 	if caPem != "" {
 		exporterConfig["tls"] = GenericMap{
 			"ca_pem": caPem,
 		}
 	}
 
+	basicAuthUsername := dest.GetConfig()[esUsername]
 	if basicAuthUsername != "" {
 		exporterConfig["user"] = basicAuthUsername
 		exporterConfig["password"] = fmt.Sprintf("${%s}", esPassword)

destinations/data/elasticsearch.yaml

@@ -36,13 +36,28 @@ spec:
         type: text
         placeholder: 'log_index'
         tooltip: 'The name of the index where logs will be stored. Defaults to log_index'
+    - name: ELASTICSEARCH_BASIC_AUTH_ENABLED
+      displayName: Enable HTTP Basic Authentication
+      componentType: checkbox
+      initialValue: false
+      componentProps:
+        required: false
+      customReadDataLabels:
+        - condition: 'true'
+          title: 'Basic Auth'
+          value: 'Enabled'
+        - condition: 'false'
+          title: 'Basic Auth'
+          value: 'Disabled'
     - name: ELASTICSEARCH_USERNAME
       displayName: Username
       componentType: input
       componentProps:
         type: text
         required: false
         tooltip: 'Username used for HTTP Basic Authentication'
+      renderCondition: ['ELASTICSEARCH_BASIC_AUTH_ENABLED', '==', 'true']
+      hideFromReadData: ['ELASTICSEARCH_BASIC_AUTH_ENABLED', '==', 'false']
     - name: ELASTICSEARCH_PASSWORD
       displayName: Password
       componentType: input
@@ -51,11 +66,29 @@ spec:
         required: false
         tooltip: 'Password used for HTTP Basic Authentication'
       secret: true
+      renderCondition: ['ELASTICSEARCH_BASIC_AUTH_ENABLED', '==', 'true']
+      hideFromReadData: ['ELASTICSEARCH_BASIC_AUTH_ENABLED', '==', 'false']
+    - name: ELASTICSEARCH_TLS_ENABLED
+      displayName: Enable TLS
+      componentType: checkbox
+      initialValue: false
+      componentProps:
+        required: false
+        tooltip: 'Secure connection (Transport Layer Security)'
+      customReadDataLabels:
+        - condition: 'true'
+          title: 'TLS'
+          value: 'Encrypted'
+        - condition: 'false'
+          title: 'TLS'
+          value: 'Unencrypted'
     - name: ELASTICSEARCH_CA_PEM
       displayName: CA Certificate
       componentType: textarea
       componentProps:
         type: text
         required: false
         placeholder: '-----BEGIN CERTIFICATE-----'
-        tooltip: 'When using https, provide the CA certificate to verify the server. If empty uses system root CA'
+        tooltip: 'When using TLS, provide the CA certificate to verify the server. If empty uses system root CA'
+      renderCondition: ['ELASTICSEARCH_TLS_ENABLED', '==', 'true']
+      hideFromReadData: ['true']

destinations/data/jaeger.yaml

@@ -21,10 +21,19 @@ spec:
         type: text
         required: true
     - name: JAEGER_TLS_ENABLED
-      displayName: Enable TLS (secure connection)
+      displayName: Enable TLS
       componentType: checkbox
+      initialValue: false
       componentProps:
         required: false
+        tooltip: 'Secure connection (Transport Layer Security)'
+      customReadDataLabels:
+        - condition: 'true'
+          title: 'TLS'
+          value: 'Encrypted'
+        - condition: 'false'
+          title: 'TLS'
+          value: 'Unencrypted'
     - name: JAEGER_CA_PEM
       displayName: Certificate Authority
       componentType: textarea
@@ -33,4 +42,6 @@ spec:
         required: false
         placeholder: '-----BEGIN CERTIFICATE-----'
         tooltip: 'When using TLS, provide the CA certificate to verify the server. If empty uses system root CA'
+      renderCondition: ['JAEGER_TLS_ENABLED', '==', 'true']
+      hideFromReadData: ['true']
   testConnectionSupported: true

destinations/model.go

@@ -28,17 +28,24 @@ type Spec struct {
 			Supported bool `yaml:"supported"`
 		}
 	}
-	Fields []Field `yaml:"fields"`
-	TestConnectionSupported bool `yaml:"testConnectionSupported"`
+	Fields                  []Field `yaml:"fields"`
+	TestConnectionSupported bool    `yaml:"testConnectionSupported"`
+}
+
+type CustomReadDataLabel struct {
+	Condition string `yaml:"condition"`
+	Title     string `yaml:"title"`
+	Value     string `yaml:"value"`
 }
 
 type Field struct {
-	Name           string                 `yaml:"name"`
-	DisplayName    string                 `yaml:"displayName"`
-	VideoURL       string                 `yaml:"videoUrl"`
-	ThumbnailURL   string                 `yaml:"thumbnailUrl"`
-	ComponentType  string                 `yaml:"componentType"`
-	ComponentProps map[string]interface{} `yaml:"componentProps"`
-	Secret         bool                   `yaml:"secret"`
-	InitialValue   string                 `yaml:"initialValue"`
+	Name                 string                 `yaml:"name"`
+	DisplayName          string                 `yaml:"displayName"`
+	ComponentType        string                 `yaml:"componentType"`
+	ComponentProps       map[string]interface{} `yaml:"componentProps"`
+	Secret               bool                   `yaml:"secret"`
+	InitialValue         string                 `yaml:"initialValue"`
+	RenderCondition      []string               `yaml:"renderCondition"`
+	HideFromReadData     []string               `yaml:"hideFromReadData"`
+	CustomReadDataLabels []*CustomReadDataLabel `yaml:"customReadDataLabels"`
 }

docs/backends/elasticsearch.mdx

@@ -65,7 +65,9 @@ spec:
     ELASTICSEARCH_URL: <Elasticsearch URL>
     # ES_TRACES_INDEX: <Traces Index>
     # ES_LOGS_INDEX: <Logs Index>
+    # ELASTICSEARCH_BASIC_AUTH_ENABLED: <Enable HTTP Basic Authentication>
     # ELASTICSEARCH_USERNAME: <Username>
+    # ELASTICSEARCH_TLS_ENABLED: <Enable TLS>
     # ELASTICSEARCH_CA_PEM: <CA Certificate>
     # Note: The commented fields above are optional.
   destinationName: elasticsearch

docs/backends/jaeger.mdx

@@ -43,7 +43,7 @@ metadata:
 spec:
   data:
     JAEGER_URL: <Jaeger OTLP gRPC Endpoint>
-    # JAEGER_TLS_ENABLED: <Enable TLS (secure connection)>
+    # JAEGER_TLS_ENABLED: <Enable TLS>
     # JAEGER_CA_PEM: <Certificate Authority>
     # Note: The commented fields above are optional.
   destinationName: jaeger

frontend/endpoints/destinations.go

@@ -101,14 +101,34 @@ type GetDestinationDetailsResponse struct {
 	Fields []Field `json:"fields"`
 }
 
+type CustomReadDataLabel struct {
+	Condition string `json:"condition"`
+	Title     string `json:"title"`
+	Value     string `json:"value"`
+}
+
 type Field struct {
-	Name                string                 `json:"name"`
-	DisplayName         string                 `json:"display_name"`
-	ComponentType       string                 `json:"component_type"`
-	ComponentProperties map[string]interface{} `json:"component_properties"`
-	VideoUrl            string                 `json:"video_url,omitempty"`
-	ThumbnailURL        string                 `json:"thumbnail_url,omitempty"`
-	InitialValue        string                 `json:"initial_value,omitempty"`
+	Name                 string                 `json:"name"`
+	DisplayName          string                 `json:"display_name"`
+	ComponentType        string                 `json:"component_type"`
+	ComponentProperties  map[string]interface{} `json:"component_properties"`
+	Secret               bool                   `json:"secret,omitempty"`
+	InitialValue         string                 `json:"initial_value,omitempty"`
+	RenderCondition      []string               `json:"render_condition,omitempty"`
+	HideFromReadData     []string               `json:"hide_from_read_data,omitempty"`
+	CustomReadDataLabels []*CustomReadDataLabel `json:"custom_read_data_labels,omitempty"`
+}
+
+func convertCustomReadDataLabels(labels []*destinations.CustomReadDataLabel) []*CustomReadDataLabel {
+	var result []*CustomReadDataLabel
+	for _, label := range labels {
+		result = append(result, &CustomReadDataLabel{
+			Condition: label.Condition,
+			Title:     label.Title,
+			Value:     label.Value,
+		})
+	}
+	return result
 }
 
 func GetDestinationTypeDetails(c *gin.Context) {
@@ -124,13 +144,15 @@ func GetDestinationTypeDetails(c *gin.Context) {
 	var resp GetDestinationDetailsResponse
 	for _, field := range destTypeConfig.Spec.Fields {
 		resp.Fields = append(resp.Fields, Field{
-			Name:                field.Name,
-			DisplayName:         field.DisplayName,
-			ComponentType:       field.ComponentType,
-			ComponentProperties: field.ComponentProps,
-			VideoUrl:            field.VideoURL,
-			ThumbnailURL:        field.ThumbnailURL,
-			InitialValue:        field.InitialValue,
+			Name:                 field.Name,
+			DisplayName:          field.DisplayName,
+			ComponentType:        field.ComponentType,
+			ComponentProperties:  field.ComponentProps,
+			Secret:               field.Secret,
+			InitialValue:         field.InitialValue,
+			RenderCondition:      field.RenderCondition,
+			HideFromReadData:     field.HideFromReadData,
+			CustomReadDataLabels: convertCustomReadDataLabels(field.CustomReadDataLabels),
 		})
 	}
 

frontend/graph/conversions.go

@@ -4,32 +4,33 @@ import (
 	"time"
 
 	"github.com/odigos-io/odigos/api/odigos/v1alpha1"
-	gqlmodel "github.com/odigos-io/odigos/frontend/graph/model"
+	"github.com/odigos-io/odigos/destinations"
+	"github.com/odigos-io/odigos/frontend/graph/model"
 	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
-func k8sKindToGql(k8sResourceKind string) gqlmodel.K8sResourceKind {
+func k8sKindToGql(k8sResourceKind string) model.K8sResourceKind {
 	switch k8sResourceKind {
 	case "Deployment":
-		return gqlmodel.K8sResourceKindDeployment
+		return model.K8sResourceKindDeployment
 	case "StatefulSet":
-		return gqlmodel.K8sResourceKindStatefulSet
+		return model.K8sResourceKindStatefulSet
 	case "DaemonSet":
-		return gqlmodel.K8sResourceKindDaemonSet
+		return model.K8sResourceKindDaemonSet
 	}
 	return ""
 }
 
-func k8sConditionStatusToGql(status v1.ConditionStatus) gqlmodel.ConditionStatus {
+func k8sConditionStatusToGql(status v1.ConditionStatus) model.ConditionStatus {
 	switch status {
 	case v1.ConditionTrue:
-		return gqlmodel.ConditionStatusTrue
+		return model.ConditionStatusTrue
 	case v1.ConditionFalse:
-		return gqlmodel.ConditionStatusFalse
+		return model.ConditionStatusFalse
 	case v1.ConditionUnknown:
-		return gqlmodel.ConditionStatusUnknown
+		return model.ConditionStatusUnknown
 	}
-	return gqlmodel.ConditionStatusUnknown
+	return model.ConditionStatusUnknown
 
 }
 
@@ -41,16 +42,16 @@ func k8sLastTransitionTimeToGql(t v1.Time) *string {
 	return &str
 }
 
-func instrumentedApplicationToActualSource(instrumentedApp v1alpha1.InstrumentedApplication) *gqlmodel.K8sActualSource {
+func instrumentedApplicationToActualSource(instrumentedApp v1alpha1.InstrumentedApplication) *model.K8sActualSource {
 	// Map the container runtime details
-	var containers []*gqlmodel.SourceContainerRuntimeDetails
+	var containers []*model.SourceContainerRuntimeDetails
 	for _, container := range instrumentedApp.Spec.RuntimeDetails {
 		var otherAgentName *string
 		if container.OtherAgent != nil {
 			otherAgentName = &container.OtherAgent.Name
 		}
 
-		containers = append(containers, &gqlmodel.SourceContainerRuntimeDetails{
+		containers = append(containers, &model.SourceContainerRuntimeDetails{
 			ContainerName:  container.ContainerName,
 			Language:       string(container.Language),
 			RuntimeVersion: container.RuntimeVersion,
@@ -59,9 +60,9 @@ func instrumentedApplicationToActualSource(instrumentedApp v1alpha1.Instrumented
 	}
 
 	// Map the conditions of the application
-	var conditions []*gqlmodel.Condition
+	var conditions []*model.Condition
 	for _, condition := range instrumentedApp.Status.Conditions {
-		conditions = append(conditions, &gqlmodel.Condition{
+		conditions = append(conditions, &model.Condition{
 			Type:               condition.Type,
 			Status:             k8sConditionStatusToGql(condition.Status),
 			Reason:             &condition.Reason,
@@ -71,16 +72,28 @@ func instrumentedApplicationToActualSource(instrumentedApp v1alpha1.Instrumented
 	}
 
 	// Return the converted K8sActualSource object
-	return &gqlmodel.K8sActualSource{
+	return &model.K8sActualSource{
 		Namespace:         instrumentedApp.Namespace,
 		Kind:              k8sKindToGql(instrumentedApp.OwnerReferences[0].Kind),
 		Name:              instrumentedApp.OwnerReferences[0].Name,
 		ServiceName:       &instrumentedApp.Name,
 		NumberOfInstances: nil,
 		AutoInstrumented:  instrumentedApp.Spec.Options != nil,
-		InstrumentedApplicationDetails: &gqlmodel.InstrumentedApplicationDetails{
+		InstrumentedApplicationDetails: &model.InstrumentedApplicationDetails{
 			Containers: containers,
 			Conditions: conditions,
 		},
 	}
 }
+
+func convertCustomReadDataLabels(labels []*destinations.CustomReadDataLabel) []*model.CustomReadDataLabel {
+	var result []*model.CustomReadDataLabel
+	for _, label := range labels {
+		result = append(result, &model.CustomReadDataLabel{
+			Condition: label.Condition,
+			Title:     label.Title,
+			Value:     label.Value,
+		})
+	}
+	return result
+}