Merge branch 'main' of https://github.com/odigos-io/odigos

odigos-io · Dec 4, 2024 · 880204f · 880204f
2 parents 2c47abd + 89aa4d0
commit 880204f
Show file tree

Hide file tree

Showing 13 changed files with 438 additions and 79 deletions.
diff --git a/README.md b/README.md
@@ -82,44 +82,48 @@ For more details, see our [quickstart guide](https://docs.odigos.io/intro).
 
 **For step-by-step instructions detailed for every destination, see these [docs](https://docs.odigos.io/backends).**
 
-### Managed
-
-
-|                         | Traces | Metrics | Logs |
-| ------------------------- | -------- | --------- | ------ |
-| New Relic               | ✅     | ✅      | ✅   |
-| Datadog                 | ✅     | ✅      | ✅   |
-| Grafana Cloud           | ✅     | ✅      | ✅   |
-| Honeycomb               | ✅     | ✅      | ✅   |
-| Chronosphere            | ✅     | ✅      |      |
-| Logz.io                 | ✅     | ✅      | ✅   |
-| qryn.cloud              | ✅     | ✅      | ✅   |
-| OpsVerse                | ✅     | ✅      | ✅   |
-| Dynatrace               | ✅     | ✅      | ✅   |
-| AWS S3                  | ✅     | ✅      | ✅   |
-| Google Cloud Monitoring | ✅     |         | ✅   |
-| Google Cloud Storage    | ✅     |         | ✅   |
-| Azure Blob Storage      | ✅     |         | ✅   |
-| Splunk                  | ✅     |         |      |
-| Lightstep               | ✅     |         |      |
-| Sentry                  | ✅     |         |      |
-| Axiom                   | ✅     |         | ✅   |
-| Sumo Logic              | ✅     | ✅      | ✅   |
-| Coralogix               | ✅     | ✅      | ✅   |
-
-### Open Source
-
-
-|               | Traces | Metrics | Logs |
-| --------------- | -------- | --------- | ------ |
-| Prometheus    |        | ✅      |      |
-| Tempo         | ✅     |         |      |
-| Loki          |        |         | ✅   |
-| Jaeger        | ✅     |         |      |
-| SigNoz        | ✅     | ✅      | ✅   |
-| qryn          | ✅     | ✅      | ✅   |
-| Elasticsearch | ✅     |         | ✅   |
-| Quickwit      | ✅     |         | ✅   |
+### Managed Destinations
+
+| Destination             | Traces | Metrics | Logs |
+|-------------------------|:------:|:-------:|:----:|
+| AppDynamics             |   ✅   |         |      |
+| Axiom                   |   ✅   |         |  ✅  |
+| AWS S3                  |   ✅   |         |  ✅  |
+| Azure Blob Storage      |   ✅   |         |  ✅  |
+| Causely                 |   ✅   |         |      |
+| Chronosphere            |   ✅   |   ✅    |      |
+| Coralogix               |   ✅   |   ✅    |  ✅  |
+| Datadog                 |   ✅   |   ✅    |  ✅  |
+| Dynatrace               |   ✅   |   ✅    |  ✅  |
+| Gigapipe                |   ✅   |         |      |
+| Google Cloud Monitoring |   ✅   |   ✅    |      |
+| Google Cloud Storage    |   ✅   |         |  ✅  |
+| Grafana Cloud           |   ✅   |   ✅    |  ✅  |
+| Honeycomb               |   ✅   |   ✅    |  ✅  |
+| Last9                   |   ✅   |   ✅    |      |
+| Lightstep               |   ✅   |         |      |
+| Logz.io                 |   ✅   |   ✅    |  ✅  |
+| New Relic               |   ✅   |   ✅    |  ✅  |
+| OpsVerse                |   ✅   |   ✅    |  ✅  |
+| Sentry                  |   ✅   |         |      |
+| Splunk                  |   ✅   |         |      |
+| Sumo Logic              |   ✅   |   ✅    |  ✅  |
+
+## Self-Hosted (Open Source) Destinations
+
+| Destination   | Traces | Metrics | Logs |
+|---------------|:------:|:-------:|:----:|
+| ClickHouse    |   ✅   |   ✅    |  ✅  |
+| Elasticsearch |   ✅   |         |  ✅  |
+| Jaeger        |   ✅   |         |      |
+| Loki          |        |         |  ✅  |
+| OTLP          |   ✅   |   ✅    |  ✅  |
+| OTLP HTTP     |   ✅   |   ✅    |  ✅  |
+| Prometheus    |        |   ✅    |      |
+| Quickwit      |   ✅   |         |      |
+| qryn          |   ✅   |   ✅    |  ✅  |
+| SigNoz        |   ✅   |   ✅    |  ✅  |
+| Tempo         |   ✅   |         |      |
 
 Can't find the destination you need? Help us by following our quick [add new destination](https://docs.odigos.io/adding-new-dest) guide and submitting a PR.
 

diff --git a/cli/cmd/install.go b/cli/cmd/install.go
@@ -12,16 +12,15 @@ import (
 
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 
+	"github.com/odigos-io/odigos/cli/cmd/resources"
+	cmdcontext "github.com/odigos-io/odigos/cli/pkg/cmd_context"
+	"github.com/odigos-io/odigos/cli/pkg/kube"
+	"github.com/odigos-io/odigos/cli/pkg/log"
 	"github.com/odigos-io/odigos/common"
 	"github.com/odigos-io/odigos/common/consts"
 	"github.com/odigos-io/odigos/common/utils"
 	k8sconsts "github.com/odigos-io/odigos/k8sutils/pkg/consts"
 
-	"github.com/odigos-io/odigos/cli/cmd/resources"
-	"github.com/odigos-io/odigos/cli/pkg/kube"
-	"github.com/odigos-io/odigos/cli/pkg/log"
-	cmdcontext "github.com/odigos-io/odigos/cli/pkg/cmd_context"
-
 	"github.com/spf13/cobra"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

diff --git a/cli/cmd/resources/odigosconfig.go b/cli/cmd/resources/odigosconfig.go
@@ -2,18 +2,18 @@ package resources
 
 import (
 	"context"
-	"encoding/json"
 
 	"github.com/odigos-io/odigos/cli/cmd/resources/resourcemanager"
 	"github.com/odigos-io/odigos/cli/pkg/kube"
 	"github.com/odigos-io/odigos/common"
 	"github.com/odigos-io/odigos/common/consts"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/yaml"
 )
 
 func NewOdigosConfiguration(ns string, config *common.OdigosConfiguration) (kube.Object, error) {
-	data, err := json.Marshal(config)
+	data, err := yaml.Marshal(config)
 	if err != nil {
 		return nil, err
 	}
@@ -47,6 +47,10 @@ func (a *odigosConfigResourceManager) Name() string { return "OdigosConfig" }
 
 func (a *odigosConfigResourceManager) InstallFromScratch(ctx context.Context) error {
 
+	sizingProfile := FilterSizeProfiles(a.config.Profiles)
+	collectorGatewayConfig := GetGatewayConfigBasedOnSize(sizingProfile)
+	a.config.CollectorGateway = collectorGatewayConfig
+
 	obj, err := NewOdigosConfiguration(a.ns, a.config)
 	if err != nil {
 		return err
@@ -57,3 +61,38 @@ func (a *odigosConfigResourceManager) InstallFromScratch(ctx context.Context) er
 	}
 	return a.client.ApplyResources(ctx, a.config.ConfigVersion, resources)
 }
+
+func GetGatewayConfigBasedOnSize(profile common.ProfileName) *common.CollectorGatewayConfiguration {
+	aggregateProfiles := append([]common.ProfileName{profile}, profilesMap[profile].Dependencies...)
+
+	for _, profile := range aggregateProfiles {
+		switch profile {
+		case sizeSProfile.ProfileName:
+			return &common.CollectorGatewayConfiguration{
+				MinReplicas:      1,
+				MaxReplicas:      5,
+				RequestCPUm:      150,
+				LimitCPUm:        300,
+				RequestMemoryMiB: 300,
+			}
+		case sizeMProfile.ProfileName:
+			return &common.CollectorGatewayConfiguration{
+				MinReplicas:      2,
+				MaxReplicas:      8,
+				RequestCPUm:      500,
+				LimitCPUm:        1000,
+				RequestMemoryMiB: 500,
+			}
+		case sizeLProfile.ProfileName:
+			return &common.CollectorGatewayConfiguration{
+				MinReplicas:      3,
+				MaxReplicas:      12,
+				RequestCPUm:      750,
+				LimitCPUm:        1250,
+				RequestMemoryMiB: 750,
+			}
+		}
+	}
+	// Return nil if no matching profile is found.
+	return nil
+}
diff --git a/cli/cmd/resources/profiles.go b/cli/cmd/resources/profiles.go
@@ -4,7 +4,6 @@ import (
 	"context"
 	"fmt"
 
-	actions "github.com/odigos-io/odigos/api/actions/v1alpha1"
 	odigosv1alpha1 "github.com/odigos-io/odigos/api/odigos/v1alpha1"
 	"github.com/odigos-io/odigos/cli/cmd/resources/profiles"
 	"github.com/odigos-io/odigos/cli/cmd/resources/resourcemanager"
@@ -20,6 +19,19 @@ type Profile struct {
 }
 
 var (
+	// sizing profiles for the collector gateway
+	sizeSProfile = Profile{
+		ProfileName:      common.ProfileName("size_s"),
+		ShortDescription: "Small size deployment profile",
+	}
+	sizeMProfile = Profile{
+		ProfileName:      common.ProfileName("size_m"),
+		ShortDescription: "Medium size deployment profile",
+	}
+	sizeLProfile = Profile{
+		ProfileName:      common.ProfileName("size_l"),
+		ShortDescription: "Large size deployment profile",
+	}
 	fullPayloadCollectionProfile = Profile{
 		ProfileName:      common.ProfileName("full-payload-collection"),
 		ShortDescription: "Collect any payload from the cluster where supported with default settings",
@@ -38,7 +50,7 @@ var (
 	semconvUpgraderProfile = Profile{
 		ProfileName:      common.ProfileName("semconv"),
 		ShortDescription: "Upgrade and align some attribute names to a newer version of the OpenTelemetry semantic conventions",
-		KubeObject:       &actions.RenameAttribute{},
+		KubeObject:       &odigosv1alpha1.Processor{},
 	}
 	categoryAttributesProfile = Profile{
 		ProfileName:      common.ProfileName("category-attributes"),
@@ -75,17 +87,37 @@ var (
 	}
 	kratosProfile = Profile{
 		ProfileName:      common.ProfileName("kratos"),
-		ShortDescription: "Bundle profile that includes db-payload-collection, semconv, category-attributes, copy-scope, hostname-as-podname, java-native-instrumentations, code-attributes, query-operation-detector",
-		Dependencies:     []common.ProfileName{"db-payload-collection", "semconv", "category-attributes", "copy-scope", "hostname-as-podname", "java-native-instrumentations", "code-attributes", "query-operation-detector", "disableNameProcessorProfile", "small-batches"},
+		ShortDescription: "Bundle profile that includes db-payload-collection, semconv, category-attributes, copy-scope, hostname-as-podname, java-native-instrumentations, code-attributes, query-operation-detector, disableNameProcessorProfile, small-batches, size_m",
+		Dependencies:     []common.ProfileName{"db-payload-collection", "semconv", "category-attributes", "copy-scope", "hostname-as-podname", "java-native-instrumentations", "code-attributes", "query-operation-detector", "disableNameProcessorProfile", "small-batches", "size_m"},
+	}
+	profilesMap = map[common.ProfileName]Profile{
+		sizeSProfile.ProfileName:                      sizeSProfile,
+		sizeMProfile.ProfileName:                      sizeMProfile,
+		sizeLProfile.ProfileName:                      sizeLProfile,
+		fullPayloadCollectionProfile.ProfileName:      fullPayloadCollectionProfile,
+		dbPayloadCollectionProfile.ProfileName:        dbPayloadCollectionProfile,
+		queryOperationDetector.ProfileName:            queryOperationDetector,
+		semconvUpgraderProfile.ProfileName:            semconvUpgraderProfile,
+		categoryAttributesProfile.ProfileName:         categoryAttributesProfile,
+		copyScopeProfile.ProfileName:                  copyScopeProfile,
+		hostnameAsPodNameProfile.ProfileName:          hostnameAsPodNameProfile,
+		javaNativeInstrumentationsProfile.ProfileName: javaNativeInstrumentationsProfile,
+		codeAttributesProfile.ProfileName:             codeAttributesProfile,
+		disableNameProcessorProfile.ProfileName:       disableNameProcessorProfile,
+		smallBatchesProfile.ProfileName:               smallBatchesProfile,
+		kratosProfile.ProfileName:                     kratosProfile,
 	}
 )
 
 func GetAvailableCommunityProfiles() []Profile {
-	return []Profile{semconvUpgraderProfile, copyScopeProfile, disableNameProcessorProfile}
+	return []Profile{semconvUpgraderProfile, copyScopeProfile, disableNameProcessorProfile, sizeSProfile, sizeMProfile,
+		sizeLProfile}
 }
 
 func GetAvailableOnPremProfiles() []Profile {
-	return append([]Profile{fullPayloadCollectionProfile, dbPayloadCollectionProfile, categoryAttributesProfile, hostnameAsPodNameProfile, javaNativeInstrumentationsProfile, kratosProfile, queryOperationDetector, smallBatchesProfile},
+	return append([]Profile{fullPayloadCollectionProfile, dbPayloadCollectionProfile, categoryAttributesProfile,
+		hostnameAsPodNameProfile, javaNativeInstrumentationsProfile, kratosProfile, queryOperationDetector,
+		smallBatchesProfile},
 		GetAvailableCommunityProfiles()...)
 }
 
@@ -153,3 +185,25 @@ func (a *profilesResourceManager) InstallFromScratch(ctx context.Context) error
 	}
 	return a.client.ApplyResources(ctx, a.config.ConfigVersion, allResources)
 }
+
+func FilterSizeProfiles(profiles []common.ProfileName) common.ProfileName {
+	// In case multiple size profiles are provided, the first one will be used.
+
+	for _, profile := range profiles {
+		// Check if the profile is a size profile.
+		switch profile {
+		case sizeSProfile.ProfileName, sizeMProfile.ProfileName, sizeLProfile.ProfileName:
+			return profile
+		}
+
+		// Check if the profile has a dependency which is a size profile.
+		profileDependencies := profilesMap[profile].Dependencies
+		for _, dependencyProfile := range profileDependencies {
+			switch dependencyProfile {
+			case sizeSProfile.ProfileName, sizeMProfile.ProfileName, sizeLProfile.ProfileName:
+				return dependencyProfile
+			}
+		}
+	}
+	return ""
+}