Skip to content

Commit

Permalink
Instrument opentelemetry (#4357)
Browse files Browse the repository at this point in the history 
* ported over flytestdlib otelutils additions

Signed-off-by: Daniel Rammer <[email protected]>

* starting on single binary

Signed-off-by: Daniel Rammer <[email protected]>

* instrumented flytepropeller

Signed-off-by: Daniel Rammer <[email protected]>

* fixed semconv issue

Signed-off-by: Daniel Rammer <[email protected]>

* spans propagating from propeller through admin grpc server to admin gorm

Signed-off-by: Daniel Rammer <[email protected]>

* filled out admin gorm instrumentation

Signed-off-by: Daniel Rammer <[email protected]>

* instrumented datacatalog with grpc and gorm

Signed-off-by: Daniel Rammer <[email protected]>

* registering tracerproviders for flyteadmin and datacatalog standalone

Signed-off-by: Daniel Rammer <[email protected]>

* instrumenting k8s client and cache

Signed-off-by: Daniel Rammer <[email protected]>

* fixed propeller compile

Signed-off-by: Daniel Rammer <[email protected]>

* updated flytepropeller cmd

Signed-off-by: Daniel Rammer <[email protected]>

* updated flytestdlib deps

Signed-off-by: Daniel Rammer <[email protected]>

* updated deps

Signed-off-by: Daniel Rammer <[email protected]>

* freezing versions in datacatalog and flyteadmin

Signed-off-by: Daniel Rammer <[email protected]>

* go mod tidy on flyteidl

Signed-off-by: Daniel Rammer <[email protected]>

* lock version of gorm

Signed-off-by: Daniel Rammer <[email protected]>

* updated config to use type parameter

Signed-off-by: Daniel Rammer <[email protected]>

* failure on unknown exporter type

Signed-off-by: Daniel Rammer <[email protected]>

* correct import

Signed-off-by: Daniel Rammer <[email protected]>

* removed Enabled flags in favor of a single ExporterType in config

Signed-off-by: Daniel Rammer <[email protected]>

---------

Signed-off-by: Daniel Rammer <[email protected]>
Signed-off-by: Eduardo Apolinario <[email protected]>
Co-authored-by: Eduardo Apolinario <[email protected]>
Co-authored-by: Kevin Su <[email protected]>
  • Loading branch information
@hamersaw @eapolinario @pingsutw
3 people authored Nov 27, 2023
1 parent ea72bbd commit 33b0c6e
Show file tree
Hide file tree
Showing 53 changed files with 1,706 additions and 1,207 deletions.
26 changes: 24 additions & 2 deletions cmd/single/start.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,12 +17,12 @@ import (
adminScheduler "github.com/flyteorg/flyte/flyteadmin/scheduler"
propellerEntrypoint "github.com/flyteorg/flyte/flytepropeller/pkg/controller"
propellerConfig "github.com/flyteorg/flyte/flytepropeller/pkg/controller/config"
"github.com/flyteorg/flyte/flytepropeller/pkg/controller/executors"
"github.com/flyteorg/flyte/flytepropeller/pkg/signals"
webhookEntrypoint "github.com/flyteorg/flyte/flytepropeller/pkg/webhook"
webhookConfig "github.com/flyteorg/flyte/flytepropeller/pkg/webhook/config"
"github.com/flyteorg/flyte/flytestdlib/contextutils"
"github.com/flyteorg/flyte/flytestdlib/logger"
"github.com/flyteorg/flyte/flytestdlib/otelutils"
"github.com/flyteorg/flyte/flytestdlib/profutils"
"github.com/flyteorg/flyte/flytestdlib/promutils"
"github.com/flyteorg/flyte/flytestdlib/promutils/labeled"
Expand Down Expand Up @@ -120,8 +120,21 @@ func startPropeller(ctx context.Context, cfg Propeller) error {
SyncPeriod: &propellerCfg.DownstreamEval.Duration,
DefaultNamespaces: namespaceConfigs,
},
NewCache: func (config *rest.Config, options cache.Options) (cache.Cache, error) {
k8sCache, err := cache.New(config, options)
if err != nil {
return k8sCache, err
}

return otelutils.WrapK8sCache(k8sCache), nil
},
NewClient: func(config *rest.Config, options client.Options) (client.Client, error) {
return executors.NewFallbackClientBuilder(propellerScope.NewSubScope("kube")).Build(nil, config, options)
k8sClient, err := client.New(config, options)
if err != nil {
return k8sClient, err
}

return otelutils.WrapK8sClient(k8sClient), nil
},
Metrics: metricsserver.Options{
// Disable metrics serving
Expand Down Expand Up @@ -191,6 +204,15 @@ var startCmd = &cobra.Command{
g, childCtx := errgroup.WithContext(ctx)
cfg := GetConfig()

for _, serviceName := range []string{otelutils.AdminClientTracer, otelutils.AdminGormTracer, otelutils.AdminServerTracer,
otelutils.BlobstoreClientTracer, otelutils.DataCatalogClientTracer, otelutils.DataCatalogGormTracer,
otelutils.DataCatalogServerTracer, otelutils.FlytePropellerTracer, otelutils.K8sClientTracer} {
if err := otelutils.RegisterTracerProvider(serviceName, otelutils.GetConfig()) ; err != nil {
logger.Errorf(ctx, "Failed to create otel tracer provider. %v", err)
return err
}
}

if !cfg.Admin.Disabled {
g.Go(func() error {
err := startAdmin(childCtx, cfg.Admin)
Expand Down
9 changes: 9 additions & 0 deletions datacatalog/cmd/entrypoints/serve.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
"github.com/flyteorg/flyte/datacatalog/pkg/runtime"
"github.com/flyteorg/flyte/flytestdlib/contextutils"
"github.com/flyteorg/flyte/flytestdlib/logger"
"github.com/flyteorg/flyte/flytestdlib/otelutils"
"github.com/flyteorg/flyte/flytestdlib/profutils"
"github.com/flyteorg/flyte/flytestdlib/promutils/labeled"
)
Expand Down Expand Up @@ -42,6 +43,14 @@ var serveCmd = &cobra.Command{
// Set Keys
labeled.SetMetricKeys(contextutils.AppNameKey, contextutils.ProjectKey, contextutils.DomainKey)

// register otel tracer providers
for _, serviceName := range []string{otelutils.DataCatalogGormTracer, otelutils.DataCatalogServerTracer} {
if err := otelutils.RegisterTracerProvider(serviceName, otelutils.GetConfig()); err != nil {
logger.Errorf(ctx, "Failed to create otel tracer provider. %v", err)
return err
}
}

return datacatalogservice.ServeInsecure(ctx, cfg)
},
}
Expand Down
70 changes: 52 additions & 18 deletions datacatalog/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,13 +11,16 @@ require (
github.com/golang/protobuf v1.5.3
github.com/jackc/pgconn v1.10.1
github.com/mitchellh/mapstructure v1.5.0
github.com/spf13/cobra v1.4.0
github.com/spf13/cobra v1.7.0
github.com/spf13/pflag v1.0.5
github.com/stretchr/testify v1.8.4
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.35.0
go.opentelemetry.io/otel v1.19.0
google.golang.org/grpc v1.56.1
gorm.io/driver/postgres v1.2.3
gorm.io/driver/sqlite v1.1.1
gorm.io/gorm v1.22.4
gorm.io/driver/sqlite v1.5.0
gorm.io/gorm v1.25.1
gorm.io/plugin/opentelemetry v0.1.4
)

require (
Expand All @@ -42,19 +45,28 @@ require (
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/coocood/freecache v1.1.1 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/emicklei/go-restful/v3 v3.9.0 // indirect
github.com/evanphx/json-patch/v5 v5.6.0 // indirect
github.com/fatih/color v1.13.0 // indirect
github.com/flyteorg/stow v0.3.7 // indirect
github.com/fsnotify/fsnotify v1.5.1 // indirect
github.com/fsnotify/fsnotify v1.6.0 // indirect
github.com/ghodss/yaml v1.0.0 // indirect
github.com/go-logr/logr v1.2.4 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-openapi/jsonpointer v0.19.6 // indirect
github.com/go-openapi/jsonreference v0.20.2 // indirect
github.com/go-openapi/swag v0.22.3 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang-jwt/jwt/v4 v4.4.1 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/google/gnostic-models v0.6.8 // indirect
github.com/google/go-cmp v0.5.9 // indirect
github.com/google/gofuzz v1.2.0 // indirect
github.com/google/uuid v1.3.0 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect
github.com/googleapis/gax-go/v2 v2.7.1 // indirect
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/inconshreveable/mousetrap v1.0.0 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/jackc/chunkreader/v2 v2.0.1 // indirect
github.com/jackc/pgio v1.0.0 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
Expand All @@ -63,48 +75,70 @@ require (
github.com/jackc/pgtype v1.9.0 // indirect
github.com/jackc/pgx/v4 v4.14.0 // indirect
github.com/jinzhu/inflection v1.0.0 // indirect
github.com/jinzhu/now v1.1.4 // indirect
github.com/jinzhu/now v1.1.5 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/magiconair/properties v1.8.6 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/mattn/go-colorable v0.1.12 // indirect
github.com/mattn/go-isatty v0.0.14 // indirect
github.com/mattn/go-sqlite3 v1.14.0 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
github.com/mattn/go-sqlite3 v1.14.15 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/ncw/swift v1.0.53 // indirect
github.com/pelletier/go-toml v1.9.4 // indirect
github.com/pelletier/go-toml/v2 v2.0.0-beta.8 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/client_golang v1.12.1 // indirect
github.com/prometheus/client_model v0.2.0 // indirect
github.com/prometheus/common v0.32.1 // indirect
github.com/prometheus/procfs v0.7.3 // indirect
github.com/sirupsen/logrus v1.7.0 // indirect
github.com/prometheus/client_golang v1.16.0 // indirect
github.com/prometheus/client_model v0.4.0 // indirect
github.com/prometheus/common v0.44.0 // indirect
github.com/prometheus/procfs v0.10.1 // indirect
github.com/sirupsen/logrus v1.9.2 // indirect
github.com/spf13/afero v1.9.2 // indirect
github.com/spf13/cast v1.4.1 // indirect
github.com/spf13/jwalterweatherman v1.1.0 // indirect
github.com/spf13/viper v1.11.0 // indirect
github.com/stretchr/objx v0.5.0 // indirect
github.com/subosito/gotenv v1.2.0 // indirect
go.opencensus.io v0.24.0 // indirect
go.opentelemetry.io/otel/exporters/jaeger v1.17.0 // indirect
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.19.0 // indirect
go.opentelemetry.io/otel/metric v1.19.0 // indirect
go.opentelemetry.io/otel/sdk v1.19.0 // indirect
go.opentelemetry.io/otel/trace v1.19.0 // indirect
golang.org/x/crypto v0.11.0 // indirect
golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e // indirect
golang.org/x/net v0.13.0 // indirect
golang.org/x/oauth2 v0.8.0 // indirect
golang.org/x/sys v0.10.0 // indirect
golang.org/x/sys v0.12.0 // indirect
golang.org/x/term v0.10.0 // indirect
golang.org/x/text v0.11.0 // indirect
golang.org/x/time v0.3.0 // indirect
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
google.golang.org/api v0.114.0 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect
google.golang.org/genproto v0.0.0-20230526161137-0005af68ea54 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20230526161137-0005af68ea54 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20230526161137-0005af68ea54 // indirect
google.golang.org/protobuf v1.30.0 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/ini.v1 v1.66.4 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
k8s.io/api v0.28.2 // indirect
k8s.io/apimachinery v0.28.2 // indirect
k8s.io/client-go v0.0.0-20210217172142-7279fc64d847 // indirect
k8s.io/client-go v0.28.1 // indirect
k8s.io/klog/v2 v2.100.1 // indirect
k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 // indirect
k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 // indirect
sigs.k8s.io/controller-runtime v0.0.0-00010101000000-000000000000 // indirect
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
sigs.k8s.io/yaml v1.3.0 // indirect
)

replace (
Expand All @@ -114,11 +148,11 @@ replace (
github.com/flyteorg/flyte/flyteplugins => ../flyteplugins
github.com/flyteorg/flyte/flytepropeller => ../flytepropeller
github.com/flyteorg/flyte/flytestdlib => ../flytestdlib

gorm.io/driver/sqlite => gorm.io/driver/sqlite v1.1.1
gorm.io/gorm => gorm.io/gorm v1.22.4
k8s.io/api => k8s.io/api v0.28.2
k8s.io/apimachinery => k8s.io/apimachinery v0.28.2
k8s.io/client-go => k8s.io/client-go v0.28.2
k8s.io/kube-openapi => k8s.io/kube-openapi v0.0.0-20230905202853-d090da108d2f
sigs.k8s.io/controller-runtime => sigs.k8s.io/controller-runtime v0.16.2

)
Loading

0 comments on commit 33b0c6e

Please sign in to comment.