Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Instrument opentelemetry #4357

Merged
merged 31 commits into from
Nov 27, 2023
Merged

Instrument opentelemetry #4357

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
31 commits
Select commit Hold shift + click to select a range
a8f8fc0
ported over flytestdlib otelutils additions
hamersaw Nov 3, 2023
46299f5
starting on single binary
hamersaw Nov 3, 2023
9f248d1
instrumented flytepropeller
hamersaw Nov 3, 2023
4f8a8f1
fixed semconv issue
hamersaw Nov 3, 2023
af5edff
spans propagating from propeller through admin grpc server to admin gorm
hamersaw Nov 3, 2023
e2d5b53
filled out admin gorm instrumentation
hamersaw Nov 3, 2023
d4e91db
instrumented datacatalog with grpc and gorm
hamersaw Nov 4, 2023
77cda4b
registering tracerproviders for flyteadmin and datacatalog standalone
hamersaw Nov 6, 2023
236da85
instrumenting k8s client and cache
hamersaw Nov 6, 2023
d9de0a8
fixed propeller compile
hamersaw Nov 6, 2023
74aad7c
updated flytepropeller cmd
hamersaw Nov 6, 2023
be7b07e
updated flytestdlib deps
hamersaw Nov 6, 2023
af9db8e
updated deps
hamersaw Nov 6, 2023
1925620
freezing versions in datacatalog and flyteadmin
hamersaw Nov 7, 2023
67467ad
Merge branch 'master' into feature/open-telemetry
hamersaw Nov 7, 2023
74d06e7
go mod tidy on flyteidl
hamersaw Nov 7, 2023
acf81e7
lock version of gorm
hamersaw Nov 8, 2023
dcf0ecb
Merge remote-tracking branch 'origin' into feature/open-telemetry
eapolinario Nov 8, 2023
f90c2a4
updated config to use type parameter
hamersaw Nov 8, 2023
0cbece9
Merge branch 'feature/open-telemetry' of https://github.com/flyteorg/…
hamersaw Nov 8, 2023
3789e33
failure on unknown exporter type
hamersaw Nov 8, 2023
adcc428
correct import
hamersaw Nov 8, 2023
c341f9d
Merge remote-tracking branch 'origin' into feature/open-telemetry
eapolinario Nov 8, 2023
ebddf82
removed Enabled flags in favor of a single ExporterType in config
hamersaw Nov 8, 2023
765be1c
Merge branch 'master' into feature/open-telemetry
hamersaw Nov 9, 2023
d402776
Merge branch 'master' into feature/open-telemetry
hamersaw Nov 10, 2023
8e7b44d
Merge branch 'master' into feature/open-telemetry
hamersaw Nov 11, 2023
92f6549
Merge branch 'master' into feature/open-telemetry
hamersaw Nov 13, 2023
13cbfa2
Merge branch 'master' into feature/open-telemetry
hamersaw Nov 15, 2023
d6cc0f5
Merge branch 'master' into feature/open-telemetry
pingsutw Nov 18, 2023
66606f2
Merge branch 'master' into feature/open-telemetry
hamersaw Nov 27, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 24 additions & 2 deletions cmd/single/start.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,12 +17,12 @@ import (
adminScheduler "github.com/flyteorg/flyte/flyteadmin/scheduler"
propellerEntrypoint "github.com/flyteorg/flyte/flytepropeller/pkg/controller"
propellerConfig "github.com/flyteorg/flyte/flytepropeller/pkg/controller/config"
"github.com/flyteorg/flyte/flytepropeller/pkg/controller/executors"
"github.com/flyteorg/flyte/flytepropeller/pkg/signals"
webhookEntrypoint "github.com/flyteorg/flyte/flytepropeller/pkg/webhook"
webhookConfig "github.com/flyteorg/flyte/flytepropeller/pkg/webhook/config"
"github.com/flyteorg/flyte/flytestdlib/contextutils"
"github.com/flyteorg/flyte/flytestdlib/logger"
"github.com/flyteorg/flyte/flytestdlib/otelutils"
"github.com/flyteorg/flyte/flytestdlib/profutils"
"github.com/flyteorg/flyte/flytestdlib/promutils"
"github.com/flyteorg/flyte/flytestdlib/promutils/labeled"
Expand Down Expand Up @@ -120,8 +120,21 @@ func startPropeller(ctx context.Context, cfg Propeller) error {
SyncPeriod: &propellerCfg.DownstreamEval.Duration,
DefaultNamespaces: namespaceConfigs,
},
NewCache: func (config *rest.Config, options cache.Options) (cache.Cache, error) {
k8sCache, err := cache.New(config, options)
if err != nil {
return k8sCache, err
}

return otelutils.WrapK8sCache(k8sCache), nil
},
NewClient: func(config *rest.Config, options client.Options) (client.Client, error) {
return executors.NewFallbackClientBuilder(propellerScope.NewSubScope("kube")).Build(nil, config, options)
k8sClient, err := client.New(config, options)
if err != nil {
return k8sClient, err
}

return otelutils.WrapK8sClient(k8sClient), nil
},
Metrics: metricsserver.Options{
// Disable metrics serving
Expand Down Expand Up @@ -191,6 +204,15 @@ var startCmd = &cobra.Command{
g, childCtx := errgroup.WithContext(ctx)
cfg := GetConfig()

for _, serviceName := range []string{otelutils.AdminClientTracer, otelutils.AdminGormTracer, otelutils.AdminServerTracer,
otelutils.BlobstoreClientTracer, otelutils.DataCatalogClientTracer, otelutils.DataCatalogGormTracer,
otelutils.DataCatalogServerTracer, otelutils.FlytePropellerTracer, otelutils.K8sClientTracer} {
if err := otelutils.RegisterTracerProvider(serviceName, otelutils.GetConfig()) ; err != nil {
logger.Errorf(ctx, "Failed to create otel tracer provider. %v", err)
return err
}
}

if !cfg.Admin.Disabled {
g.Go(func() error {
err := startAdmin(childCtx, cfg.Admin)
Expand Down
9 changes: 9 additions & 0 deletions datacatalog/cmd/entrypoints/serve.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
"github.com/flyteorg/flyte/datacatalog/pkg/runtime"
"github.com/flyteorg/flyte/flytestdlib/contextutils"
"github.com/flyteorg/flyte/flytestdlib/logger"
"github.com/flyteorg/flyte/flytestdlib/otelutils"
"github.com/flyteorg/flyte/flytestdlib/profutils"
"github.com/flyteorg/flyte/flytestdlib/promutils/labeled"
)
Expand Down Expand Up @@ -42,6 +43,14 @@ var serveCmd = &cobra.Command{
// Set Keys
labeled.SetMetricKeys(contextutils.AppNameKey, contextutils.ProjectKey, contextutils.DomainKey)

// register otel tracer providers
for _, serviceName := range []string{otelutils.DataCatalogGormTracer, otelutils.DataCatalogServerTracer} {
if err := otelutils.RegisterTracerProvider(serviceName, otelutils.GetConfig()); err != nil {
logger.Errorf(ctx, "Failed to create otel tracer provider. %v", err)
return err
}
}

return datacatalogservice.ServeInsecure(ctx, cfg)
},
}
Expand Down
70 changes: 52 additions & 18 deletions datacatalog/go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,13 +11,16 @@ require (
github.com/golang/protobuf v1.5.3
github.com/jackc/pgconn v1.10.1
github.com/mitchellh/mapstructure v1.5.0
github.com/spf13/cobra v1.4.0
github.com/spf13/cobra v1.7.0
github.com/spf13/pflag v1.0.5
github.com/stretchr/testify v1.8.4
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.35.0
go.opentelemetry.io/otel v1.19.0
google.golang.org/grpc v1.56.1
gorm.io/driver/postgres v1.2.3
gorm.io/driver/sqlite v1.1.1
gorm.io/gorm v1.22.4
gorm.io/driver/sqlite v1.5.0
gorm.io/gorm v1.25.1
gorm.io/plugin/opentelemetry v0.1.4
)

require (
Expand All @@ -42,19 +45,28 @@ require (
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/coocood/freecache v1.1.1 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/emicklei/go-restful/v3 v3.9.0 // indirect
github.com/evanphx/json-patch/v5 v5.6.0 // indirect
github.com/fatih/color v1.13.0 // indirect
github.com/flyteorg/stow v0.3.7 // indirect
github.com/fsnotify/fsnotify v1.5.1 // indirect
github.com/fsnotify/fsnotify v1.6.0 // indirect
github.com/ghodss/yaml v1.0.0 // indirect
github.com/go-logr/logr v1.2.4 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-openapi/jsonpointer v0.19.6 // indirect
github.com/go-openapi/jsonreference v0.20.2 // indirect
github.com/go-openapi/swag v0.22.3 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang-jwt/jwt/v4 v4.4.1 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/google/gnostic-models v0.6.8 // indirect
github.com/google/go-cmp v0.5.9 // indirect
github.com/google/gofuzz v1.2.0 // indirect
github.com/google/uuid v1.3.0 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect
github.com/googleapis/gax-go/v2 v2.7.1 // indirect
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/inconshreveable/mousetrap v1.0.0 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/jackc/chunkreader/v2 v2.0.1 // indirect
github.com/jackc/pgio v1.0.0 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
Expand All @@ -63,48 +75,70 @@ require (
github.com/jackc/pgtype v1.9.0 // indirect
github.com/jackc/pgx/v4 v4.14.0 // indirect
github.com/jinzhu/inflection v1.0.0 // indirect
github.com/jinzhu/now v1.1.4 // indirect
github.com/jinzhu/now v1.1.5 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/magiconair/properties v1.8.6 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/mattn/go-colorable v0.1.12 // indirect
github.com/mattn/go-isatty v0.0.14 // indirect
github.com/mattn/go-sqlite3 v1.14.0 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
github.com/mattn/go-sqlite3 v1.14.15 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/ncw/swift v1.0.53 // indirect
github.com/pelletier/go-toml v1.9.4 // indirect
github.com/pelletier/go-toml/v2 v2.0.0-beta.8 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/client_golang v1.12.1 // indirect
github.com/prometheus/client_model v0.2.0 // indirect
github.com/prometheus/common v0.32.1 // indirect
github.com/prometheus/procfs v0.7.3 // indirect
github.com/sirupsen/logrus v1.7.0 // indirect
github.com/prometheus/client_golang v1.16.0 // indirect
github.com/prometheus/client_model v0.4.0 // indirect
github.com/prometheus/common v0.44.0 // indirect
github.com/prometheus/procfs v0.10.1 // indirect
github.com/sirupsen/logrus v1.9.2 // indirect
github.com/spf13/afero v1.9.2 // indirect
github.com/spf13/cast v1.4.1 // indirect
github.com/spf13/jwalterweatherman v1.1.0 // indirect
github.com/spf13/viper v1.11.0 // indirect
github.com/stretchr/objx v0.5.0 // indirect
github.com/subosito/gotenv v1.2.0 // indirect
go.opencensus.io v0.24.0 // indirect
go.opentelemetry.io/otel/exporters/jaeger v1.17.0 // indirect
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.19.0 // indirect
go.opentelemetry.io/otel/metric v1.19.0 // indirect
go.opentelemetry.io/otel/sdk v1.19.0 // indirect
go.opentelemetry.io/otel/trace v1.19.0 // indirect
golang.org/x/crypto v0.11.0 // indirect
golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e // indirect
golang.org/x/net v0.13.0 // indirect
golang.org/x/oauth2 v0.8.0 // indirect
golang.org/x/sys v0.10.0 // indirect
golang.org/x/sys v0.12.0 // indirect
golang.org/x/term v0.10.0 // indirect
golang.org/x/text v0.11.0 // indirect
golang.org/x/time v0.3.0 // indirect
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
google.golang.org/api v0.114.0 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect
google.golang.org/genproto v0.0.0-20230526161137-0005af68ea54 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20230526161137-0005af68ea54 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20230526161137-0005af68ea54 // indirect
google.golang.org/protobuf v1.30.0 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/ini.v1 v1.66.4 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
k8s.io/api v0.28.2 // indirect
k8s.io/apimachinery v0.28.2 // indirect
k8s.io/client-go v0.0.0-20210217172142-7279fc64d847 // indirect
k8s.io/client-go v0.28.1 // indirect
k8s.io/klog/v2 v2.100.1 // indirect
k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 // indirect
k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 // indirect
sigs.k8s.io/controller-runtime v0.0.0-00010101000000-000000000000 // indirect
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
sigs.k8s.io/yaml v1.3.0 // indirect
)

replace (
Expand All @@ -114,11 +148,11 @@ replace (
github.com/flyteorg/flyte/flyteplugins => ../flyteplugins
github.com/flyteorg/flyte/flytepropeller => ../flytepropeller
github.com/flyteorg/flyte/flytestdlib => ../flytestdlib

gorm.io/driver/sqlite => gorm.io/driver/sqlite v1.1.1
gorm.io/gorm => gorm.io/gorm v1.22.4
k8s.io/api => k8s.io/api v0.28.2
k8s.io/apimachinery => k8s.io/apimachinery v0.28.2
k8s.io/client-go => k8s.io/client-go v0.28.2
k8s.io/kube-openapi => k8s.io/kube-openapi v0.0.0-20230905202853-d090da108d2f
sigs.k8s.io/controller-runtime => sigs.k8s.io/controller-runtime v0.16.2

)
Loading
Loading