From 1f19cde13ee0245f3d5aa0ac7253b664242688bd Mon Sep 17 00:00:00 2001 From: Prashanth Dintyala Date: Wed, 30 Oct 2024 18:59:04 -0400 Subject: [PATCH] observability: add support for distributed tracing Signed-off-by: Alex Aizman --- .github/workflows/build.yml | 2 +- 3rdparty/golang/mux/mux.go | 22 +++-- README.md | 1 + ais/backend/aws.go | 3 +- ais/backend/gcp.go | 3 +- ais/daemon.go | 11 +++ ais/htcommon.go | 4 +- ais/htrun.go | 13 ++- ais/prxclu.go | 5 ++ ais/prxrev.go | 3 +- cmn/client.go | 1 - cmn/config.go | 72 ++++++++++++++++ deploy/dev/local/aisnode_config.sh | 1 + deploy/dev/local/deploy.sh | 8 +- deploy/dev/utils.sh | 70 ++++++++++++++++ docs/distributed-tracing.md | 110 ++++++++++++++++++++++++ go.mod | 11 ++- go.sum | 16 +++- scripts/clean_deploy.sh | 5 +- tracing/tracing_off.go | 24 ++++++ tracing/tracing_on.go | 130 +++++++++++++++++++++++++++++ transport/obj_test.go | 2 +- 22 files changed, 490 insertions(+), 27 deletions(-) create mode 100644 docs/distributed-tracing.md create mode 100644 tracing/tracing_off.go create mode 100644 tracing/tracing_on.go diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b207e2f205..75b3f4e7dd 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -35,7 +35,7 @@ jobs: AIS_BACKEND_PROVIDERS="aws azure gcp" MODE="debug" make node # 5) cloud backends, debug, statsd # (build with StatsD, and note that Prometheus is the default when `statsd` tag is not defined) - TAGS="aws azure gcp statsd debug" make node + TAGS="aws azure gcp statsd debug oteltracing" make node # 6) statsd, debug, nethttp (note that fasthttp is used by default) TAGS="nethttp statsd debug" make node # 7) w/ mem profile (see cmd/aisnodeprofile) diff --git a/3rdparty/golang/mux/mux.go b/3rdparty/golang/mux/mux.go index 991b4763be..1ef96c8aa5 100644 --- a/3rdparty/golang/mux/mux.go +++ b/3rdparty/golang/mux/mux.go @@ -20,6 +20,7 @@ import ( "sync" "github.com/NVIDIA/aistore/cmn/cos" + "github.com/NVIDIA/aistore/tracing" ) // ServeMux is an HTTP request multiplexer. @@ -58,10 +59,11 @@ import ( // header, stripping the port number and redirecting any request containing . or // .. elements or repeated slashes to an equivalent, cleaner URL. type ServeMux struct { - mu sync.RWMutex - m map[string]muxEntry - es []muxEntry // slice of entries sorted from longest to shortest. - hosts bool // whether any patterns contain hostnames + mu sync.RWMutex + m map[string]muxEntry + es []muxEntry // slice of entries sorted from longest to shortest. + hosts bool // whether any patterns contain hostnames + tracingEnabled bool // enable tracing } type muxEntry struct { @@ -70,7 +72,9 @@ type muxEntry struct { } // NewServeMux allocates and returns a new ServeMux. -func NewServeMux() *ServeMux { return new(ServeMux) } +func NewServeMux(enableTracing bool) *ServeMux { + return &ServeMux{tracingEnabled: enableTracing} +} // DefaultServeMux is the default ServeMux used by Serve. var DefaultServeMux = &defaultServeMux @@ -299,8 +303,12 @@ func appendSorted(es []muxEntry, e muxEntry) []muxEntry { } // HandleFunc registers the handler function for the given pattern. -func (mux *ServeMux) HandleFunc(pattern string, handler func(http.ResponseWriter, *http.Request)) { +func (mux *ServeMux) HandleFunc(pattern string, handlerFunc http.HandlerFunc) { mux.mu.Lock() - mux._handle(pattern, http.HandlerFunc(handler)) + var handler http.Handler = handlerFunc + if mux.tracingEnabled { + handler = tracing.NewTraceableHandler(handler, pattern) + } + mux._handle(pattern, handler) mux.mu.Unlock() } diff --git a/README.md b/README.md index d0fd3455e2..8ced9bd373 100644 --- a/README.md +++ b/README.md @@ -176,6 +176,7 @@ Since AIS natively supports [remote backends](/docs/providers.md), you can also - [Prometheus](/docs/prometheus.md) - [Reference: all supported metrics](/docs/metrics-reference.md) - [Observability overview: StatsD and Prometheus, logs, and CLI](/docs/metrics.md) + - [Distributed Tracing](/docs/distributed-tracing.md) - [CLI: `ais show performance`](/docs/cli/show.md) - For users and developers - [Getting started](/docs/getting_started.md) diff --git a/ais/backend/aws.go b/ais/backend/aws.go index 537aca63ca..1b73e34b4e 100644 --- a/ais/backend/aws.go +++ b/ais/backend/aws.go @@ -30,6 +30,7 @@ import ( "github.com/NVIDIA/aistore/core/meta" "github.com/NVIDIA/aistore/memsys" "github.com/NVIDIA/aistore/stats" + "github.com/NVIDIA/aistore/tracing" "github.com/aws/aws-sdk-go-v2/aws" awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http" "github.com/aws/aws-sdk-go-v2/config" @@ -832,7 +833,7 @@ func loadConfig(endpoint, profile string) (aws.Config, error) { // NOTE: The AWS SDK for Go v2, uses lower case header maps by default. cfg, err := config.LoadDefaultConfig( context.Background(), - config.WithHTTPClient(cmn.NewClient(cmn.TransportArgs{})), + config.WithHTTPClient(tracing.NewTraceableClient(cmn.NewClient(cmn.TransportArgs{}))), config.WithSharedConfigProfile(profile), ) if err != nil { diff --git a/ais/backend/gcp.go b/ais/backend/gcp.go index 22e9e11654..0ec363a682 100644 --- a/ais/backend/gcp.go +++ b/ais/backend/gcp.go @@ -24,6 +24,7 @@ import ( "github.com/NVIDIA/aistore/core" "github.com/NVIDIA/aistore/core/meta" "github.com/NVIDIA/aistore/stats" + "github.com/NVIDIA/aistore/tracing" jsoniter "github.com/json-iterator/go" "google.golang.org/api/googleapi" "google.golang.org/api/iterator" @@ -113,7 +114,7 @@ func (gsbp *gsbp) createClient(ctx context.Context) (*storage.Client, error) { } return nil, cmn.NewErrFailedTo(nil, "gcp-backend: create", "http transport", err) } - opts = append(opts, option.WithHTTPClient(&http.Client{Transport: transport})) + opts = append(opts, option.WithHTTPClient(tracing.NewTraceableClient(&http.Client{Transport: transport}))) // create HTTP client client, err := storage.NewClient(ctx, opts...) if err != nil { diff --git a/ais/daemon.go b/ais/daemon.go index 49c06c8edd..d0d8b5a503 100644 --- a/ais/daemon.go +++ b/ais/daemon.go @@ -26,6 +26,7 @@ import ( "github.com/NVIDIA/aistore/hk" "github.com/NVIDIA/aistore/space" "github.com/NVIDIA/aistore/sys" + "github.com/NVIDIA/aistore/tracing" "github.com/NVIDIA/aistore/xact/xreg" "github.com/NVIDIA/aistore/xact/xs" ) @@ -242,6 +243,10 @@ func initDaemon(version, buildTime string) cos.Runner { // aux plumbing nlog.SetTitle(title) cmn.InitErrs(p.si.Name(), nil) + + // init distributed tracing + tracing.Init(&config.Tracing, p.si, version) + return p } @@ -258,6 +263,9 @@ func initDaemon(version, buildTime string) cos.Runner { nlog.SetTitle(title) cmn.InitErrs(t.si.Name(), fs.CleanPathErr) + // init distributed tracing + tracing.Init(&config.Tracing, t.si, version) + cmn.InitObjProps2Hdr() return t @@ -326,6 +334,9 @@ func Run(version, buildTime string) int { rmain := initDaemon(version, buildTime) err := daemon.rg.runAll(rmain) + // stop traceprovider, if running. + tracing.Shutdown() + if err == nil { nlog.Infoln("Terminated OK") return 0 diff --git a/ais/htcommon.go b/ais/htcommon.go index c3aa548264..18a3cbcf36 100644 --- a/ais/htcommon.go +++ b/ais/htcommon.go @@ -634,10 +634,10 @@ func (server *netServer) shutdown(config *cmn.Config) { // interface guard var _ http.Handler = (*httpMuxers)(nil) -func newMuxers() httpMuxers { +func newMuxers(enableTracing bool) httpMuxers { m := make(httpMuxers, len(htverbs)) for _, v := range htverbs { - m[v] = mux.NewServeMux() + m[v] = mux.NewServeMux(enableTracing) } return m } diff --git a/ais/htrun.go b/ais/htrun.go index 82ff55aff4..b45c2a2a43 100644 --- a/ais/htrun.go +++ b/ais/htrun.go @@ -39,6 +39,7 @@ import ( "github.com/NVIDIA/aistore/memsys" "github.com/NVIDIA/aistore/stats" "github.com/NVIDIA/aistore/sys" + "github.com/NVIDIA/aistore/tracing" "github.com/NVIDIA/aistore/xact/xreg" jsoniter "github.com/json-iterator/go" "github.com/prometheus/client_golang/prometheus/promhttp" @@ -291,16 +292,21 @@ func (h *htrun) init(config *cmn.Config) { tcpbuf = cmn.DefaultSendRecvBufferSize // ditto: targets use AIS default when not configured } - muxers := newMuxers() + // PubNet enable tracing when configuration is set. + muxers := newMuxers(tracing.IsEnabled()) g.netServ.pub = &netServer{muxers: muxers, sndRcvBufSize: tcpbuf} g.netServ.control = g.netServ.pub // if not separately configured, intra-control net is public if config.HostNet.UseIntraControl { - muxers = newMuxers() + // TODO: for now tracing is always disabled for intra-cluster traffic. + // Allow enabling through config. + muxers = newMuxers(false /*enableTracing*/) g.netServ.control = &netServer{muxers: muxers, sndRcvBufSize: 0} } g.netServ.data = g.netServ.control // if not configured, intra-data net is intra-control if config.HostNet.UseIntraData { - muxers = newMuxers() + // TODO: for now tracing is always disabled for intra-data traffic. + // Allow enabling through config. + muxers = newMuxers(false /*enableTracing*/) g.netServ.data = &netServer{muxers: muxers, sndRcvBufSize: tcpbuf} } @@ -450,7 +456,6 @@ func mustDiffer(ip1 meta.NetInfo, port1 int, use1 bool, ip2 meta.NetInfo, port2 func (h *htrun) loadSmap() (smap *smapX, reliable bool) { smap = newSmap() loaded, err := h.owner.smap.load(smap) - if err != nil { nlog.Errorln(h.String(), "failed to load Smap:", err, "- reinitializing") return diff --git a/ais/prxclu.go b/ais/prxclu.go index 424c907930..41e68678e9 100644 --- a/ais/prxclu.go +++ b/ais/prxclu.go @@ -1114,6 +1114,11 @@ func (p *proxy) setCluCfgPersistent(w http.ResponseWriter, r *http.Request, toUp to, _ := jsoniter.Marshal(toUpdate.Auth) whingeToUpdate("config.auth", string(from), string(to)) } + if toUpdate.Tracing != nil { + from, _ := jsoniter.Marshal(cmn.GCO.Get().Tracing) + to, _ := jsoniter.Marshal(cmn.GCO.Get().Tracing) + whingeToUpdate("config.tracing", string(from), string(to)) + } // do if _, err := p.owner.config.modify(ctx); err != nil { diff --git a/ais/prxrev.go b/ais/prxrev.go index 36f5847024..c3ea6c619c 100644 --- a/ais/prxrev.go +++ b/ais/prxrev.go @@ -234,7 +234,8 @@ func (rp *reverseProxy) init() { } func (rp *reverseProxy) loadOrStore(uuid string, u *url.URL, - errHdlr func(w http.ResponseWriter, r *http.Request, err error)) *httputil.ReverseProxy { + errHdlr func(w http.ResponseWriter, r *http.Request, err error), +) *httputil.ReverseProxy { revProxyIf, exists := rp.nodes.Load(uuid) if exists { shrp := revProxyIf.(*singleRProxy) diff --git a/cmn/client.go b/cmn/client.go index 73cc474a71..1b829bbf4c 100644 --- a/cmn/client.go +++ b/cmn/client.go @@ -142,7 +142,6 @@ func NewClientTLS(cargs TransportArgs, sargs TLSArgs, intra bool) *http.Client { cos.ExitLog(err) // FATAL } transport.TLSClientConfig = tlsConfig - return &http.Client{Transport: transport, Timeout: cargs.Timeout} } diff --git a/cmn/config.go b/cmn/config.go index 2b1216db55..ffd9747860 100644 --- a/cmn/config.go +++ b/cmn/config.go @@ -96,6 +96,7 @@ type ( Mirror MirrorConf `json:"mirror" allow:"cluster"` EC ECConf `json:"ec" allow:"cluster"` Log LogConf `json:"log"` + Tracing TracingConf `json:"tracing"` Periodic PeriodConf `json:"periodic"` Timeout TimeoutConf `json:"timeout"` Client ClientConf `json:"client"` @@ -138,6 +139,7 @@ type ( EC *ECConfToSet `json:"ec,omitempty"` Log *LogConfToSet `json:"log,omitempty"` Periodic *PeriodConfToSet `json:"periodic,omitempty"` + Tracing *TracingConfToSet `json:"tracing,omitempty"` Timeout *TimeoutConfToSet `json:"timeout,omitempty"` Client *ClientConfToSet `json:"client,omitempty"` Space *SpaceConfToSet `json:"space,omitempty"` @@ -241,6 +243,46 @@ type ( StatsTime *cos.Duration `json:"stats_time,omitempty"` } + // TracingConf defines the configuration used for the OpenTelemetry (OTEL) trace exporter. + // It includes settings for enabling tracing, sampling ratio, exporter endpoint, and other + // parameters necessary for distributed tracing in AIStore. + TracingConf struct { + ExporterEndpoint string `json:"exporter_endpoint"` // gRPC exporter endpoint + ExporterAuth TraceExporterAuthConf `json:"exporter_auth,omitempty"` // exporter auth config + ServiceNamePrefix string `json:"service_name_prefix"` // service name prefix used by trace exporter + ExtraAttributes map[string]string `json:"attributes,omitempty"` // any extra-attributes to be added to traces + + // SamplerProbablityStr is the percentage of traces to be sampled, expressed as a float64. + // It's stored as a string to avoid potential floating-point precision issues during json unmarshal. + // Valid values range from 0.0 to 1.0, where 1.0 means 100% sampling. + SamplerProbablityStr string `json:"sampler_probability,omitempty"` + Enabled bool `json:"enabled"` + SkipVerify bool `json:"skip_verify"` // allow insecure exporter gRPC connection + + SamplerProbablity float64 `json:"-"` + } + + // NOTE: Updating TracingConfig requires daemon restart. + TracingConfToSet struct { + ExporterEndpoint *string `json:"exporter_endpoint,omitempty"` // gRPC exporter endpoint + ExporterAuth *TraceExporterAuthConfToSet `json:"exporter_auth,omitempty"` // exporter auth config + ServiceNamePrefix *string `json:"service_name_prefix,omitempty"` // service name used by trace exporter + ExtraAttributes map[string]string `json:"attributes,omitempty"` // any extra-attributes to be added to traces + SamplerProbablityStr *string `json:"sampler_probability,omitempty"` // percentage of traces to be sampled + Enabled *bool `json:"enabled,omitempty"` + SkipVerify *bool `json:"skip_verify,omitempty"` // allow insecure exporter gRPC connection + } + + TraceExporterAuthConf struct { + TokenHeader string `json:"token_header"` // header used to pass exporter auth token + TokenFile string `json:"token_file"` // filepath from where auth token can be obtained + } + + TraceExporterAuthConfToSet struct { + TokenHeader *string `json:"token_header,omitempty"` // header used to pass exporter auth token + TokenFile *string `json:"token_file,omitempty"` // filepath from where auth token can be obtained + } + // NOTE: StatsTime is a one important timer PeriodConf struct { StatsTime cos.Duration `json:"stats_time"` // collect and publish stats; other house-keeping @@ -691,6 +733,7 @@ var ( _ Validator = (*MemsysConf)(nil) _ Validator = (*TCBConf)(nil) _ Validator = (*WritePolicyConf)(nil) + _ Validator = (*TracingConf)(nil) _ PropsValidator = (*CksumConf)(nil) _ PropsValidator = (*SpaceConf)(nil) @@ -1758,6 +1801,35 @@ func (c *ResilverConf) String() string { return "Disabled" } +/////////////////// +// Tracing Conf // +///////////////// + +const defaultSampleProbability = 1.0 + +func (c *TracingConf) Validate() error { + if !c.Enabled { + return nil + } + if c.ExporterEndpoint == "" { + return errors.New("invalid tracing.exporter_endpoint can't be empty when tracing is enabled") + } + if c.SamplerProbablityStr == "" { + c.SamplerProbablity = defaultSampleProbability + } else { + prob, err := strconv.ParseFloat(c.SamplerProbablityStr, 64) + if err != nil { + return nil + } + c.SamplerProbablity = prob + } + return nil +} + +func (tac TraceExporterAuthConf) IsEnabled() bool { + return tac.TokenFile != "" && tac.TokenHeader != "" +} + //////////////////// // ConfigToSet // //////////////////// diff --git a/deploy/dev/local/aisnode_config.sh b/deploy/dev/local/aisnode_config.sh index 7fc4ab7d6b..53dd1b7fb3 100755 --- a/deploy/dev/local/aisnode_config.sh +++ b/deploy/dev/local/aisnode_config.sh @@ -9,6 +9,7 @@ cat > $AIS_CONF_FILE </dev/null # 4. conditionally linked backends set_env_backends -# 5. finally, /dev/loop* devices, if any +# 5. /dev/loop* devices, if any # see also: TEST_LOOPBACK_SIZE TEST_LOOPBACK_COUNT=0 create_loopbacks_or_skip +# 6. conditionally enable distributed tracing + set_env_tracing_or_skip + ### end reading STDIN ============================ 5 steps above ================================= + ## NOTE: to enable StatsD instead of Prometheus, use build tag `statsd` in the make command, as follows: ## TAGS=statsd make ... ## see docs/metrics.md and docs/prometheus.md for more information. ## -if ! AIS_BACKEND_PROVIDERS=${AIS_BACKEND_PROVIDERS} make --no-print-directory -C ${AISTORE_PATH} node; then +if ! TAGS=${TAGS} AIS_BACKEND_PROVIDERS=${AIS_BACKEND_PROVIDERS} make --no-print-directory -C ${AISTORE_PATH} node; then exit_error "failed to compile 'aisnode' binary" fi diff --git a/deploy/dev/utils.sh b/deploy/dev/utils.sh index 57ddb43461..f550591c03 100644 --- a/deploy/dev/utils.sh +++ b/deploy/dev/utils.sh @@ -197,3 +197,73 @@ rm_loopbacks() { done fi } + +set_env_tracing_or_skip() { + echo "$AIS_TRACING_ENDPOINT" + if [[ -n "${AIS_TRACING_ENDPOINT}" ]]; then + TAGS="${TAGS} oteltracing" + return + fi + + echo "Enable distributed tracings (y/n)?" + read -r enable_tracing + + ## check presence + if [[ "$enable_tracing" == "" || "$enable_tracing" == "n" ]] ; then + return + fi + + is_boolean "${enable_tracing}" + + ## check presence + if [[ "$enable_tracing" == "y" ]] ; then + TAGS="${TAGS} oteltracing" + else + return + fi + + echo "Exporter endpoint (default:'localhost:4317' jaeger)" + read -r exporter_endpoint + if [[ -z "${AIS_TRACING_ENDPOINT}" ]]; then + AIS_TRACING_ENDPOINT=${exporter_endpoint:-"localhost:4317"} + fi + + echo "Exporter auth-header (default:'')" + read -r exporter_auth_header + if [[ -z "${AIS_TRACING_AUTH_TOKEN_HEADER}" ]]; then + AIS_TRACING_AUTH_TOKEN_HEADER=${exporter_auth_header} + fi + + echo "Exporter auth-token-file (default:'')" + read -r exporter_auth_token_file + if [[ -z "${AIS_TRACING_AUTH_TOKEN_FILE}" ]]; then + AIS_TRACING_AUTH_TOKEN_FILE={exporter_auth_token_file} + fi +} + + +make_tracing_conf() { + tracing_auth_conf="" + if [[ -n "${AIS_TRACING_AUTH_TOKEN_HEADER}" && -n "${AIS_TRACING_AUTH_TOKEN_FILE}" ]]; then + tracing_auth_conf=', + "exporter_auth": { + "token_header": "'"${AIS_TRACING_AUTH_TOKEN_HEADER}"'", + "token_file": "'"${AIS_TRACING_AUTH_TOKEN_FILE}"'" + }' + fi + + tracing_conf="" + if [[ -n "${AIS_TRACING_ENDPOINT}" ]]; then + tracing_conf=' + "tracing": { + "enabled": true, + "exporter_endpoint": "'${AIS_TRACING_ENDPOINT}'", + "skip_verify": true, + "service_name_prefix": "'${AIS_TRACING_SERVICE_PREFIX:-aistore}'", + "sampler_probability": "'${AIS_TRACING_SAMPLING_PROBABILITY:-1.0}'"'${tracing_auth_conf}' + }, + ' + fi + + echo "${tracing_conf}" +} \ No newline at end of file diff --git a/docs/distributed-tracing.md b/docs/distributed-tracing.md new file mode 100644 index 0000000000..58e1fd4324 --- /dev/null +++ b/docs/distributed-tracing.md @@ -0,0 +1,110 @@ +--- +layout: post +title: Distributed Tracing +permalink: /docs/distributed-tracing +redirect_from: + - /distributed-tracing.md/ + - /docs/distributed-tracing.md/ +--- + +AIStore supports distributed tracing via OpenTelemetry (OTEL), enhancing its observability capabilities alongside existing extensive [metrics and logging features](/docs/metrics.md). +Distributed tracing enables tracking client requests across AIStore's proxy and target daemons, providing better visibility into the request flow and offering valuable performance insights + +For more details: +- [Understanding Distributed Tracing](https://opentelemetry.io/docs/concepts/observability-primer/#understanding-distributed-tracing) +- [What is OpenTelemetry](https://opentelemetry.io/docs/what-is-opentelemetry/) + +> WARNING: Enabling distributed tracing introduces slight overhead in AIStore's critical data path. Enable this feature only after carefully considering its performance impact and ensuring that the benefits of enhanced observability justify the potential trade-offs. + + +## Table of Contents + +- [Getting Started](#getting-started) + - [Example operations](#example-operations) +- [Configuration](#configuration) + - [Build AIStore with tracing](#build-aistore-with-tracing) + +## Getting Started + +In this section, we use AIStore [Local Playground](/docs/getting_started.md#local-playground) and local [Jaeger](https://www.jaegertracing.io/). This is done for purely (easy-to-use-and-repropduce) demonsration purposes. + + +> #### Pre-Requisite +> - Docker + +1. Local Jaeger setup + ```sh + docker run -d --name jaeger \ + -e COLLECTOR_OTLP_ENABLED=true \ + -p 16686:16686 \ + -p 4317:4317 \ + -p 4318:4318 \ + jaegertracing/all-in-one:latest + ``` + +2. Optionally, shutdown and cleanup [Local Playground](/docs/getting_started.md#local-playground): + + ```sh + make kill clean + ``` + +3. Deploy the cluster with AuthN enabled: + ```sh + AIS_TRACING_ENDPOINT="localhost:4317" make deploy + ``` + + This will start up an AIStore cluster with distributed-tracing enabled. + +### Example operations + +```sh +ais bucket create ais://nnn +ais put README.md ais://nnn +ais get ais://nnn/README.md /dev/null +``` + +View traces at: [http://localhost:16686](http://localhost:16686/) + +## Configuration + +Cluster-wide `tracing` configuration. For list of AIStore config options refer to [configuration.md](/docs/configuration.md). + +| Option name | Default value | Description | +|---|---|---| +| `tracing.enabled` | `false` | If true, enables distributed tracing | +| `tracing.exporter_endpoint` | `''` | OTEL exporter gRPC endpoint | +| `tracing.service_name_prefix` | `aistore` | Prefix added to OTEL service name reported by exporter | +| `tracing.attributes` | `{}` | Extra attributes to be added the traces | +| `tracing.sampler_probablity` | `1` (export all traces) | Percentage of traces to sample [0,1] | +| `tracing.skip_verify` | `false` | Allow insecure (TLS) exporter gRPC connection | +| `tracing.exporter_auth.token_header` | `''` | Request header used for exporter auth token | +| `tracing.exporter_auth.token_file` | `''` | Filepath to obtain exporter auth token | + + +Sample aistore cluster configuration: + +```json +{ + ... + "tracing": { + "enabled": true, + "exporter_endpoint": "localhost:4317", + "skip_verify": true, + "service_name_prefix": "aistore", + "sampler_probability": "1.0" + }, + ... +} +``` + +### Build AIStore with tracing + +Distributed tracing is a build time option controlled using *oteltracing* build tag. For an `aisnode` binary built without this build tag, tracing configuration is ignored. + +```sh +# build with tracing support +TAGS=oteltracing make node + +# build without tracing support +make node +``` diff --git a/go.mod b/go.mod index 110073f2bc..57e13c068b 100644 --- a/go.mod +++ b/go.mod @@ -28,6 +28,10 @@ require ( github.com/tidwall/buntdb v1.3.2 github.com/tinylib/msgp v1.2.2 github.com/valyala/fasthttp v1.56.0 + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.56.0 + go.opentelemetry.io/otel v1.31.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.31.0 + go.opentelemetry.io/otel/sdk v1.31.0 golang.org/x/crypto v0.28.0 golang.org/x/sync v0.8.0 golang.org/x/sys v0.26.0 @@ -66,6 +70,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.2 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.32.2 // indirect github.com/beorn7/perks v1.0.1 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/census-instrumentation/opencensus-proto v0.4.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cncf/xds/go v0.0.0-20240905190251-b4127c9b8d78 // indirect @@ -93,6 +98,7 @@ require ( github.com/google/uuid v1.6.0 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect github.com/googleapis/gax-go/v2 v2.13.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0 // indirect github.com/imdario/mergo v0.3.16 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/klauspost/compress v1.17.11 // indirect @@ -119,12 +125,11 @@ require ( go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/contrib/detectors/gcp v1.30.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.56.0 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.56.0 // indirect - go.opentelemetry.io/otel v1.31.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.31.0 // indirect go.opentelemetry.io/otel/metric v1.31.0 // indirect - go.opentelemetry.io/otel/sdk v1.31.0 // indirect go.opentelemetry.io/otel/sdk/metric v1.31.0 // indirect go.opentelemetry.io/otel/trace v1.31.0 // indirect + go.opentelemetry.io/proto/otlp v1.3.1 // indirect golang.org/x/net v0.30.0 // indirect golang.org/x/oauth2 v0.23.0 // indirect golang.org/x/term v0.25.0 // indirect diff --git a/go.sum b/go.sum index d32853e3a7..09c419ab63 100644 --- a/go.sum +++ b/go.sum @@ -89,6 +89,8 @@ github.com/aws/smithy-go v1.22.0 h1:uunKnWlcoL3zO7q+gG2Pk53joueEOsnNB28QdMsmiMM= github.com/aws/smithy-go v1.22.0/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/census-instrumentation/opencensus-proto v0.4.1 h1:iKLQ0xPNFxR/2hzXZMrBo8f1j86j5WHzznCCQxV/b8g= github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw= @@ -185,6 +187,8 @@ github.com/googleapis/enterprise-certificate-proxy v0.3.4 h1:XYIDZApgAnrN1c855gT github.com/googleapis/enterprise-certificate-proxy v0.3.4/go.mod h1:YKe7cfqYXjKGpGvmSg28/fFvhNzinZQm8DGnaburhGA= github.com/googleapis/gax-go/v2 v2.13.0 h1:yitjD5f7jQHhyDsnhKEBU52NdvvdSeGzlAnDPT0hH1s= github.com/googleapis/gax-go/v2 v2.13.0/go.mod h1:Z/fvTZXF8/uw7Xu5GuslPw+bplx6SS338j1Is2S+B7A= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0 h1:asbCHRVmodnJTuQ3qamDwqVOIjwqUPTYmYuemVOx+Ys= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0/go.mod h1:ggCgvZ2r7uOoQjOyu2Y1NhHmEPPzzuhWgcza5M1Ji1I= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= @@ -255,8 +259,8 @@ github.com/prometheus/common v0.60.0 h1:+V9PAREWNvJMAuJ1x1BaWl9dewMW4YrHZQbx0sJN github.com/prometheus/common v0.60.0/go.mod h1:h0LYf1R1deLSKtD4Vdg8gy4RuOvENW2J/h19V5NADQw= github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= -github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= -github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= github.com/schollz/progressbar/v2 v2.13.2/go.mod h1:6YZjqdthH6SCZKv2rqGryrxPtfmRB/DWZxSMfCXPyD8= github.com/seiflotfy/cuckoofilter v0.0.0-20240715131351-a2f2c23f1771 h1:emzAzMZ1L9iaKCTxdy3Em8Wv4ChIAGnfiz18Cda70g4= github.com/seiflotfy/cuckoofilter v0.0.0-20240715131351-a2f2c23f1771/go.mod h1:bR6DqgcAl1zTcOX8/pE2Qkj9XO00eCNqmKb7lXP8EAg= @@ -318,6 +322,10 @@ go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.56.0 h1:UP6IpuH go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.56.0/go.mod h1:qxuZLtbq5QDtdeSHsS7bcf6EH6uO6jUAgk764zd3rhM= go.opentelemetry.io/otel v1.31.0 h1:NsJcKPIW0D0H3NgzPDHmo0WW6SptzPdqg/L1zsIm2hY= go.opentelemetry.io/otel v1.31.0/go.mod h1:O0C14Yl9FgkjqcCZAsE053C13OaddMYr/hz6clDkEJE= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.31.0 h1:K0XaT3DwHAcV4nKLzcQvwAgSyisUghWoY20I7huthMk= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.31.0/go.mod h1:B5Ki776z/MBnVha1Nzwp5arlzBbE3+1jk+pGmaP5HME= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.31.0 h1:FFeLy03iVTXP6ffeN2iXrxfGsZGCjVx0/4KlizjyBwU= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.31.0/go.mod h1:TMu73/k1CP8nBUpDLc71Wj/Kf7ZS9FK5b53VapRsP9o= go.opentelemetry.io/otel/metric v1.31.0 h1:FSErL0ATQAmYHUIzSezZibnyVlft1ybhy4ozRPcF2fE= go.opentelemetry.io/otel/metric v1.31.0/go.mod h1:C3dEloVbLuYoX41KpmAhOqNriGbA+qqH6PQ5E5mUfnY= go.opentelemetry.io/otel/sdk v1.31.0 h1:xLY3abVHYZ5HSfOg3l2E5LUj2Cwva5Y7yGxnSW9H5Gk= @@ -326,6 +334,10 @@ go.opentelemetry.io/otel/sdk/metric v1.31.0 h1:i9hxxLJF/9kkvfHppyLL55aW7iIJz4Jjx go.opentelemetry.io/otel/sdk/metric v1.31.0/go.mod h1:CRInTMVvNhUKgSAMbKyTMxqOBC0zgyxzW55lZzX43Y8= go.opentelemetry.io/otel/trace v1.31.0 h1:ffjsj1aRouKewfr85U2aGagJ46+MvodynlQ1HYdmJys= go.opentelemetry.io/otel/trace v1.31.0/go.mod h1:TXZkRk7SM2ZQLtR6eoAWQFIHPvzQ06FJAsO1tJg480A= +go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0= +go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= diff --git a/scripts/clean_deploy.sh b/scripts/clean_deploy.sh index 481d9d4107..624ccb46b4 100755 --- a/scripts/clean_deploy.sh +++ b/scripts/clean_deploy.sh @@ -32,6 +32,7 @@ mountpath_cnt=5 deployment="local" remote_alias="remais" cleanup="false" +tracing="n" usage="NAME: $(basename "$0") - locally deploy AIS clusters for development @@ -55,6 +56,7 @@ OPTIONS: --https Use HTTPS (note: X509 certificates may be required) --standby When starting up, do not join cluster - wait instead for admin request (advanced usage, target-only) --transient Do not store config changes, keep all the updates in memory + --tracing Enable distributed tracing -h, --help Show this help text " @@ -81,6 +83,7 @@ while (( "$#" )); do --azure) AIS_BACKEND_PROVIDERS="${AIS_BACKEND_PROVIDERS} azure"; shift;; --gcp) AIS_BACKEND_PROVIDERS="${AIS_BACKEND_PROVIDERS} gcp"; shift;; --ht) AIS_BACKEND_PROVIDERS="${AIS_BACKEND_PROVIDERS} ht"; shift;; + --tracing) tracing="y\n${AIS_TRACING_ENDPOINT}\n${AIS_TRACING_AUTH_TOKEN_HEADER}\n${AIS_TRACING_AUTH_TOKEN_FILE}"; shift;; --loopback) loopback=$2; @@ -158,7 +161,7 @@ if [[ ${cleanup} == "true" ]]; then fi if [[ ${deployment} == "local" || ${deployment} == "all" ]]; then - echo -e "${target_cnt}\n${proxy_cnt}\n${mountpath_cnt}\nn\nn\nn\n${loopback}\n" |\ + echo -e "${target_cnt}\n${proxy_cnt}\n${mountpath_cnt}\nn\nn\nn\n${loopback}\n${tracing}\n" |\ AIS_BACKEND_PROVIDERS="${AIS_BACKEND_PROVIDERS}" make deploy "RUN_ARGS=${RUN_ARGS}" fi diff --git a/tracing/tracing_off.go b/tracing/tracing_off.go new file mode 100644 index 0000000000..f5c94fcf71 --- /dev/null +++ b/tracing/tracing_off.go @@ -0,0 +1,24 @@ +//go:build !oteltracing + +// Package tracing offers support for distributed tracing utilizing OpenTelemetry (OTEL). +/* + * Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. + */ +package tracing + +import ( + "net/http" + + "github.com/NVIDIA/aistore/cmn" + "github.com/NVIDIA/aistore/core/meta" +) + +func IsEnabled() bool { return false } + +func Init(_ *cmn.TracingConf, _ *meta.Snode, _ string) {} + +func Shutdown() {} + +func NewTraceableHandler(handler http.Handler, _ string) http.Handler { return handler } + +func NewTraceableClient(client *http.Client) *http.Client { return client } diff --git a/tracing/tracing_on.go b/tracing/tracing_on.go new file mode 100644 index 0000000000..64b9948066 --- /dev/null +++ b/tracing/tracing_on.go @@ -0,0 +1,130 @@ +//go:build oteltracing + +// Package tracing offers support for distributed tracing utilizing OpenTelemetry (OTEL). +/* + * Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. + */ +package tracing + +import ( + "context" + "net/http" + "os" + "strings" + + "github.com/NVIDIA/aistore/cmn" + "github.com/NVIDIA/aistore/cmn/cos" + "github.com/NVIDIA/aistore/cmn/nlog" + "github.com/NVIDIA/aistore/core/meta" + "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc" + "go.opentelemetry.io/otel/propagation" + "go.opentelemetry.io/otel/sdk/resource" + "go.opentelemetry.io/otel/sdk/trace" + semconv "go.opentelemetry.io/otel/semconv/v1.4.0" +) + +var tp *trace.TracerProvider + +func loadAccessToken(tokenFilePath string) string { + cos.AssertMsg(tokenFilePath != "", "token filepath cannot be empty") + data, err := os.ReadFile(tokenFilePath) + cos.AssertNoErr(err) + return strings.TrimSpace(string(data)) +} + +func newExporter(conf *cmn.TracingConf) (trace.SpanExporter, error) { + headers := map[string]string{} + if conf.ExporterAuth.IsEnabled() { + token := loadAccessToken(conf.ExporterAuth.TokenFile) + headers[conf.ExporterAuth.TokenHeader] = token + } + + options := []otlptracegrpc.Option{ + otlptracegrpc.WithHeaders(headers), + otlptracegrpc.WithEndpoint(conf.ExporterEndpoint), + otlptracegrpc.WithRetry(otlptracegrpc.RetryConfig{Enabled: true}), + } + if conf.SkipVerify { + options = append(options, otlptracegrpc.WithInsecure()) + } + + return otlptracegrpc.New(context.Background(), options...) +} + +// newResource returns a resource describing this application. +func newResource(conf *cmn.TracingConf, snode *meta.Snode, version string) *resource.Resource { + servicePrefix := strings.TrimSuffix(conf.ServiceNamePrefix, "-") + if servicePrefix == "" { + servicePrefix = "aistore" // TODO -- constant + } + serviceName := servicePrefix + "-" + snode.DaeType + attrs := []attribute.KeyValue{ + semconv.ServiceNameKey.String(serviceName), + attribute.String("version", version), + attribute.String("daemonID", snode.DaeID), + attribute.String("pod", os.Getenv("MY_POD")), // TODO: get from consts + } + for k, v := range conf.ExtraAttributes { + attrs = append(attrs, attribute.String(k, v)) + } + r, _ := resource.Merge( + resource.Default(), + resource.NewWithAttributes( + semconv.SchemaURL, + attrs..., + ), + ) + return r +} + +func IsEnabled() bool { + return tp != nil +} + +func Init(conf *cmn.TracingConf, snode *meta.Snode, version string) { + if conf == nil || !conf.Enabled { + nlog.Infof("distributed tracing not enabled") + return + } + + cos.AssertMsg(conf.ExporterEndpoint != "", "exporter endpoint can't be empty") + exp, err := newExporter(conf) + cos.AssertNoErr(err) + + tp = trace.NewTracerProvider( + trace.WithSampler(trace.ParentBased(trace.TraceIDRatioBased(*conf.SamplerProbablity))), + trace.WithBatcher(exp), + trace.WithResource(newResource(conf, snode, version)), + ) + + otel.SetTextMapPropagator( + propagation.NewCompositeTextMapPropagator( + propagation.TraceContext{}, + propagation.Baggage{}, + ), + ) + otel.SetTracerProvider(tp) +} + +func Shutdown() { + if tp != nil { + return + } + if err := tp.Shutdown(context.Background()); err != nil { + cos.ExitLog(err) + } +} + +func NewTraceableHandler(handler http.Handler, operation string) http.Handler { + return otelhttp.NewHandler(handler, operation) +} + +func NewTraceableClient(client *http.Client) *http.Client { + if IsEnabled() { + client.Transport = otelhttp.NewTransport(client.Transport) + } + return client +} diff --git a/transport/obj_test.go b/transport/obj_test.go index c1e8425a12..1d22306e42 100644 --- a/transport/obj_test.go +++ b/transport/obj_test.go @@ -94,7 +94,7 @@ func TestMain(t *testing.M) { sc := transport.Init(&dummyStatsTracker{}) go sc.Run() - objmux = mux.NewServeMux() + objmux = mux.NewServeMux(false /*enableTracing*/) path := transport.ObjURLPath("") objmux.HandleFunc(path, transport.RxAnyStream) objmux.HandleFunc(path+"/", transport.RxAnyStream)