diff --git a/README.md b/README.md index 97e566a..36f04ff 100644 --- a/README.md +++ b/README.md @@ -353,12 +353,9 @@ certbot renew --- - -> Yaml file fields - BenchMarking ------------ ```bash ab -c 1000 -n 10000 http://localhost:/health -``` \ No newline at end of file +``` diff --git a/cmd/cli/flags.go b/cmd/cli/flags.go index 5fac8dc..f44dca1 100644 --- a/cmd/cli/flags.go +++ b/cmd/cli/flags.go @@ -20,4 +20,5 @@ var ( flagPrevKey = "prevkey" flagCfgFile = "cfgfile" flagCfgPath = "cfgpath" + flagMetric = "metric" ) diff --git a/cmd/cli/grpcxy.go b/cmd/cli/grpcxy.go index 0653af6..d533745 100644 --- a/cmd/cli/grpcxy.go +++ b/cmd/cli/grpcxy.go @@ -11,8 +11,11 @@ import ( "github.com/kenriortega/ngonx/pkg/errors" "github.com/kenriortega/ngonx/pkg/logger" + "github.com/kenriortega/ngonx/pkg/otelify" "github.com/spf13/cobra" "github.com/talos-systems/grpc-proxy/proxy" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/trace" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials" @@ -25,6 +28,25 @@ var grpcCmd = &cobra.Command{ Use: "grpc", Short: "Run ngonx as a grpc proxy", Run: func(cmd *cobra.Command, args []string) { + + enableMetric, err := cmd.Flags().GetBool(flagMetric) + if err != nil { + logger.LogError(errors.Errorf("proxy: %v", err).Error()) + } + if enableMetric { + // TODO: pass from compile vars + flush := otelify.InitProvider( + "ngonx", + "v0.4.5", + "dev", + // TODO: pass from yml file object + "0.0.0.0:55680", + ) + defer flush() + // Exporter Metrics + go otelify.ExposeMetricServer(configFromYaml.ProxyGateway.PortExporterProxy) + } + var opts []grpc.ServerOption lis, err := net.Listen("tcp", configFromYaml.GrpcProxy.Listener) @@ -34,6 +56,9 @@ var grpcCmd = &cobra.Command{ logger.LogInfo(fmt.Sprintf("Proxy running at %q\n", configFromYaml.GrpcProxy.Listener)) simpleBackendGen := func(hostname string) proxy.Backend { + ctx, span := otel.Tracer("grpcxy.simpleBackendGen").Start(context.Background(), "simpleBackendGen") + defer span.End() + traceID := trace.SpanContextFromContext(ctx).TraceID().String() return &proxy.SingleBackend{ GetConn: func(ctx context.Context) (context.Context, *grpc.ClientConn, error) { md, _ := metadata.FromIncomingContext(ctx) @@ -41,7 +66,9 @@ var grpcCmd = &cobra.Command{ outCtx := metadata.NewOutgoingContext(ctx, md.Copy()) if configFromYaml.GrpcSSL.Enable { creds, sslErr := credentials.NewClientTLSFromFile( - configFromYaml.GrpcClientCert, "") + configFromYaml.GrpcClientCert, + "", + ) if sslErr != nil { logger.LogError(errors.Errorf("grpc: failed to parse credentials: %v", sslErr).Error()) } @@ -51,6 +78,10 @@ var grpcCmd = &cobra.Command{ grpc.WithTransportCredentials(creds), grpc.WithCodec(proxy.Codec()), ) //nolint: staticcheck + if err != nil { + otelify.InstrumentedError(span, "grpcxy.grpc.DialContext", traceID, err) + } + otelify.InstrumentedInfo(span, "grpcxy.grpc.DialContext", traceID) return outCtx, conn, err } conn, err := grpc.DialContext( @@ -59,27 +90,48 @@ var grpcCmd = &cobra.Command{ grpc.WithInsecure(), grpc.WithCodec(proxy.Codec()), ) //nolint: staticcheck - + if err != nil { + otelify.InstrumentedError(span, "grpcxy.grpc.DialContext", traceID, err) + } + otelify.InstrumentedInfo(span, "grpcxy.grpc.DialContext", traceID) return outCtx, conn, err }, } } director = func(ctx context.Context, fullMethodName string) (proxy.Mode, []proxy.Backend, error) { + ctx, span := otel.Tracer("grpcxy.director").Start(context.Background(), "director") + defer span.End() + traceID := trace.SpanContextFromContext(ctx).TraceID().String() for _, bkd := range configFromYaml.GrpcEndpoints { // Make sure we never forward internal services. if !strings.HasPrefix(fullMethodName, bkd.Name) { + otelify.InstrumentedError( + span, + "grpcxy.not.strings.HasPrefix", + traceID, + errors.NewError("Unknown method"), + ) + return proxy.One2One, nil, status.Errorf(codes.Unimplemented, "Unknown method") } md, ok := metadata.FromIncomingContext(ctx) if ok { if _, exists := md[":authority"]; exists { + otelify.InstrumentedInfo(span, "director.proxy.One2One", traceID) + return proxy.One2One, []proxy.Backend{ simpleBackendGen(bkd.HostURI), }, nil } } } + otelify.InstrumentedError( + span, + "grpcxy.One2One", + traceID, + errors.NewError("Unknown method"), + ) return proxy.One2One, nil, status.Errorf(codes.Unimplemented, "Unknown method") } opts = append(opts, @@ -110,7 +162,7 @@ var grpcCmd = &cobra.Command{ } func init() { - + grpcCmd.Flags().Bool(flagMetric, false, "Action for enable metrics OTEL") rootCmd.AddCommand(grpcCmd) } diff --git a/cmd/cli/proxy.go b/cmd/cli/proxy.go index 449308c..28013dd 100644 --- a/cmd/cli/proxy.go +++ b/cmd/cli/proxy.go @@ -1,6 +1,8 @@ package cli import ( + "context" + domain "github.com/kenriortega/ngonx/internal/proxy/domain" handlers "github.com/kenriortega/ngonx/internal/proxy/handlers" services "github.com/kenriortega/ngonx/internal/proxy/services" @@ -9,7 +11,7 @@ import ( "github.com/kenriortega/ngonx/pkg/genkey" "github.com/kenriortega/ngonx/pkg/httpsrv" "github.com/kenriortega/ngonx/pkg/logger" - "github.com/kenriortega/ngonx/pkg/metric" + "github.com/kenriortega/ngonx/pkg/otelify" "github.com/spf13/cobra" ) @@ -17,6 +19,24 @@ var proxyCmd = &cobra.Command{ Use: "proxy", Short: "Run ngonx as a reverse proxy", Run: func(cmd *cobra.Command, args []string) { + enableMetric, err := cmd.Flags().GetBool(flagMetric) + if err != nil { + logger.LogError(errors.Errorf("proxy: %v", err).Error()) + } + if enableMetric { + // TODO: pass from compile vars + flush := otelify.InitProvider( + "ngonx", + "v0.4.5", + "dev", + // TODO: pass from yml file object + "0.0.0.0:55680", + ) + defer flush() + // Exporter Metrics + go otelify.ExposeMetricServer(configFromYaml.ProxyGateway.PortExporterProxy) + } + port, err := cmd.Flags().GetInt(flagPort) if err != nil { logger.LogError(errors.Errorf("proxy: %v", err).Error()) @@ -29,15 +49,14 @@ var proxyCmd = &cobra.Command{ if err != nil { logger.LogError(errors.Errorf("proxy: %v", err).Error()) } - // Exporter Metrics - go metric.ExposeMetricServer(configFromYaml.ProxyGateway.PortExporterProxy) + // proxy logic engine := configFromYaml.ProxyCache.Engine securityType := configFromYaml.ProxySecurity.Type key := configFromYaml.ProxyCache.Key + "_" + securityType var proxyRepository domain.ProxyRepository - clientBadger := badgerdb.GetBadgerDB(false) + clientBadger := badgerdb.GetBadgerDB(context.Background(), false) proxyRepository = domain.NewProxyRepository(clientBadger) h := handlers.ProxyHandler{ Service: services.NewProxyService(proxyRepository), @@ -61,7 +80,6 @@ var proxyCmd = &cobra.Command{ } for _, endpoints := range configFromYaml.ProxyGateway.EnpointsProxy { - h.ProxyGateway(endpoints, engine, key, securityType) } @@ -91,6 +109,7 @@ var proxyCmd = &cobra.Command{ func init() { proxyCmd.Flags().Int(flagPort, 5000, "Port to serve to run proxy") proxyCmd.Flags().Bool(flagGenApiKey, false, "Action for generate hash for protected routes") + proxyCmd.Flags().Bool(flagMetric, false, "Action for enable metrics OTEL") proxyCmd.Flags().String(flagPrevKey, "", "Action for save a previous hash for protected routes to validate JWT") rootCmd.AddCommand(proxyCmd) diff --git a/examples/otelp/main.go b/examples/otelp/main.go new file mode 100644 index 0000000..a99b236 --- /dev/null +++ b/examples/otelp/main.go @@ -0,0 +1,146 @@ +package main + +import ( + "context" + "fmt" + "log" + "math/rand" + "net/http" + "time" + + "github.com/gorilla/mux" + "github.com/kenriortega/ngonx/pkg/logger" + "github.com/kenriortega/ngonx/pkg/otelify" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/trace" + "go.uber.org/zap" +) + +// global vars...gasp! +var addr = "127.0.0.1:8000" +var tracer trace.Tracer +var httpClient http.Client + +func main() { + flush := otelify.InitProvider( + "example", + "v0.4.5", + "test", + "0.0.0.0:55680", + ) + defer flush() + + // initiate globals + tracer = otel.Tracer("example-app") + httpClient = http.Client{Transport: otelhttp.NewTransport(http.DefaultTransport)} + // create and start server + server := instrumentedServer(handler) + + fmt.Println("listening...") + log.Fatal(server.ListenAndServe()) +} + +func handler(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + longRunningProcess(ctx) + + // check cache + if shouldExecute(40) { + url := "http://" + addr + "/" + + resp, err := instrumentedGet(ctx, url) + defer resp.Body.Close() + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + } + } + + // query database + if shouldExecute(40) { + url := "http://" + addr + "/" + + resp, err := instrumentedGet(ctx, url) + defer resp.Body.Close() + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + } + } +} + +// #nosec +func shouldExecute(percent int) bool { + return rand.Int()%100 < percent +} + +func longRunningProcess(ctx context.Context) { + ctx, sp := tracer.Start(ctx, "Long Running Process") + defer sp.End() + + time.Sleep(time.Millisecond * 50) + sp.AddEvent("halfway done!") + time.Sleep(time.Millisecond * 50) +} + +/*** +Server +***/ +func instrumentedServer(handler http.HandlerFunc) *http.Server { + // OpenMetrics handler : metrics and exemplars + omHandleFunc := func(w http.ResponseWriter, r *http.Request) { + start := time.Now() + + handler.ServeHTTP(w, r) + + ctx := r.Context() + traceID := trace.SpanContextFromContext(ctx).TraceID().String() + + otelify.MetricRequestLatencyProxy.(prometheus.ExemplarObserver).ObserveWithExemplar( + time.Since(start).Seconds(), prometheus.Labels{"traceID": traceID}, + ) + + // log the trace id with other fields so we can discover traces through logs + logger.LogInfo( + "http request", + zap.String("traceID", traceID), + zap.String("path", r.URL.Path), + zap.Duration("latency", time.Since(start)), + ) + } + + // OTel handler : traces + otelHandler := otelhttp.NewHandler(http.HandlerFunc(omHandleFunc), "http") + + r := mux.NewRouter() + r.Handle("/", otelHandler) + r.Handle("/metrics", promhttp.HandlerFor(prometheus.DefaultGatherer, promhttp.HandlerOpts{ + EnableOpenMetrics: true, + })) + + return &http.Server{ + Handler: r, + Addr: "0.0.0.0:8000", + } +} + +/*** +Client +***/ +func instrumentedGet(ctx context.Context, url string) (*http.Response, error) { + // create http request + req, err := http.NewRequestWithContext(ctx, "GET", url, nil) + if err != nil { + panic(err) + } + + return httpClient.Do(req) +} + +func handleErr(err error, message string) { + if err != nil { + panic(fmt.Sprintf("%s: %s", err, message)) + } +} diff --git a/go.mod b/go.mod index cfc0627..19b82ca 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/kenriortega/ngonx -go 1.16 +go 1.17 require ( github.com/dgraph-io/badger/v3 v3.2103.0 @@ -10,16 +10,76 @@ require ( github.com/golang/snappy v0.0.3 // indirect github.com/gorilla/mux v1.8.0 github.com/gorilla/websocket v1.4.2 + github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.11.0 github.com/rs/cors v1.8.0 github.com/satori/go.uuid v1.2.0 github.com/spf13/cobra v1.2.1 github.com/spf13/viper v1.8.1 github.com/talos-systems/grpc-proxy v0.2.0 + go.opentelemetry.io/otel v1.2.0 + go.opentelemetry.io/otel/sdk v1.2.0 go.uber.org/zap v1.19.0 golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a // indirect golang.org/x/net v0.0.0-20210510120150-4163338589ed // indirect golang.org/x/sys v0.0.0-20211031064116-611d5d643895 // indirect - google.golang.org/grpc v1.40.0 + google.golang.org/grpc v1.42.0 gopkg.in/natefinch/lumberjack.v2 v2.0.0 ) + +require ( + github.com/DataDog/zstd v1.4.1 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash v1.1.0 // indirect + github.com/cespare/xxhash/v2 v2.1.1 // indirect + github.com/dgraph-io/ristretto v0.0.4-0.20210309073149-3836124cdc5a // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/dustin/go-humanize v1.0.0 // indirect + github.com/fsnotify/fsnotify v1.4.9 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect + github.com/google/flatbuffers v1.12.0 // indirect + github.com/hashicorp/errwrap v1.0.0 // indirect + github.com/hashicorp/go-multierror v1.0.0 // indirect + github.com/hashicorp/hcl v1.0.0 // indirect + github.com/inconshreveable/mousetrap v1.0.0 // indirect + github.com/magefile/mage v1.9.0 // indirect + github.com/magiconair/properties v1.8.5 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect + github.com/mitchellh/mapstructure v1.4.1 // indirect + github.com/pelletier/go-toml v1.9.3 // indirect + github.com/prometheus/client_model v0.2.0 // indirect + github.com/prometheus/common v0.26.0 // indirect + github.com/prometheus/procfs v0.6.0 // indirect + github.com/spf13/afero v1.6.0 // indirect + github.com/spf13/cast v1.3.1 // indirect + github.com/spf13/jwalterweatherman v1.1.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/subosito/gotenv v1.2.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.2.0 + go.uber.org/atomic v1.7.0 // indirect + go.uber.org/multierr v1.6.0 // indirect + golang.org/x/text v0.3.6 // indirect + golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect + google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c // indirect + google.golang.org/protobuf v1.27.1 // indirect + gopkg.in/ini.v1 v1.62.0 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect +) + +require ( + github.com/cenkalti/backoff/v4 v4.1.1 // indirect + github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect + go.opencensus.io v0.23.0 + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.26.1 + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.2.0 // indirect + go.opentelemetry.io/otel/trace v1.2.0 + go.opentelemetry.io/proto/otlp v0.10.0 // indirect +) + +require ( + github.com/felixge/httpsnoop v1.0.2 // indirect + go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.2.0 + go.opentelemetry.io/otel/internal/metric v0.24.0 // indirect + go.opentelemetry.io/otel/metric v0.24.0 // indirect +) diff --git a/go.sum b/go.sum index 2bd4fb4..bee838f 100644 --- a/go.sum +++ b/go.sum @@ -37,6 +37,7 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DataDog/zstd v1.4.1 h1:3oxKN3wbHibqx897utPC2LTQU4J+IHWWJO+glkAkpFM= @@ -61,6 +62,8 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= +github.com/cenkalti/backoff/v4 v4.1.1 h1:G2HAfAmvm/GcKan2oOQpBXOd2tT2G57ZnZGWa1PxPBQ= +github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= @@ -73,7 +76,10 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= +github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= @@ -100,9 +106,11 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= +github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/felixge/httpsnoop v1.0.2 h1:+nS9g82KMXccJ/wp0zyRW9ZBHFETmMGtkk+2CTTrW4o= +github.com/felixge/httpsnoop v1.0.2/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= @@ -206,6 +214,7 @@ github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= @@ -399,7 +408,31 @@ go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.26.1 h1:/PDcqsmxpbI/3ERJ6s6cwF13ZSH5m9NNCOPsoeazEhA= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.26.1/go.mod h1:4vatbW3QwS11DK0H0SB7FR31/VbthXcYorswdkVXdyg= +go.opentelemetry.io/otel v1.0.1/go.mod h1:OPEOD4jIT2SlZPMmwT6FqZz2C0ZNdQqiWcoK6M0SNFU= +go.opentelemetry.io/otel v1.1.0/go.mod h1:7cww0OW51jQ8IaZChIEdqLwgh+44+7uiTdWsAL0wQpA= +go.opentelemetry.io/otel v1.2.0 h1:YOQDvxO1FayUcT9MIhJhgMyNO1WqoduiyvQHzGN0kUQ= +go.opentelemetry.io/otel v1.2.0/go.mod h1:aT17Fk0Z1Nor9e0uisf98LrntPGMnk4frBO9+dkf69I= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.2.0 h1:xzbcGykysUh776gzD1LUPsNNHKWN0kQWDnJhn1ddUuk= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.2.0/go.mod h1:14T5gr+Y6s2AgHPqBMgnGwp04csUjQmYXFWPeiBoq5s= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.2.0 h1:VsgsSCDwOSuO8eMVh63Cd4nACMqgjpmAeJSIvVNneD0= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.2.0/go.mod h1:9mLBBnPRf3sf+ASVH2p9xREXVBvwib02FxcKnavtExg= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.2.0 h1:OiYdrCq1Ctwnovp6EofSPwlp5aGy4LgKNbkg7PtEUw8= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.2.0/go.mod h1:DUFCmFkXr0VtAHl5Zq2JRx24G6ze5CAq8YfdD36RdX8= +go.opentelemetry.io/otel/internal/metric v0.24.0 h1:O5lFy6kAl0LMWBjzy3k//M8VjEaTDWL9DPJuqZmWIAA= +go.opentelemetry.io/otel/internal/metric v0.24.0/go.mod h1:PSkQG+KuApZjBpC6ea6082ZrWUUy/w132tJ/LOU3TXk= +go.opentelemetry.io/otel/metric v0.24.0 h1:Rg4UYHS6JKR1Sw1TxnI13z7q/0p/XAbgIqUTagvLJuU= +go.opentelemetry.io/otel/metric v0.24.0/go.mod h1:tpMFnCD9t+BEGiWY2bWF5+AwjuAdM0lSowQ4SBA3/K4= +go.opentelemetry.io/otel/sdk v1.2.0 h1:wKN260u4DesJYhyjxDa7LRFkuhH7ncEVKU37LWcyNIo= +go.opentelemetry.io/otel/sdk v1.2.0/go.mod h1:jNN8QtpvbsKhgaC6V5lHiejMoKD+V8uadoSafgHPx1U= +go.opentelemetry.io/otel/trace v1.0.1/go.mod h1:5g4i4fKLaX2BQpSBsxw8YYcgKpMMSW3x7ZTuYBr3sUk= +go.opentelemetry.io/otel/trace v1.1.0/go.mod h1:i47XtdcBQiktu5IsrPqOHe8w+sBmnLwwHt8wiUsWGTI= +go.opentelemetry.io/otel/trace v1.2.0 h1:Ys3iqbqZhcf28hHzrm5WAquMkDHNZTUkw7KHbuNjej0= +go.opentelemetry.io/otel/trace v1.2.0/go.mod h1:N5FLswTubnxKxOJHM7XZC074qpeEdLy3CgAVsdMucK0= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= +go.opentelemetry.io/proto/otlp v0.10.0 h1:n7brgtEbDvXEgGyKKo8SobKT1e9FewlDtXzkVP5djoE= +go.opentelemetry.io/proto/otlp v0.10.0/go.mod h1:zG20xCK0szZ1xdokeSOwEcmlXu+x9kkdRe6N1DhKcfU= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.10 h1:z+mqJhf6ss6BSfSM671tgKyZBFPTTJM+HLxnhPC3wu0= @@ -577,6 +610,7 @@ golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211031064116-611d5d643895 h1:iaNpwpnrgL5jzWS0vCNnfa8HqzxveCFpFx3uC/X4Tps= @@ -745,8 +779,9 @@ google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAG google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.40.0 h1:AGJ0Ih4mHjSeibYkFGh1dD9KJ/eOtZ93I6hoHhukQ5Q= -google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k= +google.golang.org/grpc v1.42.0 h1:XT2/MFpuPFsEX2fWh3YQtHkZ+WYZFQRfaUgLZYj/p6A= +google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -758,8 +793,9 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/internal/proxy/domain/proxyrepository.go b/internal/proxy/domain/proxyrepository.go index acc63bc..91e918d 100644 --- a/internal/proxy/domain/proxyrepository.go +++ b/internal/proxy/domain/proxyrepository.go @@ -6,7 +6,9 @@ import ( badger "github.com/dgraph-io/badger/v3" "github.com/go-redis/redis/v8" "github.com/kenriortega/ngonx/pkg/errors" - "github.com/kenriortega/ngonx/pkg/logger" + "github.com/kenriortega/ngonx/pkg/otelify" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/trace" ) // ProxyRepositoryStorage struct repository storage @@ -32,32 +34,38 @@ func NewProxyRepository(clients ...interface{}) ProxyRepositoryStorage { // SaveKEY save a key on the database func (r ProxyRepositoryStorage) SaveKEY(engine, key, apikey string) error { + ctx, span := otel.Tracer("proxy.repo").Start(context.Background(), "SaveKEY") + defer span.End() + traceID := trace.SpanContextFromContext(ctx).TraceID().String() switch engine { case "badger": if err := r.clientBadger.Update(func(txn *badger.Txn) error { if err := txn.Set([]byte(key), []byte(apikey)); err != nil { - logger.LogError(errors.ErrSavekeyUpdateTX.Error()) + otelify.InstrumentedError(span, "badger", traceID, err) return errors.ErrSavekeyUpdateTX } - logger.LogInfo("proxy: savekey was successful") - + otelify.InstrumentedInfo(span, "repo.SaveKey", traceID) return nil }); err != nil { - + otelify.InstrumentedError(span, "badger", traceID, err) return errors.ErrSavekeyUpdate } return nil case "redis": if _, err := r.clientRdb.HSet(context.TODO(), key, apikey).Result(); err != nil { - logger.LogError(errors.Errorf("proxy redis: %v", err).Error()) + otelify.InstrumentedError(span, "redis", traceID, err) } + otelify.InstrumentedInfo(span, "repo.SaveKey", traceID) } return nil } // GetKEY get key from the database func (r ProxyRepositoryStorage) GetKEY(engine, key string) (string, error) { + ctx, span := otel.Tracer("proxy.repo").Start(context.Background(), "GetKEY") + defer span.End() + traceID := trace.SpanContextFromContext(ctx).TraceID().String() var apikey string switch engine { @@ -65,26 +73,31 @@ func (r ProxyRepositoryStorage) GetKEY(engine, key string) (string, error) { if err := r.clientBadger.View(func(txn *badger.Txn) error { item, err := txn.Get([]byte(key)) if err != nil { + otelify.InstrumentedError(span, "badger", traceID, err) return errors.ErrGetkeyTX } if err := item.Value(func(value []byte) error { apikey = string(value) return nil }); err != nil { + otelify.InstrumentedError(span, "badger", traceID, err) return errors.ErrGetkeyValue } return nil }); err != nil { + otelify.InstrumentedError(span, "badger", traceID, err) return "", errors.ErrGetkeyView } case "redis": value, err := r.clientRdb.Get(context.TODO(), key).Result() if err == redis.Nil || err != nil { + otelify.InstrumentedError(span, "redis", traceID, err) return "", err } apikey = value } + otelify.InstrumentedInfo(span, "repo.GetKey", traceID) return apikey, nil } diff --git a/internal/proxy/handlers/proxy.go b/internal/proxy/handlers/proxy.go index e82f761..2b13d54 100644 --- a/internal/proxy/handlers/proxy.go +++ b/internal/proxy/handlers/proxy.go @@ -1,17 +1,20 @@ package proxy import ( + "context" + "encoding/json" "fmt" "net/http" "net/http/httputil" "net/url" - "strings" "time" "github.com/kenriortega/ngonx/pkg/errors" "github.com/kenriortega/ngonx/pkg/logger" - "github.com/kenriortega/ngonx/pkg/metric" - "github.com/prometheus/client_golang/prometheus" + "github.com/kenriortega/ngonx/pkg/otelify" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/trace" "github.com/gbrlsnchs/jwt/v3" domain "github.com/kenriortega/ngonx/internal/proxy/domain" @@ -47,37 +50,65 @@ func (ph *ProxyHandler) SaveSecretKEY(engine, key, apikey string) { } // ProxyGateway handler for management all request -func (ph *ProxyHandler) ProxyGateway(endpoints domain.ProxyEndpoint, engine, key, securityType string) { +func (ph *ProxyHandler) ProxyGateway( + endpoints domain.ProxyEndpoint, + engine, + key, + securityType string, +) { + ctx, span := otel.Tracer("proxy.gateway").Start(context.Background(), "ProxyGateway") + defer span.End() + traceID := trace.SpanContextFromContext(ctx).TraceID().String() for _, endpoint := range endpoints.Endpoints { + start := time.Now() target, err := url.Parse( fmt.Sprintf("%s%s", endpoints.HostURI, endpoint.PathEndpoint), ) if err != nil { + span.RecordError(err) + span.SetStatus(codes.Error, err.Error()) logger.LogError(errors.Errorf("proxy: %v", err).Error()) - } + if endpoint.PathProtected { + var err error proxy = httputil.NewSingleHostReverseProxy(target) originalDirector := proxy.Director proxy.Director = func(req *http.Request) { originalDirector(req) - metricRegister(req, target) - switch securityType { case "jwt": - err := checkJWTSecretKeyFromRequest(req, key) - proxy.ModifyResponse = modifyResponse(err) + err = checkJWT(ctx, req, key) case "apikey": - err := checkAPIKEYSecretKeyFromRequest(req, ph, engine, key) - proxy.ModifyResponse = modifyResponse(err) + err = checkAPIKEY(ctx, req, ph, engine, key) } - + otelRegisterByRequest(ctx, start, req, err) + } + proxy.ModifyResponse = func(resp *http.Response) error { + resp.Header.Set("X-Proxy", "Ngonx") + if err != nil { + return err + } + return nil } - proxy.ErrorHandler = func(rw http.ResponseWriter, r *http.Request, err error) { - rw.WriteHeader(http.StatusInternalServerError) - _, _ = rw.Write([]byte(err.Error())) + proxy.ErrorHandler = func(w http.ResponseWriter, req *http.Request, err error) { + rpm := ResponseMiddleware{ + Message: err.Error(), + Code: http.StatusBadGateway, + } + w.WriteHeader(rpm.Code) + w.Header().Set("Content-Type", "application/json") + bytes, err := json.Marshal(&rpm) + if err != nil { + logger.LogError(err.Error()) + } + _, err = w.Write(bytes) + if err != nil { + logger.LogError(err.Error()) + } + } http.Handle( endpoint.PathToProxy, @@ -93,9 +124,13 @@ func (ph *ProxyHandler) ProxyGateway(endpoints domain.ProxyEndpoint, engine, key originalDirector := proxy.Director proxy.Director = func(req *http.Request) { originalDirector(req) - metricRegister(req, target) - + otelRegisterByRequest(ctx, start, req, nil) + } + proxy.ModifyResponse = func(resp *http.Response) error { + resp.Header.Set("X-Proxy", "Ngonx") + return nil } + http.Handle( endpoint.PathToProxy, http.StripPrefix( @@ -105,106 +140,5 @@ func (ph *ProxyHandler) ProxyGateway(endpoints domain.ProxyEndpoint, engine, key ) } } -} - -func metricRegister(req *http.Request, target *url.URL) { - metric.CountersByEndpoint.With( - prometheus.Labels{ - "proxyPath": req.RequestURI, - "endpointPath": target.String(), - "ipAddr": extractIpAddr(req), - "method": req.Method, - }, - ).Inc() - metric.TotalRequests.With( - prometheus.Labels{ - "path": req.RequestURI, - "service": "proxy", - }, - ).Inc() - -} - -// checkJWTSecretKeyFromRequest check jwt for request -func checkJWTSecretKeyFromRequest(req *http.Request, key string) error { - header := req.Header.Get("Authorization") // pass to constanst - hs := jwt.NewHS256([]byte(key)) - now := time.Now() - if !strings.HasPrefix(header, "Bearer ") { - logger.LogError(errors.Errorf("proxy: %v", errors.ErrBearerTokenFormat).Error()) - - return errors.ErrBearerTokenFormat - } - - token := strings.Split(header, " ")[1] - pl := JWTPayload{} - expValidator := jwt.ExpirationTimeValidator(now) - validatePayload := jwt.ValidatePayload(&pl.Payload, expValidator) - - _, err := jwt.Verify([]byte(token), hs, &pl, validatePayload) - - if errors.ErrorIs(err, jwt.ErrExpValidation) { - logger.LogError(errors.Errorf("proxy: %v", errors.ErrTokenExpValidation).Error()) - - return errors.ErrTokenExpValidation - } - if errors.ErrorIs(err, jwt.ErrHMACVerification) { - logger.LogError(errors.Errorf("proxy: %v", errors.ErrTokenHMACValidation).Error()) - - return errors.ErrTokenHMACValidation - } - - return nil -} - -// checkAPIKEYSecretKeyFromRequest check apikey from request -func checkAPIKEYSecretKeyFromRequest(req *http.Request, ph *ProxyHandler, engine, key string) error { - apikey, err := ph.Service.GetKEY(engine, key) - header := req.Header.Get("X-API-KEY") // pass to constants - if err != nil { - logger.LogError(errors.Errorf("proxy: %v", errors.ErrGetkeyView).Error()) - - } - if apikey == header { - logger.LogInfo("proxy: check secret from request OK") - return nil - } else { - logger.LogError(errors.Errorf("proxy: Invalid API KEY").Error()) - return errors.NewError("Invalid API KEY") - } -} - -// modifyResponse modify response -func modifyResponse(err error) func(*http.Response) error { - return func(resp *http.Response) error { - resp.Header.Set("X-Proxy", "Ngonx") - - if err != nil { - return err - } - return nil - } -} - -func extractIpAddr(req *http.Request) string { - ipAddress := req.RemoteAddr - fwdAddress := req.Header.Get("X-Forwarded-For") // capitalisation doesn't matter - if fwdAddress != "" { - // Got X-Forwarded-For - ipAddress = fwdAddress // If it's a single IP, then awesome! - - // If we got an array... grab the first IP - ips := strings.Split(fwdAddress, ", ") - if len(ips) > 1 { - ipAddress = ips[0] - } - } - remoteAddrToParse := "" - if strings.Contains(ipAddress, "[::1]") { - remoteAddrToParse = strings.Replace(ipAddress, "[::1]", "localhost", -1) - ipAddress = strings.Split(remoteAddrToParse, ":")[0] - } else { - ipAddress = strings.Split(ipAddress, ":")[0] - } - return ipAddress + otelify.InstrumentedInfo(span, "proxy.Gateway", traceID) } diff --git a/internal/proxy/handlers/utils.go b/internal/proxy/handlers/utils.go new file mode 100644 index 0000000..277bc0d --- /dev/null +++ b/internal/proxy/handlers/utils.go @@ -0,0 +1,103 @@ +package proxy + +import ( + "context" + "net/http" + "strings" + "time" + + "github.com/gbrlsnchs/jwt/v3" + "github.com/kenriortega/ngonx/pkg/errors" + "github.com/kenriortega/ngonx/pkg/logger" + "github.com/kenriortega/ngonx/pkg/otelify" + "github.com/prometheus/client_golang/prometheus" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/trace" + "go.uber.org/zap" +) + +func otelRegisterByRequest(ctx context.Context, start time.Time, req *http.Request, err error) { + + traceID := trace.SpanContextFromContext(ctx).TraceID().String() + + otelify.MetricRequestLatencyProxy.(prometheus.ExemplarObserver).ObserveWithExemplar( + time.Since(start).Seconds(), prometheus.Labels{"traceID": traceID}, + ) + + if err != nil { + logger.LogError( + "proxy.Director.Metric", + zap.String("traceID", traceID), + zap.String("path", req.URL.Path), + zap.Duration("latency", time.Since(start)), + ) + + return + } + logger.LogInfo( + "proxy.Director.Metric", + zap.String("traceID", traceID), + zap.String("path", req.URL.Path), + zap.Duration("latency", time.Since(start)), + ) +} + +// checkJWT check jwt for request +func checkJWT(ctx context.Context, req *http.Request, key string) error { + ctx, span := otel.Tracer("proxy.gateway.checkJWT").Start(ctx, "checkJWT") + defer span.End() + traceID := trace.SpanContextFromContext(ctx).TraceID().String() + + header := req.Header.Get("Authorization") // pass to constanst + hs := jwt.NewHS256([]byte(key)) + now := time.Now() + if !strings.HasPrefix(header, "Bearer ") { + otelify.InstrumentedError(span, "checkJWT.bearer", traceID, errors.ErrBearerTokenFormat) + return errors.ErrBearerTokenFormat + } + + token := strings.Split(header, " ")[1] + pl := JWTPayload{} + expValidator := jwt.ExpirationTimeValidator(now) + validatePayload := jwt.ValidatePayload(&pl.Payload, expValidator) + + _, err := jwt.Verify([]byte(token), hs, &pl, validatePayload) + + if errors.ErrorIs(err, jwt.ErrExpValidation) { + otelify.InstrumentedError(span, "checkJWT.expValidation", traceID, errors.ErrTokenExpValidation) + return errors.ErrTokenExpValidation + } + if errors.ErrorIs(err, jwt.ErrHMACVerification) { + otelify.InstrumentedError(span, "checkJWT.HMACValidation", traceID, errors.ErrTokenHMACValidation) + return errors.ErrTokenHMACValidation + } + otelify.InstrumentedInfo(span, "checkJWT", traceID) + return nil +} + +// checkAPIKEY check apikey from request +func checkAPIKEY( + ctx context.Context, + req *http.Request, + ph *ProxyHandler, + engine, key string, +) error { + ctx, span := otel.Tracer("proxy.gateway.checkAPIKey").Start(ctx, "checkAPIKEY") + defer span.End() + traceID := trace.SpanContextFromContext(ctx).TraceID().String() + + header := req.Header.Get("X-API-KEY") + apikey, err := ph.Service.GetKEY(engine, key) + if err != nil { + otelify.InstrumentedError(span, "checkAPIKEY.GetKEY", traceID, errors.ErrGetkeyView) + return errors.ErrGetkeyView + } + if apikey == header { + otelify.InstrumentedInfo(span, "checkAPIKEY", traceID) + return nil + } else { + invalidKeyErr := errors.NewError("Invalid API KEY") + otelify.InstrumentedError(span, "chackAPIKEY.invalidHeader", traceID, invalidKeyErr) + return invalidKeyErr + } +} diff --git a/internal/proxy/services/proxy.go b/internal/proxy/services/proxy.go index 4237ac9..f37b412 100644 --- a/internal/proxy/services/proxy.go +++ b/internal/proxy/services/proxy.go @@ -1,7 +1,12 @@ package proxy import ( + "context" + domain "github.com/kenriortega/ngonx/internal/proxy/domain" + "github.com/kenriortega/ngonx/pkg/otelify" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/trace" ) // ProxyService interface service for proxy repository funcionalities @@ -22,19 +27,28 @@ func NewProxyService(repository domain.ProxyRepository) DefaultProxyService { // SaveSecretKEY save secret key func (s DefaultProxyService) SaveSecretKEY(engine, key, apikey string) (string, error) { - + ctx, span := otel.Tracer("proxy.service.SaveSecretKEY").Start(context.Background(), "ProxyGateway") + defer span.End() + traceID := trace.SpanContextFromContext(ctx).TraceID().String() err := s.repo.SaveKEY(engine, key, apikey) if err != nil { + otelify.InstrumentedError(span, "SaveKey", traceID, err) return "failed", err } + otelify.InstrumentedInfo(span, "service.SaveSecretKEY", traceID) return "ok", nil } // GetKEY get key func (s DefaultProxyService) GetKEY(engine, key string) (string, error) { + ctx, span := otel.Tracer("proxy.service.GetKEY").Start(context.Background(), "ProxyGateway") + defer span.End() + traceID := trace.SpanContextFromContext(ctx).TraceID().String() result, err := s.repo.GetKEY(engine, key) if err != nil { + otelify.InstrumentedError(span, "GetKey", traceID, err) return "failed", err } + otelify.InstrumentedInfo(span, "service.GetKey", traceID) return result, nil } diff --git a/internal/proxy/services/proxy_test.go b/internal/proxy/services/proxy_test.go index c954291..b378efb 100644 --- a/internal/proxy/services/proxy_test.go +++ b/internal/proxy/services/proxy_test.go @@ -1,6 +1,8 @@ package proxy import ( + "context" + domain "github.com/kenriortega/ngonx/internal/proxy/domain" "github.com/kenriortega/ngonx/pkg/badgerdb" @@ -10,7 +12,7 @@ import ( var proxyRepository domain.ProxyRepository func Test_SaveSecretKEY(t *testing.T) { - clientBadger := badgerdb.GetBadgerDB(false) + clientBadger := badgerdb.GetBadgerDB(context.Background(), false) proxyRepository = domain.NewProxyRepository(clientBadger) err := proxyRepository.SaveKEY("badger", "key", "apikey") if err != nil { @@ -19,7 +21,7 @@ func Test_SaveSecretKEY(t *testing.T) { } func Test_GetKEY(t *testing.T) { - clientBadger := badgerdb.GetBadgerDB(false) + clientBadger := badgerdb.GetBadgerDB(context.Background(), false) proxyRepository = domain.NewProxyRepository(clientBadger) result, err := proxyRepository.GetKEY("badger", "key") if err != nil { diff --git a/ngonx.yaml b/ngonx.yaml index 2341187..cd5fc54 100644 --- a/ngonx.yaml +++ b/ngonx.yaml @@ -38,7 +38,7 @@ proxy: # maps of microservices with routes services_proxy: - name: microA - host_uri: http://localhost:3001 + host_uri: http://localhost:5000 endpoints: - path_endpoints: /api/v1/health/ path_proxy: /health/ diff --git a/pkg/badgerdb/client.go b/pkg/badgerdb/client.go index fc9ccfc..e1ce5ce 100644 --- a/pkg/badgerdb/client.go +++ b/pkg/badgerdb/client.go @@ -1,16 +1,23 @@ package badgerdb import ( + "context" + badger "github.com/dgraph-io/badger/v3" - "github.com/kenriortega/ngonx/pkg/errors" - "github.com/kenriortega/ngonx/pkg/logger" + "github.com/kenriortega/ngonx/pkg/otelify" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/trace" ) var pathDB = "./badger.data" // GetBadgerDB return `*badger.DB` // this client provide GET and SAVE methods -func GetBadgerDB(embedMem bool) *badger.DB { +func GetBadgerDB(ctx context.Context, embedMem bool) *badger.DB { + ctx, span := otel.Tracer("badger.client").Start(ctx, "GetBadgerDB") + defer span.End() + traceID := trace.SpanContextFromContext(ctx).TraceID().String() + var opt badger.Options var clientBadger *badger.DB if embedMem { @@ -21,12 +28,12 @@ func GetBadgerDB(embedMem bool) *badger.DB { db, err := badger.Open(opt) if err != nil { - logger.LogError(errors.Errorf("badger: %v", err).Error()) - + otelify.InstrumentedError(span, "badger", traceID, err) panic(err) } clientBadger = db - // defer clientBadger.Close() + + otelify.InstrumentedInfo(span, "proxy.GetBadgerDB", traceID) return clientBadger } diff --git a/pkg/metric/client.go b/pkg/metric/client.go deleted file mode 100644 index 1a77c7b..0000000 --- a/pkg/metric/client.go +++ /dev/null @@ -1,45 +0,0 @@ -package metric - -import ( - "fmt" - "log" - "net/http" - - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promhttp" -) - -var ( - CountersByEndpoint = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "counter_request_by_microservicio", - Help: "Register all call to the endpoints", - }, - []string{ - "proxyPath", - "endpointPath", - "ipAddr", - "method", - }, - ) - - TotalRequests = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "http_requests_total", - Help: "Number of get requests.", - }, - []string{"path", "service"}, - ) -) - -func init() { - - prometheus.MustRegister(CountersByEndpoint) - prometheus.MustRegister(TotalRequests) -} - -func ExposeMetricServer(configPort int) { - http.Handle("/metrics", promhttp.Handler()) - port := fmt.Sprintf(":%d", configPort) - log.Fatal(http.ListenAndServe(port, nil)) -} diff --git a/pkg/otelify/otelify.go b/pkg/otelify/otelify.go new file mode 100644 index 0000000..77e7815 --- /dev/null +++ b/pkg/otelify/otelify.go @@ -0,0 +1,89 @@ +package otelify + +import ( + "context" + "log" + + "github.com/kenriortega/ngonx/pkg/errors" + "github.com/kenriortega/ngonx/pkg/logger" + "go.opentelemetry.io/otel/trace" + "go.uber.org/zap" + "google.golang.org/grpc" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc" + "go.opentelemetry.io/otel/propagation" + "go.opentelemetry.io/otel/sdk/resource" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + semconv "go.opentelemetry.io/otel/semconv/v1.7.0" +) + +// Initializes an OTLP exporter, and configures the corresponding trace and +// metric providers. +func InitProvider(name, version, namEnv, endpoint string) func() { + ctx := context.Background() + + // Set up a trace exporter + traceExporter, err := otlptracegrpc.New(ctx, + otlptracegrpc.WithInsecure(), + otlptracegrpc.WithEndpoint(endpoint), + otlptracegrpc.WithDialOption(grpc.WithBlock()), + ) + handleErr(err, "failed to create trace exporter") + + // Register the trace exporter with a TracerProvider, using a batch + // span processor to aggregate spans before export. + bsp := sdktrace.NewBatchSpanProcessor(traceExporter) + tracerProvider := sdktrace.NewTracerProvider( + sdktrace.WithSampler(sdktrace.AlwaysSample()), + sdktrace.WithResource(NewResource(name, version, namEnv)), + sdktrace.WithSpanProcessor(bsp), + ) + otel.SetTracerProvider(tracerProvider) + + // set global propagator to tracecontext (the default is no-op). + otel.SetTextMapPropagator(propagation.TraceContext{}) + + return func() { + // Shutdown will flush any remaining spans and shut down the exporter. + handleErr(tracerProvider.Shutdown(ctx), "failed to shutdown TracerProvider") + } +} +func handleErr(err error, message string) { + if err != nil { + log.Fatalf("%s: %v", message, err) + } +} + +// NewResource returns a resource describing this application. +func NewResource(name, version, namEnv string) *resource.Resource { + r, _ := resource.Merge( + resource.Default(), + resource.NewWithAttributes( + semconv.SchemaURL, + semconv.ServiceNameKey.String(name), + semconv.ServiceVersionKey.String(version), + attribute.String("environment", namEnv), + ), + ) + return r +} + +func InstrumentedError(span trace.Span, source, traceID string, err error) { + span.RecordError(err) + span.SetStatus(codes.Error, err.Error()) + logger.LogError( + errors.Errorf("%s: %v", source, err).Error(), + zap.String("traceID", traceID), + ) +} + +func InstrumentedInfo(span trace.Span, source, traceID string) { + logger.LogInfo( + source, + zap.String("traceID", traceID), + ) + span.SetAttributes(attribute.String(source, "Success")) +} diff --git a/pkg/otelify/prometheus.go b/pkg/otelify/prometheus.go new file mode 100644 index 0000000..4e2a469 --- /dev/null +++ b/pkg/otelify/prometheus.go @@ -0,0 +1,24 @@ +package otelify + +import ( + "fmt" + "net/http" + + "github.com/kenriortega/ngonx/pkg/logger" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "github.com/prometheus/client_golang/prometheus/promhttp" +) + +var MetricRequestLatencyProxy = promauto.NewHistogram(prometheus.HistogramOpts{ + Namespace: "ngonx", + Name: "request_latency_seconds", + Help: "Request Latency", + Buckets: prometheus.ExponentialBuckets(.0001, 2, 50), +}) + +func ExposeMetricServer(configPort int) { + http.Handle("/metrics", promhttp.Handler()) + port := fmt.Sprintf(":%d", configPort) + logger.LogError(http.ListenAndServe(port, nil).Error()) +} diff --git a/scripts/request/micro-a.rest b/scripts/request/micro-a.rest index a83b466..3da02c5 100644 --- a/scripts/request/micro-a.rest +++ b/scripts/request/micro-a.rest @@ -1,10 +1,10 @@ ### Micro A -GET http://localhost:3001/api/v1/health/ +GET http://localhost:5000/api/v1/health/ ### -GET http://localhost:3001/api/v1/version/ +GET http://localhost:5000/api/v1/version/ ### Gateway Proxy @@ -12,31 +12,15 @@ GET http://localhost:3001/api/v1/version/ GET http://localhost:35000/health/ ### -GET https://localhost:30443/health/ - -### Gateway LB - -GET http://localhost:3030/health/ -Authorization: Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJCYW5rIiwic3ViIjoiU3lzdGVtQXBwIiwiYXVkIjoiaHR0cDovL2xvY2FsaG9zdDo4MDAwIiwiZXhwIjoxNjI0ODk3ODQ4LCJuYmYiOjE2MjQ4OTkwNDgsImlhdCI6MTYyNDg5NzI0OCwianRpIjoiYXV0aC1zZXJ2ZXItMSIsIlVzZXJuYW1lIjoiTUFkbWluIiwiUm9sZSI6IkFETUlOIn0.TJn1Luy6jFsuRiQN0fAJYYQTxYWnO1wgWAUTGThY1Gg ### Proxy - GET http://localhost:35000/version/ Authorization: Bearerx eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJCYW5rIiwic3ViIjoiU3lzdGVtQXBwIiwiYXVkIjoiaHR0cDovL2xvY2FsaG9zdDo4MDAwIiwiZXhwIjoxNjI0ODk3ODQ4LCJuYmYiOjE2MjQ4OTkwNDgsImlhdCI6MTYyNDg5NzI0OCwianRpIjoiYXV0aC1zZXJ2ZXItMSIsIlVzZXJuYW1lIjoiTUFkbWluIiwiUm9sZSI6IkFETUlOIn0.TJn1Luy6jFsuRiQN0fAJYYQTxYWnO1wgWAUTGThY1Gg - ### GET http://localhost:35000/version/ -X-API-KEY: 6d5cdc4a5e51f11da6561b8e02bcbe3e4979066c7f9e71e780f15e23dd0f34ee - -### SSL -GET https://localhost:30443/version/ -X-API-KEY: 6d5cdc4a5e51f11da6561b8e02bcbe3e4979066c7f9e71e780f15e23dd0f34ee - - -#### Metrics +X-API-KEY: 6a1244e55eb42986b7c20742d29c04d74afa1b5b39d7e558c849c7e2141b1bf1 -GET http://localhost:10000/metrics ### Admin MNTG