From aa10d2df1448743797cab9ee7bb04da7ca04e3e1 Mon Sep 17 00:00:00 2001 From: Jordan Sussman Date: Wed, 28 Jun 2023 10:25:59 -0500 Subject: [PATCH 01/59] init jaeger --- docker-compose.yml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index d030743e6..7562d1e6d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -156,5 +156,25 @@ services: cap_add: - IPC_LOCK + jaeger: + image: jaegertracing/all-in-one:1.46 + container_name: jaeger + networks: + - vela + environment: + COLLECTOR_ZIPKIN_HOST_PORT: ":9411" + COLLECTOR_OTLP_ENABLED: true + ports: + - '6831:6831/udp' + - '6832:6832/udp' + - '5778:5778' + - '16686:16686' + - '4317:4317' + - '4318:4318' + - '14250:14250' + - '14268:14268' + - '14269:14269' + - '9411:9411' + networks: vela: \ No newline at end of file From 4301fbe926c455d44948f5f2df8b8286172d5999 Mon Sep 17 00:00:00 2001 From: davidvader Date: Wed, 28 Jun 2023 11:02:40 -0500 Subject: [PATCH 02/59] wip: otel gin and gorm --- cmd/vela-server/server.go | 14 +++++++++++- cmd/vela-server/tracing.go | 32 ++++++++++++++++++++++++++ database/context.go | 4 +++- database/context_test.go | 2 +- database/database.go | 11 +++++++++ database/opts.go | 15 ++++++++++++- docker-compose.yml | 29 ++++++++++++------------ go.mod | 21 +++++++++++++++-- go.sum | 46 ++++++++++++++++++++++++++++++++++---- 9 files changed, 150 insertions(+), 24 deletions(-) create mode 100644 cmd/vela-server/tracing.go diff --git a/cmd/vela-server/server.go b/cmd/vela-server/server.go index 0d95e88e5..00ea5468a 100644 --- a/cmd/vela-server/server.go +++ b/cmd/vela-server/server.go @@ -20,6 +20,7 @@ import ( "github.com/go-vela/server/router/middleware" "github.com/sirupsen/logrus" "github.com/urfave/cli/v2" + "go.opentelemetry.io/contrib/instrumentation/github.com/gin-gonic/gin/otelgin" "golang.org/x/sync/errgroup" "k8s.io/apimachinery/pkg/util/wait" @@ -62,7 +63,17 @@ func server(c *cli.Context) error { return err } - database, err := database.FromCLIContext(c) + tp, err := tracerProvider() + if err != nil { + logrus.Fatal(err) + } + defer func() { + if err := tp.Shutdown(context.Background()); err != nil { + logrus.Printf("Error shutting down tracer provider: %v", err) + } + }() + + database, err := database.FromCLIContext(c, tp) if err != nil { return err } @@ -108,6 +119,7 @@ func server(c *cli.Context) error { middleware.DefaultRepoEvents(c.StringSlice("default-repo-events")), middleware.AllowlistSchedule(c.StringSlice("vela-schedule-allowlist")), middleware.ScheduleFrequency(c.Duration("schedule-minimum-frequency")), + otelgin.Middleware("vela-server"), ) addr, err := url.Parse(c.String("server-addr")) diff --git a/cmd/vela-server/tracing.go b/cmd/vela-server/tracing.go new file mode 100644 index 000000000..0a92e2563 --- /dev/null +++ b/cmd/vela-server/tracing.go @@ -0,0 +1,32 @@ +package main + +import ( + "context" + + "go.opentelemetry.io/otel" + otlp "go.opentelemetry.io/otel/exporters/otlp/otlptrace" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" + "go.opentelemetry.io/otel/propagation" + sdktrace "go.opentelemetry.io/otel/sdk/trace" +) + +func tracerProvider() (*sdktrace.TracerProvider, error) { + // exporter, err := stdout.New(stdout.WithPrettyPrint()) + + // client := nil + client := otlptracehttp.NewClient() + + exporter, err := otlp.New(context.TODO(), client) + if err != nil { + return nil, err + } + + tp := sdktrace.NewTracerProvider( + sdktrace.WithSampler(sdktrace.AlwaysSample()), + sdktrace.WithBatcher(exporter), + ) + + otel.SetTracerProvider(tp) + otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(propagation.TraceContext{}, propagation.Baggage{})) + return tp, nil +} diff --git a/database/context.go b/database/context.go index fd08cc371..1c4811213 100644 --- a/database/context.go +++ b/database/context.go @@ -9,6 +9,7 @@ import ( "github.com/sirupsen/logrus" "github.com/urfave/cli/v2" + sdktrace "go.opentelemetry.io/otel/sdk/trace" ) const key = "database" @@ -40,7 +41,7 @@ func ToContext(c Setter, d Interface) { } // FromCLIContext creates and returns a database engine from the urfave/cli context. -func FromCLIContext(c *cli.Context) (Interface, error) { +func FromCLIContext(c *cli.Context, tp *sdktrace.TracerProvider) (Interface, error) { logrus.Debug("creating database engine from CLI configuration") return New( @@ -52,5 +53,6 @@ func FromCLIContext(c *cli.Context) (Interface, error) { WithDriver(c.String("database.driver")), WithEncryptionKey(c.String("database.encryption.key")), WithSkipCreation(c.Bool("database.skip_creation")), + WithTracerProvider(tp), ) } diff --git a/database/context_test.go b/database/context_test.go index 4037b26c0..8513ca4ab 100644 --- a/database/context_test.go +++ b/database/context_test.go @@ -134,7 +134,7 @@ func TestDatabase_FromCLIContext(t *testing.T) { // run tests for _, test := range tests { t.Run(test.name, func(t *testing.T) { - _, err := FromCLIContext(test.context) + _, err := FromCLIContext(test.context, nil) if test.failure { if err == nil { diff --git a/database/database.go b/database/database.go index c612ba3aa..732f97d8d 100644 --- a/database/database.go +++ b/database/database.go @@ -21,7 +21,9 @@ import ( "github.com/go-vela/server/database/worker" "github.com/go-vela/types/constants" "github.com/sirupsen/logrus" + "github.com/uptrace/opentelemetry-go-extra/otelgorm" + sdktrace "go.opentelemetry.io/otel/sdk/trace" "gorm.io/driver/postgres" "gorm.io/driver/sqlite" "gorm.io/gorm" @@ -46,6 +48,8 @@ type ( EncryptionKey string // specifies to skip creating tables and indexes for the database engine SkipCreation bool + + TracerProvider *sdktrace.TracerProvider } // engine represents the functionality that implements the Interface. @@ -129,6 +133,12 @@ func New(opts ...EngineOpt) (Interface, error) { return nil, err } + logrus.Info("initializing gorm tracing") + otelPlugin := otelgorm.NewPlugin() + if err := e.client.Use(otelPlugin); err != nil { + return nil, err + } + // set the maximum amount of time a connection may be reused db.SetConnMaxLifetime(e.config.ConnectionLife) // set the maximum number of connections in the idle connection pool @@ -164,5 +174,6 @@ func NewTest() (Interface, error) { WithDriver("sqlite3"), WithEncryptionKey("A1B2C3D4E5G6H7I8J9K0LMNOPQRSTUVW"), WithSkipCreation(false), + WithTracerProvider(nil), ) } diff --git a/database/opts.go b/database/opts.go index 3714989c5..7d69aca24 100644 --- a/database/opts.go +++ b/database/opts.go @@ -4,7 +4,11 @@ package database -import "time" +import ( + "time" + + sdktrace "go.opentelemetry.io/otel/sdk/trace" +) // EngineOpt represents a configuration option to initialize the database engine. type EngineOpt func(*engine) error @@ -88,3 +92,12 @@ func WithSkipCreation(skipCreation bool) EngineOpt { return nil } } + +func WithTracerProvider(tp *sdktrace.TracerProvider) EngineOpt { + return func(e *engine) error { + // set to skip creating tables and indexes in the database engine + e.config.TracerProvider = tp + + return nil + } +} diff --git a/docker-compose.yml b/docker-compose.yml index 7562d1e6d..a83789d9e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -22,6 +22,7 @@ services: networks: - vela environment: + OTEL_EXPORTER_OTLP_ENDPOINT: http://jaeger:4318 DATABASE_DRIVER: postgres DATABASE_ADDR: 'postgres://vela:zB7mrKDTZqNeNTD8z47yG4DHywspAh@postgres:5432/vela?sslmode=disable' DATABASE_COMPRESSION_LEVEL: 3 @@ -52,7 +53,7 @@ services: depends_on: - postgres - redis - - vault + # - vault # The `worker` compose service hosts the Vela build daemon. # @@ -142,19 +143,19 @@ services: # This component is used for storing sensitive data like secrets. # # https://www.vaultproject.io/ - vault: - image: vault:latest - container_name: vault - command: server -dev - networks: - - vela - environment: - VAULT_DEV_LISTEN_ADDRESS: 0.0.0.0:8200 - VAULT_DEV_ROOT_TOKEN_ID: vela - ports: - - '8200:8200' - cap_add: - - IPC_LOCK + # vault: + # image: vault:latest + # container_name: vault + # command: server -dev + # networks: + # - vela + # environment: + # VAULT_DEV_LISTEN_ADDRESS: 0.0.0.0:8200 + # VAULT_DEV_ROOT_TOKEN_ID: vela + # ports: + # - '8200:8200' + # cap_add: + # - IPC_LOCK jaeger: image: jaegertracing/all-in-one:1.46 diff --git a/go.mod b/go.mod index dea6f6204..f4bc7a73c 100644 --- a/go.mod +++ b/go.mod @@ -30,7 +30,13 @@ require ( github.com/redis/go-redis/v9 v9.0.5 github.com/sirupsen/logrus v1.9.3 github.com/spf13/afero v1.9.5 + github.com/uptrace/opentelemetry-go-extra/otelgorm v0.2.2 github.com/urfave/cli/v2 v2.25.6 + go.opentelemetry.io/contrib/instrumentation/github.com/gin-gonic/gin/otelgin v0.42.0 + go.opentelemetry.io/otel v1.16.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.16.0 + go.opentelemetry.io/otel/sdk v1.16.0 go.starlark.net v0.0.0-20230612165344-9532f5667272 golang.org/x/oauth2 v0.9.0 golang.org/x/sync v0.3.0 @@ -51,6 +57,7 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/bytedance/sonic v1.9.1 // indirect github.com/cenkalti/backoff/v3 v3.0.0 // indirect + github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect github.com/cloudflare/circl v1.3.3 // indirect @@ -61,7 +68,8 @@ require ( github.com/ghodss/yaml v1.0.0 // indirect github.com/gin-contrib/sse v0.1.0 // indirect github.com/go-jose/go-jose/v3 v3.0.0 // indirect - github.com/go-logr/logr v1.2.3 // indirect + github.com/go-logr/logr v1.2.4 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-playground/validator/v10 v10.14.0 // indirect @@ -72,6 +80,7 @@ require ( github.com/google/go-querystring v1.1.0 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/gorilla/css v1.0.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-rootcerts v1.0.2 // indirect github.com/hashicorp/go-secure-stdlib/parseutil v0.1.6 // indirect @@ -112,8 +121,13 @@ require ( github.com/spf13/cast v1.3.1 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.11 // indirect + github.com/uptrace/opentelemetry-go-extra/otelsql v0.2.2 // indirect github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect github.com/yuin/gopher-lua v1.1.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0 // indirect + go.opentelemetry.io/otel/metric v1.16.0 // indirect + go.opentelemetry.io/otel/trace v1.16.0 // indirect + go.opentelemetry.io/proto/otlp v0.20.0 // indirect golang.org/x/arch v0.3.0 // indirect golang.org/x/crypto v0.10.0 // indirect golang.org/x/net v0.11.0 // indirect @@ -121,7 +135,10 @@ require ( golang.org/x/text v0.10.0 // indirect golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/protobuf v1.30.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20230626202813-9b080da550b3 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230626202813-9b080da550b3 // indirect + google.golang.org/grpc v1.56.1 // indirect + google.golang.org/protobuf v1.31.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/klog/v2 v2.90.1 // indirect diff --git a/go.sum b/go.sum index 2465b7cca..f0e283469 100644 --- a/go.sum +++ b/go.sum @@ -83,6 +83,8 @@ github.com/bytedance/sonic v1.9.1 h1:6iJ6NqdoxCDr6mbY8h18oSO+cShGSMRGCEo7F2h0x8s github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U= github.com/cenkalti/backoff/v3 v3.0.0 h1:ske+9nBpD9qZsTBoF41nW5L+AIuFBKMeze18XQ3eG1c= github.com/cenkalti/backoff/v3 v3.0.0/go.mod h1:cIeZDE3IrqwwJl6VUwCN6trj1oXrTS4rc0ij+ULvLYs= +github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= +github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -131,8 +133,11 @@ github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2 github.com/go-jose/go-jose/v3 v3.0.0 h1:s6rrhirfEP/CGIoc6p+PZAeogN2SxKav6Wp7+dyMWVo= github.com/go-jose/go-jose/v3 v3.0.0/go.mod h1:RNkWWRld676jZEYoV3+XK8L2ZnNSvIsxFMht0mSX+u8= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= -github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= +github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= @@ -151,6 +156,7 @@ github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69 github.com/golang-jwt/jwt/v5 v5.0.0 h1:1n1XNM9hk7O9mnQoNBGolZvzebBQ7p93ULHRc28XJUE= github.com/golang-jwt/jwt/v5 v5.0.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/glog v1.1.0 h1:/d3pCKDPWNnvIWe0vVUpNP32qc8U3PDVxySP/y360qE= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -227,6 +233,8 @@ github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY= github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c= github.com/goware/urlx v0.3.2 h1:gdoo4kBHlkqZNaf6XlQ12LGtQOmpKJrR04Rc3RnpJEo= github.com/goware/urlx v0.3.2/go.mod h1:h8uwbJy68o+tQXCGZNa9D73WN8n0r9OBae5bUnLcgjw= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rHAxPBD8KFhJpmcqms= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -381,6 +389,10 @@ github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU= github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= +github.com/uptrace/opentelemetry-go-extra/otelgorm v0.2.2 h1:ptBsJKjRXx46Qtw3h1CFSxLhbUfGENxXP5DAqAaAgZc= +github.com/uptrace/opentelemetry-go-extra/otelgorm v0.2.2/go.mod h1:I31DilV6DKiHDUJBEP/Bou+UZeeNDz6LqZpJCTV9q/Y= +github.com/uptrace/opentelemetry-go-extra/otelsql v0.2.2 h1:USRngIQppxeyb39XzkVHXwQesKK0+JSwnHE/1c7fgic= +github.com/uptrace/opentelemetry-go-extra/otelsql v0.2.2/go.mod h1:1frv9RN1rlTq0jzCq+mVuEQisubZCQ4OU6S/8CaHzGY= github.com/urfave/cli/v2 v2.25.6 h1:yuSkgDSZfH3L1CjF2/5fNNg2KbM47pY2EvjBq4ESQnU= github.com/urfave/cli/v2 v2.25.6/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= @@ -400,6 +412,25 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opentelemetry.io/contrib/instrumentation/github.com/gin-gonic/gin/otelgin v0.42.0 h1:l7AmwSVqozWKKXeZHycpdmpycQECRpoGwJ1FW2sWfTo= +go.opentelemetry.io/contrib/instrumentation/github.com/gin-gonic/gin/otelgin v0.42.0/go.mod h1:Ep4uoO2ijR0f49Pr7jAqyTjSCyS1SRL18wwttKfwqXA= +go.opentelemetry.io/contrib/propagators/b3 v1.17.0 h1:ImOVvHnku8jijXqkwCSyYKRDt2YrnGXD4BbhcpfbfJo= +go.opentelemetry.io/otel v1.16.0 h1:Z7GVAX/UkAXPKsy94IU+i6thsQS4nb7LviLpnaNeW8s= +go.opentelemetry.io/otel v1.16.0/go.mod h1:vl0h9NUa1D5s1nv3A5vZOYWn8av4K8Ml6JDeHrT/bx4= +go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0 h1:t4ZwRPU+emrcvM2e9DHd0Fsf0JTPVcbfa/BhTDF03d0= +go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0/go.mod h1:vLarbg68dH2Wa77g71zmKQqlQ8+8Rq3GRG31uc0WcWI= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0 h1:cbsD4cUcviQGXdw8+bo5x2wazq10SKz8hEbtCRPcU78= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0/go.mod h1:JgXSGah17croqhJfhByOLVY719k1emAXC8MVhCIJlRs= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.16.0 h1:iqjq9LAB8aK++sKVcELezzn655JnBNdsDhghU4G/So8= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.16.0/go.mod h1:hGXzO5bhhSHZnKvrDaXB82Y9DRFour0Nz/KrBh7reWw= +go.opentelemetry.io/otel/metric v1.16.0 h1:RbrpwVG1Hfv85LgnZ7+txXioPDoh6EdbZHo26Q3hqOo= +go.opentelemetry.io/otel/metric v1.16.0/go.mod h1:QE47cpOmkwipPiefDwo2wDzwJrlfxxNYodqc4xnGCo4= +go.opentelemetry.io/otel/sdk v1.16.0 h1:Z1Ok1YsijYL0CSJpHt4cS3wDDh7p572grzNrBMiMWgE= +go.opentelemetry.io/otel/sdk v1.16.0/go.mod h1:tMsIuKXuuIWPBAOrH+eHtvhTL+SntFtXF9QD68aP6p4= +go.opentelemetry.io/otel/trace v1.16.0 h1:8JRpaObFoW0pxuVPapkgH8UhHQj+bJW8jJsCZEu5MQs= +go.opentelemetry.io/otel/trace v1.16.0/go.mod h1:Yt9vYq1SdNz3xdjZZK7wcXv1qv2pwLkqr2QVwea0ef0= +go.opentelemetry.io/proto/otlp v0.20.0 h1:BLOA1cZBAGSbRiNuGCCKiFrCdYB7deeHDeD1SueyOfA= +go.opentelemetry.io/proto/otlp v0.20.0/go.mod h1:3QgjzPALBIv9pcknj2EXGPXjYPFdUh/RQfF8Lz3+Vnw= go.starlark.net v0.0.0-20230612165344-9532f5667272 h1:2/wtqS591wZyD2OsClsVBKRPEvBsQt/Js+fsCiYhwu8= go.starlark.net v0.0.0-20230612165344-9532f5667272/go.mod h1:jxU+3+j+71eXOW14274+SmmuW82qJzl6iZSeqEtTGds= golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= @@ -698,6 +729,11 @@ google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20230526203410-71b5a4ffd15e h1:Ao9GzfUMPH3zjVfzXG5rlWlk+Q8MXWKwWpwVQE1MXfw= +google.golang.org/genproto/googleapis/api v0.0.0-20230626202813-9b080da550b3 h1:wl7z+A0jkB3Rl8Hz74SqGDlnnn5VlL2CV+9UTdZOo00= +google.golang.org/genproto/googleapis/api v0.0.0-20230626202813-9b080da550b3/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230626202813-9b080da550b3 h1:QJuqz7YzNTyKDspkp2lrzqtq4lf2AhUSpXTsGP5SbLw= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230626202813-9b080da550b3/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -714,6 +750,8 @@ google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.56.1 h1:z0dNfjIl0VpaZ9iSVjA6daGatAYwPGstTjt5vkRMFkQ= +google.golang.org/grpc v1.56.1/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -726,8 +764,8 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= -google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= From ced4353181a79715f70a564a03a298c16a9345f3 Mon Sep 17 00:00:00 2001 From: davidvader Date: Wed, 28 Jun 2023 12:01:54 -0500 Subject: [PATCH 03/59] wip: added context to ONLY CreateSchedule --- api/schedule/create.go | 3 ++- cmd/vela-server/server.go | 5 ++++- database/context.go | 2 ++ database/database.go | 2 +- database/schedule/count_active_test.go | 7 +++++-- database/schedule/count_repo_test.go | 7 +++++-- database/schedule/count_test.go | 7 +++++-- database/schedule/create.go | 17 ++++++++++++++++- database/schedule/create_test.go | 5 ++++- database/schedule/delete_test.go | 5 ++++- database/schedule/get_repo_test.go | 5 ++++- database/schedule/get_test.go | 5 ++++- database/schedule/interface.go | 4 +++- database/schedule/list_active_test.go | 7 +++++-- database/schedule/list_repo_test.go | 7 +++++-- database/schedule/list_test.go | 7 +++++-- database/schedule/update_test.go | 9 +++++++-- 17 files changed, 81 insertions(+), 23 deletions(-) diff --git a/api/schedule/create.go b/api/schedule/create.go index 365df170e..18beaa030 100644 --- a/api/schedule/create.go +++ b/api/schedule/create.go @@ -183,7 +183,8 @@ func CreateSchedule(c *gin.Context) { s, _ = database.FromContext(c).GetScheduleForRepo(r, dbSchedule.GetName()) } else { // send API call to create the schedule - err = database.FromContext(c).CreateSchedule(s) + ctx := c.Request.Context() + err = database.FromContext(c).CreateSchedule(ctx, s) if err != nil { retErr := fmt.Errorf("unable to create new schedule %s: %w", r.GetName(), err) diff --git a/cmd/vela-server/server.go b/cmd/vela-server/server.go index 00ea5468a..227f5263d 100644 --- a/cmd/vela-server/server.go +++ b/cmd/vela-server/server.go @@ -67,6 +67,7 @@ func server(c *cli.Context) error { if err != nil { logrus.Fatal(err) } + defer func() { if err := tp.Shutdown(context.Background()); err != nil { logrus.Printf("Error shutting down tracer provider: %v", err) @@ -119,7 +120,9 @@ func server(c *cli.Context) error { middleware.DefaultRepoEvents(c.StringSlice("default-repo-events")), middleware.AllowlistSchedule(c.StringSlice("vela-schedule-allowlist")), middleware.ScheduleFrequency(c.Duration("schedule-minimum-frequency")), - otelgin.Middleware("vela-server"), + + // inject service middleware + otelgin.Middleware("vela-server-dv1", otelgin.WithTracerProvider(tp)), ) addr, err := url.Parse(c.String("server-addr")) diff --git a/database/context.go b/database/context.go index 1c4811213..3809ba2d5 100644 --- a/database/context.go +++ b/database/context.go @@ -31,6 +31,8 @@ func FromContext(c context.Context) Interface { return nil } + // assign as inflight property + return d } diff --git a/database/database.go b/database/database.go index 732f97d8d..78fd8216f 100644 --- a/database/database.go +++ b/database/database.go @@ -134,7 +134,7 @@ func New(opts ...EngineOpt) (Interface, error) { } logrus.Info("initializing gorm tracing") - otelPlugin := otelgorm.NewPlugin() + otelPlugin := otelgorm.NewPlugin(otelgorm.WithTracerProvider(e.config.TracerProvider)) if err := e.client.Use(otelPlugin); err != nil { return nil, err } diff --git a/database/schedule/count_active_test.go b/database/schedule/count_active_test.go index aa417efaa..c6325acd1 100644 --- a/database/schedule/count_active_test.go +++ b/database/schedule/count_active_test.go @@ -5,6 +5,7 @@ package schedule import ( + "context" "reflect" "testing" @@ -46,12 +47,14 @@ func TestSchedule_Engine_CountActiveSchedules(t *testing.T) { _sqlite := testSqlite(t) defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() - err := _sqlite.CreateSchedule(_scheduleOne) + ctx := context.TODO() + + err := _sqlite.CreateSchedule(ctx, _scheduleOne) if err != nil { t.Errorf("unable to create test schedule for sqlite: %v", err) } - err = _sqlite.CreateSchedule(_scheduleTwo) + err = _sqlite.CreateSchedule(ctx, _scheduleTwo) if err != nil { t.Errorf("unable to create test schedule for sqlite: %v", err) } diff --git a/database/schedule/count_repo_test.go b/database/schedule/count_repo_test.go index 792c21b1d..cf959ff52 100644 --- a/database/schedule/count_repo_test.go +++ b/database/schedule/count_repo_test.go @@ -5,6 +5,7 @@ package schedule import ( + "context" "reflect" "testing" @@ -50,12 +51,14 @@ func TestSchedule_Engine_CountSchedulesForRepo(t *testing.T) { _sqlite := testSqlite(t) defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() - err := _sqlite.CreateSchedule(_scheduleOne) + ctx := context.TODO() + + err := _sqlite.CreateSchedule(ctx, _scheduleOne) if err != nil { t.Errorf("unable to create test schedule for sqlite: %v", err) } - err = _sqlite.CreateSchedule(_scheduleTwo) + err = _sqlite.CreateSchedule(ctx, _scheduleTwo) if err != nil { t.Errorf("unable to create test schedule for sqlite: %v", err) } diff --git a/database/schedule/count_test.go b/database/schedule/count_test.go index 01cfbbae4..121cb5b64 100644 --- a/database/schedule/count_test.go +++ b/database/schedule/count_test.go @@ -5,6 +5,7 @@ package schedule import ( + "context" "reflect" "testing" @@ -44,12 +45,14 @@ func TestSchedule_Engine_CountSchedules(t *testing.T) { _sqlite := testSqlite(t) defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() - err := _sqlite.CreateSchedule(_scheduleOne) + ctx := context.TODO() + + err := _sqlite.CreateSchedule(ctx, _scheduleOne) if err != nil { t.Errorf("unable to create test schedule for sqlite: %v", err) } - err = _sqlite.CreateSchedule(_scheduleTwo) + err = _sqlite.CreateSchedule(ctx, _scheduleTwo) if err != nil { t.Errorf("unable to create test schedule for sqlite: %v", err) } diff --git a/database/schedule/create.go b/database/schedule/create.go index a6e532e8c..b6c8a9b6c 100644 --- a/database/schedule/create.go +++ b/database/schedule/create.go @@ -6,18 +6,32 @@ package schedule import ( + "context" + "github.com/go-vela/types/constants" "github.com/go-vela/types/database" "github.com/go-vela/types/library" "github.com/sirupsen/logrus" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) // CreateSchedule creates a new schedule in the database. -func (e *engine) CreateSchedule(s *library.Schedule) error { +func (e *engine) CreateSchedule(ctx context.Context, s *library.Schedule) error { e.logger.WithFields(logrus.Fields{ "schedule": s.GetName(), }).Tracef("creating schedule %s in the database", s.GetName()) + span := trace.SpanFromContext(ctx) + + if span.IsRecording() { + span.SetAttributes( + attribute.String("db", "do-operation"), + ) + } + + logrus.Infof("CreateSchedule using span_id:", span.SpanContext().TraceID()) + // cast the library type to database type schedule := database.ScheduleFromLibrary(s) @@ -29,6 +43,7 @@ func (e *engine) CreateSchedule(s *library.Schedule) error { // send query to the database return e.client. + WithContext(ctx). Table(constants.TableSchedule). Create(schedule). Error diff --git a/database/schedule/create_test.go b/database/schedule/create_test.go index 015654151..3e7100e55 100644 --- a/database/schedule/create_test.go +++ b/database/schedule/create_test.go @@ -5,6 +5,7 @@ package schedule import ( + "context" "testing" "github.com/DATA-DOG/go-sqlmock" @@ -55,10 +56,12 @@ VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10) RETURNING "id"`). }, } + ctx := context.TODO() + // run tests for _, test := range tests { t.Run(test.name, func(t *testing.T) { - err := test.database.CreateSchedule(_schedule) + err := test.database.CreateSchedule(ctx, _schedule) if test.failure { if err == nil { diff --git a/database/schedule/delete_test.go b/database/schedule/delete_test.go index 3d1a3cd48..3a7b58b6f 100644 --- a/database/schedule/delete_test.go +++ b/database/schedule/delete_test.go @@ -5,6 +5,7 @@ package schedule import ( + "context" "testing" "github.com/DATA-DOG/go-sqlmock" @@ -32,7 +33,9 @@ func TestSchedule_Engine_DeleteSchedule(t *testing.T) { _sqlite := testSqlite(t) defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() - err := _sqlite.CreateSchedule(_schedule) + ctx := context.TODO() + + err := _sqlite.CreateSchedule(ctx, _schedule) if err != nil { t.Errorf("unable to create test schedule for sqlite: %v", err) } diff --git a/database/schedule/get_repo_test.go b/database/schedule/get_repo_test.go index fb29bea37..629d11117 100644 --- a/database/schedule/get_repo_test.go +++ b/database/schedule/get_repo_test.go @@ -5,6 +5,7 @@ package schedule import ( + "context" "reflect" "testing" @@ -43,7 +44,9 @@ func TestSchedule_Engine_GetScheduleForRepo(t *testing.T) { _sqlite := testSqlite(t) defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() - err := _sqlite.CreateSchedule(_schedule) + ctx := context.TODO() + + err := _sqlite.CreateSchedule(ctx, _schedule) if err != nil { t.Errorf("unable to create test schedule for sqlite: %v", err) } diff --git a/database/schedule/get_test.go b/database/schedule/get_test.go index acea35b2a..02771bf30 100644 --- a/database/schedule/get_test.go +++ b/database/schedule/get_test.go @@ -5,6 +5,7 @@ package schedule import ( + "context" "reflect" "testing" @@ -37,7 +38,9 @@ func TestSchedule_Engine_GetSchedule(t *testing.T) { _sqlite := testSqlite(t) defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() - err := _sqlite.CreateSchedule(_schedule) + ctx := context.TODO() + + err := _sqlite.CreateSchedule(ctx, _schedule) if err != nil { t.Errorf("unable to create test schedule for sqlite: %v", err) } diff --git a/database/schedule/interface.go b/database/schedule/interface.go index afa163063..8a24599c7 100644 --- a/database/schedule/interface.go +++ b/database/schedule/interface.go @@ -5,6 +5,8 @@ package schedule import ( + "context" + "github.com/go-vela/types/library" ) @@ -31,7 +33,7 @@ type ScheduleInterface interface { // CountSchedulesForRepo defines a function that gets the count of schedules by repo ID. CountSchedulesForRepo(*library.Repo) (int64, error) // CreateSchedule defines a function that creates a new schedule. - CreateSchedule(*library.Schedule) error + CreateSchedule(context.Context, *library.Schedule) error // DeleteSchedule defines a function that deletes an existing schedule. DeleteSchedule(*library.Schedule) error // GetSchedule defines a function that gets a schedule by ID. diff --git a/database/schedule/list_active_test.go b/database/schedule/list_active_test.go index 69c690def..f098b9648 100644 --- a/database/schedule/list_active_test.go +++ b/database/schedule/list_active_test.go @@ -5,6 +5,7 @@ package schedule import ( + "context" "reflect" "testing" @@ -55,12 +56,14 @@ func TestSchedule_Engine_ListActiveSchedules(t *testing.T) { _sqlite := testSqlite(t) defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() - err := _sqlite.CreateSchedule(_scheduleOne) + ctx := context.TODO() + + err := _sqlite.CreateSchedule(ctx, _scheduleOne) if err != nil { t.Errorf("unable to create test schedule for sqlite: %v", err) } - err = _sqlite.CreateSchedule(_scheduleTwo) + err = _sqlite.CreateSchedule(ctx, _scheduleTwo) if err != nil { t.Errorf("unable to create test schedule for sqlite: %v", err) } diff --git a/database/schedule/list_repo_test.go b/database/schedule/list_repo_test.go index df56c0ee4..0007a5d8a 100644 --- a/database/schedule/list_repo_test.go +++ b/database/schedule/list_repo_test.go @@ -5,6 +5,7 @@ package schedule import ( + "context" "reflect" "testing" @@ -59,12 +60,14 @@ func TestSchedule_Engine_ListSchedulesForRepo(t *testing.T) { _sqlite := testSqlite(t) defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() - err := _sqlite.CreateSchedule(_scheduleOne) + ctx := context.TODO() + + err := _sqlite.CreateSchedule(ctx, _scheduleOne) if err != nil { t.Errorf("unable to create test schedule for sqlite: %v", err) } - err = _sqlite.CreateSchedule(_scheduleTwo) + err = _sqlite.CreateSchedule(ctx, _scheduleTwo) if err != nil { t.Errorf("unable to create test schedule for sqlite: %v", err) } diff --git a/database/schedule/list_test.go b/database/schedule/list_test.go index b0c3bb417..05856ed1e 100644 --- a/database/schedule/list_test.go +++ b/database/schedule/list_test.go @@ -5,6 +5,7 @@ package schedule import ( + "context" "reflect" "testing" @@ -54,12 +55,14 @@ func TestSchedule_Engine_ListSchedules(t *testing.T) { _sqlite := testSqlite(t) defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() - err := _sqlite.CreateSchedule(_scheduleOne) + ctx := context.TODO() + + err := _sqlite.CreateSchedule(ctx, _scheduleOne) if err != nil { t.Errorf("unable to create test schedule for sqlite: %v", err) } - err = _sqlite.CreateSchedule(_scheduleTwo) + err = _sqlite.CreateSchedule(ctx, _scheduleTwo) if err != nil { t.Errorf("unable to create test schedule for sqlite: %v", err) } diff --git a/database/schedule/update_test.go b/database/schedule/update_test.go index 2554cefdc..5f97ca003 100644 --- a/database/schedule/update_test.go +++ b/database/schedule/update_test.go @@ -5,6 +5,7 @@ package schedule import ( + "context" "testing" "time" @@ -41,7 +42,9 @@ WHERE "id" = $10`). _sqlite := testSqlite(t) defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() - err := _sqlite.CreateSchedule(_schedule) + ctx := context.TODO() + + err := _sqlite.CreateSchedule(ctx, _schedule) if err != nil { t.Errorf("unable to create test schedule for sqlite: %v", err) } @@ -113,7 +116,9 @@ func TestSchedule_Engine_UpdateSchedule_NotConfig(t *testing.T) { _sqlite := testSqlite(t) defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() - err := _sqlite.CreateSchedule(_schedule) + ctx := context.TODO() + + err := _sqlite.CreateSchedule(ctx, _schedule) if err != nil { t.Errorf("unable to create test schedule for sqlite: %v", err) } From fe6b3eb255617dafc2dd13f3d0fc186c966b4fe9 Mon Sep 17 00:00:00 2001 From: Jordan Sussman Date: Wed, 28 Jun 2023 13:31:15 -0500 Subject: [PATCH 04/59] update docker compose --- docker-compose.yml | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index a83789d9e..c11bf0bf3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -158,24 +158,15 @@ services: # - IPC_LOCK jaeger: - image: jaegertracing/all-in-one:1.46 + image: jaegertracing/all-in-one:latest container_name: jaeger networks: - vela environment: - COLLECTOR_ZIPKIN_HOST_PORT: ":9411" COLLECTOR_OTLP_ENABLED: true ports: - - '6831:6831/udp' - - '6832:6832/udp' - - '5778:5778' - '16686:16686' - - '4317:4317' - '4318:4318' - - '14250:14250' - - '14268:14268' - - '14269:14269' - - '9411:9411' networks: vela: \ No newline at end of file From 7752be5a38e3dea7b69989f30bc30433f6fcd02c Mon Sep 17 00:00:00 2001 From: Jordan Sussman Date: Wed, 28 Jun 2023 13:58:26 -0500 Subject: [PATCH 05/59] attempt to mask gorm values --- database/database.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/database/database.go b/database/database.go index 78fd8216f..df558863a 100644 --- a/database/database.go +++ b/database/database.go @@ -134,7 +134,7 @@ func New(opts ...EngineOpt) (Interface, error) { } logrus.Info("initializing gorm tracing") - otelPlugin := otelgorm.NewPlugin(otelgorm.WithTracerProvider(e.config.TracerProvider)) + otelPlugin := otelgorm.NewPlugin(otelgorm.WithTracerProvider(e.config.TracerProvider), otelgorm.WithoutQueryVariables()) if err := e.client.Use(otelPlugin); err != nil { return nil, err } From 0799b9107040deef54dfebf781890e801827af00 Mon Sep 17 00:00:00 2001 From: JordanBrockopp Date: Wed, 28 Jun 2023 14:16:46 -0500 Subject: [PATCH 06/59] chore: save work --- api/repo/create.go | 4 ++-- api/repo/delete.go | 2 +- api/repo/repair.go | 4 ++-- api/schedule/create.go | 3 +-- api/scm/sync.go | 2 +- api/scm/sync_org.go | 2 +- database/schedule/create.go | 11 +++++------ go.mod | 5 ++++- go.sum | 6 ++++++ scm/github/access.go | 9 +++++---- scm/github/authentication.go | 2 +- scm/github/changeset.go | 5 +++-- scm/github/deployment.go | 9 +++++---- scm/github/github.go | 16 ++++++++++++++-- scm/github/github_test.go | 2 +- scm/github/org.go | 3 ++- scm/github/repo.go | 29 +++++++++++++++-------------- scm/github/repo_test.go | 15 ++++++++------- scm/github/webhook.go | 2 +- scm/github/webhook_test.go | 2 +- scm/service.go | 6 +++--- 21 files changed, 82 insertions(+), 57 deletions(-) diff --git a/api/repo/create.go b/api/repo/create.go index e52cd23f4..b4b9d5222 100644 --- a/api/repo/create.go +++ b/api/repo/create.go @@ -98,7 +98,7 @@ func CreateRepo(c *gin.Context) { }).Infof("creating new repo %s", input.GetFullName()) // get repo information from the source - r, err := scm.FromContext(c).GetRepo(u, input) + r, err := scm.FromContext(c).GetRepo(c.Request.Context(), u, input) if err != nil { retErr := fmt.Errorf("unable to retrieve repo info for %s from source: %w", r.GetFullName(), err) @@ -256,7 +256,7 @@ func CreateRepo(c *gin.Context) { // check if we should create the webhook if c.Value("webhookvalidation").(bool) { // send API call to create the webhook - h, _, err = scm.FromContext(c).Enable(u, r, h) + h, _, err = scm.FromContext(c).Enable(c.Request.Context(), u, r, h) if err != nil { retErr := fmt.Errorf("unable to create webhook for %s: %w", r.GetFullName(), err) diff --git a/api/repo/delete.go b/api/repo/delete.go index 63649f2eb..0237da953 100644 --- a/api/repo/delete.go +++ b/api/repo/delete.go @@ -70,7 +70,7 @@ func DeleteRepo(c *gin.Context) { }).Infof("deleting repo %s", r.GetFullName()) // send API call to remove the webhook - err := scm.FromContext(c).Disable(u, r.GetOrg(), r.GetName()) + err := scm.FromContext(c).Disable(c.Request.Context(), u, r.GetOrg(), r.GetName()) if err != nil { retErr := fmt.Errorf("unable to delete webhook for %s: %w", r.GetFullName(), err) diff --git a/api/repo/repair.go b/api/repo/repair.go index 192983d1c..4e1854307 100644 --- a/api/repo/repair.go +++ b/api/repo/repair.go @@ -68,7 +68,7 @@ func RepairRepo(c *gin.Context) { // check if we should create the webhook if c.Value("webhookvalidation").(bool) { // send API call to remove the webhook - err := scm.FromContext(c).Disable(u, r.GetOrg(), r.GetName()) + err := scm.FromContext(c).Disable(c.Request.Context(), u, r.GetOrg(), r.GetName()) if err != nil { retErr := fmt.Errorf("unable to delete webhook for %s: %w", r.GetFullName(), err) @@ -87,7 +87,7 @@ func RepairRepo(c *gin.Context) { } // send API call to create the webhook - hook, _, err = scm.FromContext(c).Enable(u, r, hook) + hook, _, err = scm.FromContext(c).Enable(c.Request.Context(), u, r, hook) if err != nil { retErr := fmt.Errorf("unable to create webhook for %s: %w", r.GetFullName(), err) diff --git a/api/schedule/create.go b/api/schedule/create.go index 18beaa030..cb40d68de 100644 --- a/api/schedule/create.go +++ b/api/schedule/create.go @@ -183,8 +183,7 @@ func CreateSchedule(c *gin.Context) { s, _ = database.FromContext(c).GetScheduleForRepo(r, dbSchedule.GetName()) } else { // send API call to create the schedule - ctx := c.Request.Context() - err = database.FromContext(c).CreateSchedule(ctx, s) + err = database.FromContext(c).CreateSchedule(c.Request.Context(), s) if err != nil { retErr := fmt.Errorf("unable to create new schedule %s: %w", r.GetName(), err) diff --git a/api/scm/sync.go b/api/scm/sync.go index b160f5fa0..bb2443c21 100644 --- a/api/scm/sync.go +++ b/api/scm/sync.go @@ -72,7 +72,7 @@ func SyncRepo(c *gin.Context) { logger.Infof("syncing repo %s", r.GetFullName()) // retrieve repo from source code manager service - _, err := scm.FromContext(c).GetRepo(u, r) + _, err := scm.FromContext(c).GetRepo(c.Request.Context(), u, r) // if there is an error retrieving repo, we know it is deleted: set to inactive if err != nil { diff --git a/api/scm/sync_org.go b/api/scm/sync_org.go index ac18ec551..15b29f5fe 100644 --- a/api/scm/sync_org.go +++ b/api/scm/sync_org.go @@ -108,7 +108,7 @@ func SyncReposForOrg(c *gin.Context) { // iterate through captured repos and check if they are in GitHub for _, repo := range repos { - _, err := scm.FromContext(c).GetRepo(u, repo) + _, err := scm.FromContext(c).GetRepo(c.Request.Context(), u, repo) // if repo cannot be captured from GitHub, set to inactive in database if err != nil { repo.SetActive(false) diff --git a/database/schedule/create.go b/database/schedule/create.go index b6c8a9b6c..333512b9e 100644 --- a/database/schedule/create.go +++ b/database/schedule/create.go @@ -18,19 +18,18 @@ import ( // CreateSchedule creates a new schedule in the database. func (e *engine) CreateSchedule(ctx context.Context, s *library.Schedule) error { - e.logger.WithFields(logrus.Fields{ - "schedule": s.GetName(), - }).Tracef("creating schedule %s in the database", s.GetName()) - span := trace.SpanFromContext(ctx) - if span.IsRecording() { span.SetAttributes( attribute.String("db", "do-operation"), ) } - logrus.Infof("CreateSchedule using span_id:", span.SpanContext().TraceID()) + e.logger.WithFields(logrus.Fields{ + "schedule": s.GetName(), + "span_id": trace.SpanFromContext(ctx).SpanContext().SpanID(), + "trace_id": trace.SpanFromContext(ctx).SpanContext().TraceID(), + }).Tracef("creating schedule %s in the database", s.GetName()) // cast the library type to database type schedule := database.ScheduleFromLibrary(s) diff --git a/go.mod b/go.mod index f4bc7a73c..892241175 100644 --- a/go.mod +++ b/go.mod @@ -33,10 +33,13 @@ require ( github.com/uptrace/opentelemetry-go-extra/otelgorm v0.2.2 github.com/urfave/cli/v2 v2.25.6 go.opentelemetry.io/contrib/instrumentation/github.com/gin-gonic/gin/otelgin v0.42.0 + go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.42.0 + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.42.0 go.opentelemetry.io/otel v1.16.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.16.0 go.opentelemetry.io/otel/sdk v1.16.0 + go.opentelemetry.io/otel/trace v1.16.0 go.starlark.net v0.0.0-20230612165344-9532f5667272 golang.org/x/oauth2 v0.9.0 golang.org/x/sync v0.3.0 @@ -64,6 +67,7 @@ require ( github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/fatih/color v1.10.0 // indirect + github.com/felixge/httpsnoop v1.0.3 // indirect github.com/gabriel-vasile/mimetype v1.4.2 // indirect github.com/ghodss/yaml v1.0.0 // indirect github.com/gin-contrib/sse v0.1.0 // indirect @@ -126,7 +130,6 @@ require ( github.com/yuin/gopher-lua v1.1.0 // indirect go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0 // indirect go.opentelemetry.io/otel/metric v1.16.0 // indirect - go.opentelemetry.io/otel/trace v1.16.0 // indirect go.opentelemetry.io/proto/otlp v0.20.0 // indirect golang.org/x/arch v0.3.0 // indirect golang.org/x/crypto v0.10.0 // indirect diff --git a/go.sum b/go.sum index f0e283469..16387e1a2 100644 --- a/go.sum +++ b/go.sum @@ -119,6 +119,8 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg= github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= +github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk= +github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU= github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= @@ -414,6 +416,10 @@ go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opentelemetry.io/contrib/instrumentation/github.com/gin-gonic/gin/otelgin v0.42.0 h1:l7AmwSVqozWKKXeZHycpdmpycQECRpoGwJ1FW2sWfTo= go.opentelemetry.io/contrib/instrumentation/github.com/gin-gonic/gin/otelgin v0.42.0/go.mod h1:Ep4uoO2ijR0f49Pr7jAqyTjSCyS1SRL18wwttKfwqXA= +go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.42.0 h1:0vzgiFDsCh/jxRCR1xcRrtMoeCu2itXz/PsXst5P8rI= +go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.42.0/go.mod h1:y0vOY2OKFMOTvwxKfurStPayUUKGHlNeVqNneHmFXr0= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.42.0 h1:pginetY7+onl4qN1vl0xW/V/v6OBZ0vVdH+esuJgvmM= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.42.0/go.mod h1:XiYsayHc36K3EByOO6nbAXnAWbrUxdjUROCEeeROOH8= go.opentelemetry.io/contrib/propagators/b3 v1.17.0 h1:ImOVvHnku8jijXqkwCSyYKRDt2YrnGXD4BbhcpfbfJo= go.opentelemetry.io/otel v1.16.0 h1:Z7GVAX/UkAXPKsy94IU+i6thsQS4nb7LviLpnaNeW8s= go.opentelemetry.io/otel v1.16.0/go.mod h1:vl0h9NUa1D5s1nv3A5vZOYWn8av4K8Ml6JDeHrT/bx4= diff --git a/scm/github/access.go b/scm/github/access.go index ee47cf3d1..0918d8db3 100644 --- a/scm/github/access.go +++ b/scm/github/access.go @@ -5,6 +5,7 @@ package github import ( + "context" "strings" "github.com/sirupsen/logrus" @@ -32,7 +33,7 @@ func (c *client) OrgAccess(u *library.User, org string) (string, error) { } // create GitHub OAuth client with user's token - client := c.newClientToken(*u.Token) + client := c.newClientToken(context.TODO(), *u.Token) // send API call to capture org access level for user membership, _, err := client.Organizations.GetOrgMembership(ctx, *u.Name, org) @@ -68,7 +69,7 @@ func (c *client) RepoAccess(u *library.User, token, org, repo string) (string, e } // create github oauth client with the given token - client := c.newClientToken(token) + client := c.newClientToken(context.TODO(), token) // send API call to capture repo access level for user perm, _, err := client.Repositories.GetPermissionLevel(ctx, org, repo, u.GetName()) @@ -99,7 +100,7 @@ func (c *client) TeamAccess(u *library.User, org, team string) (string, error) { } // create GitHub OAuth client with user's token - client := c.newClientToken(u.GetToken()) + client := c.newClientToken(context.TODO(), u.GetToken()) teams := []*github.Team{} // set the max per page for the options to capture the list of repos @@ -149,7 +150,7 @@ func (c *client) ListUsersTeamsForOrg(u *library.User, org string) ([]string, er }).Tracef("capturing %s team membership for org %s", u.GetName(), org) // create GitHub OAuth client with user's token - client := c.newClientToken(u.GetToken()) + client := c.newClientToken(context.TODO(), u.GetToken()) teams := []*github.Team{} // set the max per page for the options to capture the list of repos diff --git a/scm/github/authentication.go b/scm/github/authentication.go index 3b4ccf14b..4d3aa6333 100644 --- a/scm/github/authentication.go +++ b/scm/github/authentication.go @@ -22,7 +22,7 @@ func (c *client) Authorize(token string) (string, error) { c.Logger.Trace("authorizing user with token") // create GitHub OAuth client with user's token - client := c.newClientToken(token) + client := c.newClientToken(context.TODO(), token) // send API call to capture the current user making the call u, _, err := client.Users.Get(ctx, "") diff --git a/scm/github/changeset.go b/scm/github/changeset.go index da8ca221a..f77fa045c 100644 --- a/scm/github/changeset.go +++ b/scm/github/changeset.go @@ -5,6 +5,7 @@ package github import ( + "context" "fmt" "github.com/sirupsen/logrus" @@ -22,7 +23,7 @@ func (c *client) Changeset(u *library.User, r *library.Repo, sha string) ([]stri }).Tracef("capturing commit changeset for %s/commit/%s", r.GetFullName(), sha) // create GitHub OAuth client with user's token - client := c.newClientToken(u.GetToken()) + client := c.newClientToken(context.TODO(), u.GetToken()) s := []string{} // set the max per page for the options to capture the commit @@ -51,7 +52,7 @@ func (c *client) ChangesetPR(u *library.User, r *library.Repo, number int) ([]st }).Tracef("capturing pull request changeset for %s/pull/%d", r.GetFullName(), number) // create GitHub OAuth client with user's token - client := c.newClientToken(u.GetToken()) + client := c.newClientToken(context.TODO(), u.GetToken()) s := []string{} f := []*github.CommitFile{} diff --git a/scm/github/deployment.go b/scm/github/deployment.go index fea3f8d03..58169291f 100644 --- a/scm/github/deployment.go +++ b/scm/github/deployment.go @@ -5,6 +5,7 @@ package github import ( + "context" "encoding/json" "github.com/sirupsen/logrus" @@ -23,7 +24,7 @@ func (c *client) GetDeployment(u *library.User, r *library.Repo, id int64) (*lib }).Tracef("capturing deployment %d for repo %s", id, r.GetFullName()) // create GitHub OAuth client with user's token - client := c.newClientToken(*u.Token) + client := c.newClientToken(context.TODO(), *u.Token) // send API call to capture the deployment deployment, _, err := client.Repositories.GetDeployment(ctx, r.GetOrg(), r.GetName(), id) @@ -61,7 +62,7 @@ func (c *client) GetDeploymentCount(u *library.User, r *library.Repo) (int64, er }).Tracef("counting deployments for repo %s", r.GetFullName()) // create GitHub OAuth client with user's token - client := c.newClientToken(*u.Token) + client := c.newClientToken(context.TODO(), *u.Token) // create variable to track the deployments deployments := []*github.Deployment{} @@ -103,7 +104,7 @@ func (c *client) GetDeploymentList(u *library.User, r *library.Repo, page, perPa }).Tracef("listing deployments for repo %s", r.GetFullName()) // create GitHub OAuth client with user's token - client := c.newClientToken(*u.Token) + client := c.newClientToken(context.TODO(), *u.Token) // set pagination options for listing deployments opts := &github.DeploymentsListOptions{ @@ -157,7 +158,7 @@ func (c *client) CreateDeployment(u *library.User, r *library.Repo, d *library.D }).Tracef("creating deployment for repo %s", r.GetFullName()) // create GitHub OAuth client with user's token - client := c.newClientToken(*u.Token) + client := c.newClientToken(context.TODO(), *u.Token) var payload interface{} if d.Payload == nil { diff --git a/scm/github/github.go b/scm/github/github.go index ebae7ec76..0f358affd 100644 --- a/scm/github/github.go +++ b/scm/github/github.go @@ -7,12 +7,17 @@ package github import ( "context" "fmt" + "net/http" + "net/http/httptrace" "net/url" "github.com/google/go-github/v53/github" "github.com/sirupsen/logrus" "golang.org/x/oauth2" + + "go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace" + "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" ) const ( @@ -143,14 +148,14 @@ func NewTest(urls ...string) (*client, error) { } // helper function to return the GitHub OAuth client. -func (c *client) newClientToken(token string) *github.Client { +func (c *client) newClientToken(ctx context.Context, token string) *github.Client { // create the token object for the client ts := oauth2.StaticTokenSource( &oauth2.Token{AccessToken: token}, ) // create the OAuth client - tc := oauth2.NewClient(context.Background(), ts) + tc := oauth2.NewClient(ctx, ts) // if c.SkipVerify { // tc.Transport.(*oauth2.Transport).Base = &http.Transport{ // Proxy: http.ProxyFromEnvironment, @@ -160,6 +165,13 @@ func (c *client) newClientToken(token string) *github.Client { // } // } + tc.Transport = otelhttp.NewTransport( + http.DefaultTransport, + otelhttp.WithClientTrace(func(ctx context.Context) *httptrace.ClientTrace { + return otelhttptrace.NewClientTrace(ctx) + }), + ) + // create the GitHub client from the OAuth client github := github.NewClient(tc) diff --git a/scm/github/github_test.go b/scm/github/github_test.go index 1c10db60a..ce1548018 100644 --- a/scm/github/github_test.go +++ b/scm/github/github_test.go @@ -74,7 +74,7 @@ func TestGithub_newClientToken(t *testing.T) { client, _ := NewTest(s.URL) // run test - got := client.newClientToken("foobar") + got := client.newClientToken(context.TODO(), "foobar") //nolint:staticcheck // ignore false positive if got == nil { diff --git a/scm/github/org.go b/scm/github/org.go index ef9e43ca4..751bc1c1f 100644 --- a/scm/github/org.go +++ b/scm/github/org.go @@ -5,6 +5,7 @@ package github import ( + "context" "net/http" "github.com/sirupsen/logrus" @@ -20,7 +21,7 @@ func (c *client) GetOrgName(u *library.User, o string) (string, error) { }).Tracef("retrieving org information for %s", o) // create GitHub OAuth client with user's token - client := c.newClientToken(u.GetToken()) + client := c.newClientToken(context.TODO(), u.GetToken()) // send an API call to get the org info orgInfo, resp, err := client.Organizations.Get(ctx, o) diff --git a/scm/github/repo.go b/scm/github/repo.go index 682a2db94..7d5eb4767 100644 --- a/scm/github/repo.go +++ b/scm/github/repo.go @@ -5,6 +5,7 @@ package github import ( + "context" "fmt" "net/http" "strconv" @@ -56,7 +57,7 @@ func (c *client) Config(u *library.User, r *library.Repo, ref string) ([]byte, e }).Tracef("capturing configuration file for %s/commit/%s", r.GetFullName(), ref) // create GitHub OAuth client with user's token - client := c.newClientToken(*u.Token) + client := c.newClientToken(context.TODO(), *u.Token) files := []string{".vela.yml", ".vela.yaml"} @@ -93,7 +94,7 @@ func (c *client) Config(u *library.User, r *library.Repo, ref string) ([]byte, e } // Disable deactivates a repo by deleting the webhook. -func (c *client) Disable(u *library.User, org, name string) error { +func (c *client) Disable(ctx context.Context, u *library.User, org, name string) error { c.Logger.WithFields(logrus.Fields{ "org": org, "repo": name, @@ -101,7 +102,7 @@ func (c *client) Disable(u *library.User, org, name string) error { }).Tracef("deleting repository webhooks for %s/%s", org, name) // create GitHub OAuth client with user's token - client := c.newClientToken(*u.Token) + client := c.newClientToken(ctx, *u.Token) // send API call to capture the hooks for the repo hooks, _, err := client.Repositories.ListHooks(ctx, org, name, nil) @@ -151,7 +152,7 @@ func (c *client) Disable(u *library.User, org, name string) error { } // Enable activates a repo by creating the webhook. -func (c *client) Enable(u *library.User, r *library.Repo, h *library.Hook) (*library.Hook, string, error) { +func (c *client) Enable(ctx context.Context, u *library.User, r *library.Repo, h *library.Hook) (*library.Hook, string, error) { c.Logger.WithFields(logrus.Fields{ "org": r.GetOrg(), "repo": r.GetName(), @@ -159,7 +160,7 @@ func (c *client) Enable(u *library.User, r *library.Repo, h *library.Hook) (*lib }).Tracef("creating repository webhook for %s/%s", r.GetOrg(), r.GetName()) // create GitHub OAuth client with user's token - client := c.newClientToken(*u.Token) + client := c.newClientToken(ctx, *u.Token) // always listen to repository events in case of repo name change events := []string{eventRepository} @@ -225,7 +226,7 @@ func (c *client) Update(u *library.User, r *library.Repo, hookID int64) error { }).Tracef("updating repository webhook for %s/%s", r.GetOrg(), r.GetName()) // create GitHub OAuth client with user's token - client := c.newClientToken(*u.Token) + client := c.newClientToken(context.TODO(), *u.Token) // always listen to repository events in case of repo name change events := []string{eventRepository} @@ -277,7 +278,7 @@ func (c *client) Status(u *library.User, b *library.Build, org, name string) err }).Tracef("setting commit status for %s/%s/%d @ %s", org, name, b.GetNumber(), b.GetCommit()) // create GitHub OAuth client with user's token - client := c.newClientToken(*u.Token) + client := c.newClientToken(context.TODO(), *u.Token) context := fmt.Sprintf("%s/%s", c.config.StatusContext, b.GetEvent()) url := fmt.Sprintf("%s/%s/%s/%d", c.config.WebUIAddress, org, name, b.GetNumber()) @@ -370,7 +371,7 @@ func (c *client) Status(u *library.User, b *library.Build, org, name string) err } // GetRepo gets repo information from Github. -func (c *client) GetRepo(u *library.User, r *library.Repo) (*library.Repo, error) { +func (c *client) GetRepo(ctx context.Context, u *library.User, r *library.Repo) (*library.Repo, error) { c.Logger.WithFields(logrus.Fields{ "org": r.GetOrg(), "repo": r.GetName(), @@ -378,7 +379,7 @@ func (c *client) GetRepo(u *library.User, r *library.Repo) (*library.Repo, error }).Tracef("retrieving repository information for %s", r.GetFullName()) // create GitHub OAuth client with user's token - client := c.newClientToken(u.GetToken()) + client := c.newClientToken(ctx, u.GetToken()) // send an API call to get the repo info repo, _, err := client.Repositories.Get(ctx, r.GetOrg(), r.GetName()) @@ -398,7 +399,7 @@ func (c *client) GetOrgAndRepoName(u *library.User, o string, r string) (string, }).Tracef("retrieving repository information for %s/%s", o, r) // create GitHub OAuth client with user's token - client := c.newClientToken(u.GetToken()) + client := c.newClientToken(context.TODO(), u.GetToken()) // send an API call to get the repo info repo, _, err := client.Repositories.Get(ctx, o, r) @@ -416,7 +417,7 @@ func (c *client) ListUserRepos(u *library.User) ([]*library.Repo, error) { }).Tracef("listing source repositories for %s", u.GetName()) // create GitHub OAuth client with user's token - client := c.newClientToken(u.GetToken()) + client := c.newClientToken(context.TODO(), u.GetToken()) r := []*github.Repository{} f := []*library.Repo{} @@ -487,7 +488,7 @@ func (c *client) GetPullRequest(u *library.User, r *library.Repo, number int) (s }).Tracef("retrieving pull request %d for repo %s", number, r.GetFullName()) // create GitHub OAuth client with user's token - client := c.newClientToken(u.GetToken()) + client := c.newClientToken(context.TODO(), u.GetToken()) pull, _, err := client.PullRequests.Get(ctx, r.GetOrg(), r.GetName(), number) if err != nil { @@ -511,7 +512,7 @@ func (c *client) GetHTMLURL(u *library.User, org, repo, name, ref string) (strin }).Tracef("capturing html_url for %s/%s/%s@%s", org, repo, name, ref) // create GitHub OAuth client with user's token - client := c.newClientToken(*u.Token) + client := c.newClientToken(context.TODO(), *u.Token) // set the reference for the options to capture the repository contents opts := &github.RepositoryContentGetOptions{ @@ -547,7 +548,7 @@ func (c *client) GetBranch(u *library.User, r *library.Repo) (string, string, er }).Tracef("retrieving branch %s for repo %s", r.GetBranch(), r.GetFullName()) // create GitHub OAuth client with user's token - client := c.newClientToken(u.GetToken()) + client := c.newClientToken(context.TODO(), u.GetToken()) data, _, err := client.Repositories.GetBranch(ctx, r.GetOrg(), r.GetName(), r.GetBranch(), true) if err != nil { diff --git a/scm/github/repo_test.go b/scm/github/repo_test.go index 7482c71ce..7c2e77a08 100644 --- a/scm/github/repo_test.go +++ b/scm/github/repo_test.go @@ -5,6 +5,7 @@ package github import ( + "context" "fmt" "net/http" "net/http/httptest" @@ -450,7 +451,7 @@ func TestGithub_Disable(t *testing.T) { client, _ := NewTest(s.URL, "https://foo.bar.com") // run test - err := client.Disable(u, "foo", "bar") + err := client.Disable(context.TODO(), u, "foo", "bar") if resp.Code != http.StatusOK { t.Errorf("Disable returned %v, want %v", resp.Code, http.StatusOK) @@ -484,7 +485,7 @@ func TestGithub_Disable_NotFoundHooks(t *testing.T) { client, _ := NewTest(s.URL, "https://foo.bar.com") // run test - err := client.Disable(u, "foo", "bar") + err := client.Disable(context.TODO(), u, "foo", "bar") if resp.Code != http.StatusOK { t.Errorf("Disable returned %v, want %v", resp.Code, http.StatusOK) @@ -523,7 +524,7 @@ func TestGithub_Disable_HooksButNotFound(t *testing.T) { client, _ := NewTest(s.URL, "https://foos.ball.com") // run test - err := client.Disable(u, "foo", "bar") + err := client.Disable(context.TODO(), u, "foo", "bar") if resp.Code != http.StatusOK { t.Errorf("Disable returned %v, want %v", resp.Code, http.StatusOK) @@ -565,7 +566,7 @@ func TestGithub_Disable_MultipleHooks(t *testing.T) { client, _ := NewTest(s.URL, "https://foo.bar.com") // run test - err := client.Disable(u, "foo", "bar") + err := client.Disable(context.TODO(), u, "foo", "bar") if count != wantCount { t.Errorf("Count returned %d, want %d", count, wantCount) @@ -622,7 +623,7 @@ func TestGithub_Enable(t *testing.T) { client, _ := NewTest(s.URL) // run test - got, _, err := client.Enable(u, r, new(library.Hook)) + got, _, err := client.Enable(context.TODO(), u, r, new(library.Hook)) if resp.Code != http.StatusOK { t.Errorf("Enable returned %v, want %v", resp.Code, http.StatusOK) @@ -1032,7 +1033,7 @@ func TestGithub_GetRepo(t *testing.T) { client, _ := NewTest(s.URL) // run test - got, err := client.GetRepo(u, r) + got, err := client.GetRepo(context.TODO(), u, r) if resp.Code != http.StatusOK { t.Errorf("GetRepo returned %v, want %v", resp.Code, http.StatusOK) @@ -1075,7 +1076,7 @@ func TestGithub_GetRepo_Fail(t *testing.T) { client, _ := NewTest(s.URL) // run test - _, err := client.GetRepo(u, r) + _, err := client.GetRepo(context.TODO(), u, r) if err == nil { t.Error("GetRepo should return error") diff --git a/scm/github/webhook.go b/scm/github/webhook.go index 43615e8cb..d773137e6 100644 --- a/scm/github/webhook.go +++ b/scm/github/webhook.go @@ -102,7 +102,7 @@ func (c *client) VerifyWebhook(request *http.Request, r *library.Repo) error { func (c *client) RedeliverWebhook(ctx context.Context, u *library.User, r *library.Repo, h *library.Hook) error { // create GitHub OAuth client with user's token //nolint:contextcheck // do not need to pass context in this instance - client := c.newClientToken(*u.Token) + client := c.newClientToken(context.TODO(), *u.Token) // capture the delivery ID of the hook using GitHub API deliveryID, err := c.getDeliveryID(ctx, client, r, h) diff --git a/scm/github/webhook_test.go b/scm/github/webhook_test.go index 1bef4c0ca..23c2e523c 100644 --- a/scm/github/webhook_test.go +++ b/scm/github/webhook_test.go @@ -1347,7 +1347,7 @@ func TestGithub_GetDeliveryID(t *testing.T) { client, _ := NewTest(s.URL, "https://foo.bar.com") - ghClient := client.newClientToken(*u.Token) + ghClient := client.newClientToken(context.TODO(), *u.Token) // run test got, err := client.getDeliveryID(ctx, ghClient, _repo, _hook) diff --git a/scm/service.go b/scm/service.go index c92cbedbf..32d34cb87 100644 --- a/scm/service.go +++ b/scm/service.go @@ -95,10 +95,10 @@ type Service interface { ConfigBackoff(*library.User, *library.Repo, string) ([]byte, error) // Disable defines a function that deactivates // a repo by destroying the webhook. - Disable(*library.User, string, string) error + Disable(context.Context, *library.User, string, string) error // Enable defines a function that activates // a repo by creating the webhook. - Enable(*library.User, *library.Repo, *library.Hook) (*library.Hook, string, error) + Enable(context.Context, *library.User, *library.Repo, *library.Hook) (*library.Hook, string, error) // Update defines a function that updates // a webhook for a specified repo. Update(*library.User, *library.Repo, int64) error @@ -116,7 +116,7 @@ type Service interface { GetPullRequest(*library.User, *library.Repo, int) (string, string, string, string, error) // GetRepo defines a function that retrieves // details for a repo. - GetRepo(*library.User, *library.Repo) (*library.Repo, error) + GetRepo(context.Context, *library.User, *library.Repo) (*library.Repo, error) // GetOrgAndRepoName defines a function that retrieves // the name of the org and repo in the SCM. GetOrgAndRepoName(*library.User, string, string) (string, string, error) From 453d5bc5ae3b552c632df33022657717bf65fbb1 Mon Sep 17 00:00:00 2001 From: Jordan Sussman Date: Wed, 28 Jun 2023 14:23:40 -0500 Subject: [PATCH 07/59] fix github creds --- scm/github/github.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scm/github/github.go b/scm/github/github.go index 0f358affd..0b84523ec 100644 --- a/scm/github/github.go +++ b/scm/github/github.go @@ -7,7 +7,6 @@ package github import ( "context" "fmt" - "net/http" "net/http/httptrace" "net/url" @@ -166,7 +165,7 @@ func (c *client) newClientToken(ctx context.Context, token string) *github.Clien // } tc.Transport = otelhttp.NewTransport( - http.DefaultTransport, + tc.Transport, otelhttp.WithClientTrace(func(ctx context.Context) *httptrace.ClientTrace { return otelhttptrace.NewClientTrace(ctx) }), From 52ce32383aa8f13344f83aeaff5b09925a448763 Mon Sep 17 00:00:00 2001 From: Jordan Sussman Date: Wed, 28 Jun 2023 14:42:32 -0500 Subject: [PATCH 08/59] add back vault --- docker-compose.yml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index c11bf0bf3..8f2020515 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -143,19 +143,19 @@ services: # This component is used for storing sensitive data like secrets. # # https://www.vaultproject.io/ - # vault: - # image: vault:latest - # container_name: vault - # command: server -dev - # networks: - # - vela - # environment: - # VAULT_DEV_LISTEN_ADDRESS: 0.0.0.0:8200 - # VAULT_DEV_ROOT_TOKEN_ID: vela - # ports: - # - '8200:8200' - # cap_add: - # - IPC_LOCK + vault: + image: hashicorp/vault:latest + container_name: vault + command: server -dev + networks: + - vela + environment: + VAULT_DEV_LISTEN_ADDRESS: 0.0.0.0:8200 + VAULT_DEV_ROOT_TOKEN_ID: vela + ports: + - '8200:8200' + cap_add: + - IPC_LOCK jaeger: image: jaegertracing/all-in-one:latest From 208410b15cecc7c37fc58c299db50c6e39324e11 Mon Sep 17 00:00:00 2001 From: Jordan Sussman Date: Wed, 28 Jun 2023 15:44:57 -0500 Subject: [PATCH 09/59] make the serviceName as vela-server --- cmd/vela-server/server.go | 2 +- cmd/vela-server/tracing.go | 14 +++++++++++--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/cmd/vela-server/server.go b/cmd/vela-server/server.go index 227f5263d..453492c74 100644 --- a/cmd/vela-server/server.go +++ b/cmd/vela-server/server.go @@ -122,7 +122,7 @@ func server(c *cli.Context) error { middleware.ScheduleFrequency(c.Duration("schedule-minimum-frequency")), // inject service middleware - otelgin.Middleware("vela-server-dv1", otelgin.WithTracerProvider(tp)), + otelgin.Middleware("vela-server", otelgin.WithTracerProvider(tp)), ) addr, err := url.Parse(c.String("server-addr")) diff --git a/cmd/vela-server/tracing.go b/cmd/vela-server/tracing.go index 0a92e2563..193da3032 100644 --- a/cmd/vela-server/tracing.go +++ b/cmd/vela-server/tracing.go @@ -2,6 +2,8 @@ package main import ( "context" + "go.opentelemetry.io/otel/sdk/resource" + semconv "go.opentelemetry.io/otel/semconv/v1.20.0" "go.opentelemetry.io/otel" otlp "go.opentelemetry.io/otel/exporters/otlp/otlptrace" @@ -11,9 +13,6 @@ import ( ) func tracerProvider() (*sdktrace.TracerProvider, error) { - // exporter, err := stdout.New(stdout.WithPrettyPrint()) - - // client := nil client := otlptracehttp.NewClient() exporter, err := otlp.New(context.TODO(), client) @@ -21,9 +20,18 @@ func tracerProvider() (*sdktrace.TracerProvider, error) { return nil, err } + // TODO: make the serviceName configurable + res, err := resource.New(context.TODO(), resource.WithAttributes( + semconv.ServiceName("vela-server"), + )) + if err != nil { + return nil, err + } + tp := sdktrace.NewTracerProvider( sdktrace.WithSampler(sdktrace.AlwaysSample()), sdktrace.WithBatcher(exporter), + sdktrace.WithResource(res), ) otel.SetTracerProvider(tp) From efd0487822dc0657e281945f3fd319824f3268ef Mon Sep 17 00:00:00 2001 From: Jordan Sussman Date: Thu, 29 Jun 2023 10:39:50 -0500 Subject: [PATCH 10/59] pass around the proper context --- api/user/get_source.go | 8 ++++++-- scm/github/github.go | 2 +- scm/github/repo.go | 9 ++++++--- scm/github/repo_test.go | 4 ++-- scm/service.go | 2 +- 5 files changed, 16 insertions(+), 9 deletions(-) diff --git a/api/user/get_source.go b/api/user/get_source.go index 2ea8734c3..9dbc14069 100644 --- a/api/user/get_source.go +++ b/api/user/get_source.go @@ -6,6 +6,7 @@ package user import ( "fmt" + "go.opentelemetry.io/otel/trace" "net/http" "github.com/gin-gonic/gin" @@ -41,12 +42,15 @@ import ( func GetSourceRepos(c *gin.Context) { // capture middleware values u := user.Retrieve(c) + ctx := c.Request.Context() // update engine logger with API metadata // // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields logrus.WithFields(logrus.Fields{ - "user": u.GetName(), + "user": u.GetName(), + "span_id": trace.SpanFromContext(ctx).SpanContext().SpanID(), + "trace_id": trace.SpanFromContext(ctx).SpanContext().TraceID(), }).Infof("reading available SCM repos for user %s", u.GetName()) // variables to capture requested data @@ -54,7 +58,7 @@ func GetSourceRepos(c *gin.Context) { output := make(map[string][]library.Repo) // send API call to capture the list of repos for the user - srcRepos, err := scm.FromContext(c).ListUserRepos(u) + srcRepos, err := scm.FromContext(c).ListUserRepos(ctx, u) if err != nil { retErr := fmt.Errorf("unable to get SCM repos for user %s: %w", u.GetName(), err) diff --git a/scm/github/github.go b/scm/github/github.go index 0b84523ec..0da5c7bb1 100644 --- a/scm/github/github.go +++ b/scm/github/github.go @@ -167,7 +167,7 @@ func (c *client) newClientToken(ctx context.Context, token string) *github.Clien tc.Transport = otelhttp.NewTransport( tc.Transport, otelhttp.WithClientTrace(func(ctx context.Context) *httptrace.ClientTrace { - return otelhttptrace.NewClientTrace(ctx) + return otelhttptrace.NewClientTrace(ctx, otelhttptrace.WithoutSubSpans()) }), ) diff --git a/scm/github/repo.go b/scm/github/repo.go index 7d5eb4767..269283854 100644 --- a/scm/github/repo.go +++ b/scm/github/repo.go @@ -7,6 +7,7 @@ package github import ( "context" "fmt" + "go.opentelemetry.io/otel/trace" "net/http" "strconv" "strings" @@ -411,13 +412,15 @@ func (c *client) GetOrgAndRepoName(u *library.User, o string, r string) (string, } // ListUserRepos returns a list of all repos the user has access to. -func (c *client) ListUserRepos(u *library.User) ([]*library.Repo, error) { +func (c *client) ListUserRepos(ctx context.Context, u *library.User) ([]*library.Repo, error) { c.Logger.WithFields(logrus.Fields{ - "user": u.GetName(), + "user": u.GetName(), + "span_id": trace.SpanFromContext(ctx).SpanContext().SpanID(), + "trace_id": trace.SpanFromContext(ctx).SpanContext().TraceID(), }).Tracef("listing source repositories for %s", u.GetName()) // create GitHub OAuth client with user's token - client := c.newClientToken(context.TODO(), u.GetToken()) + client := c.newClientToken(ctx, u.GetToken()) r := []*github.Repository{} f := []*library.Repo{} diff --git a/scm/github/repo_test.go b/scm/github/repo_test.go index 7c2e77a08..f00405ea1 100644 --- a/scm/github/repo_test.go +++ b/scm/github/repo_test.go @@ -1198,7 +1198,7 @@ func TestGithub_ListUserRepos(t *testing.T) { client, _ := NewTest(s.URL) // run test - got, err := client.ListUserRepos(u) + got, err := client.ListUserRepos(context.TODO(), u) if err != nil { t.Errorf("Status returned err: %v", err) @@ -1236,7 +1236,7 @@ func TestGithub_ListUserRepos_Ineligible(t *testing.T) { client, _ := NewTest(s.URL) // run test - got, err := client.ListUserRepos(u) + got, err := client.ListUserRepos(context.TODO(), u) if err != nil { t.Errorf("Status returned err: %v", err) diff --git a/scm/service.go b/scm/service.go index 32d34cb87..01314f313 100644 --- a/scm/service.go +++ b/scm/service.go @@ -107,7 +107,7 @@ type Service interface { Status(*library.User, *library.Build, string, string) error // ListUserRepos defines a function that retrieves // all repos with admin rights for the user. - ListUserRepos(*library.User) ([]*library.Repo, error) + ListUserRepos(context.Context, *library.User) ([]*library.Repo, error) // GetBranch defines a function that retrieves // a branch for a repo. GetBranch(*library.User, *library.Repo) (string, string, error) From 646f64b5e631b6728fc6c11854bbded2b0071580 Mon Sep 17 00:00:00 2001 From: davidvader Date: Thu, 29 Jun 2023 10:41:39 -0500 Subject: [PATCH 11/59] cli flag and opt-in enable tracing --- cmd/vela-server/main.go | 7 +++++ cmd/vela-server/server.go | 34 +++++++++++++++++-------- database/context.go | 6 ++--- database/database.go | 16 +++++++----- database/opts.go | 9 ++++--- database/schedule/create.go | 8 ------ docker-compose.yml | 1 + {cmd/vela-server => tracing}/tracing.go | 30 ++++++++++++++++++++-- 8 files changed, 76 insertions(+), 35 deletions(-) rename {cmd/vela-server => tracing}/tracing.go (63%) diff --git a/cmd/vela-server/main.go b/cmd/vela-server/main.go index 19463fe0b..7c6ef8652 100644 --- a/cmd/vela-server/main.go +++ b/cmd/vela-server/main.go @@ -222,6 +222,13 @@ func main() { Usage: "limit which repos can be utilize the schedule feature within the system", Value: &cli.StringSlice{}, }, + // tracing flags + &cli.BoolFlag{ + EnvVars: []string{"VELA_ENABLE_TRACING", "ENABLE_TRACING"}, + Name: "enable-tracing", + Value: true, + Usage: "enable otel tracing", + }, } // Add Database Flags app.Flags = append(app.Flags, database.Flags...) diff --git a/cmd/vela-server/server.go b/cmd/vela-server/server.go index 453492c74..cf38b7879 100644 --- a/cmd/vela-server/server.go +++ b/cmd/vela-server/server.go @@ -18,6 +18,7 @@ import ( "github.com/go-vela/server/database" "github.com/go-vela/server/router" "github.com/go-vela/server/router/middleware" + "github.com/go-vela/server/tracing" "github.com/sirupsen/logrus" "github.com/urfave/cli/v2" "go.opentelemetry.io/contrib/instrumentation/github.com/gin-gonic/gin/otelgin" @@ -63,22 +64,33 @@ func server(c *cli.Context) error { return err } - tp, err := tracerProvider() + // create the tracing config + tc, err := tracing.SetupTracing(c) if err != nil { - logrus.Fatal(err) + return err } - defer func() { - if err := tp.Shutdown(context.Background()); err != nil { - logrus.Printf("Error shutting down tracer provider: %v", err) - } - }() - - database, err := database.FromCLIContext(c, tp) + database, err := database.FromCLIContext(c, tc) if err != nil { return err } + // opt-in for telemetry + // todo: does this work for gorm by inheriting context + tracingMiddleware := func(ctx *gin.Context) {} + if tc.EnableTracing { + // initialize tracing middleware + tracingMiddleware = otelgin.Middleware("vela-server", otelgin.WithTracerProvider(tc.TracerProvider)) + + // shutdown tracing when server closes + defer func() { + err := tc.TracerProvider.Shutdown(context.Background()) + if err != nil { + logrus.Errorf("unable to shutdown tracer provider: %v", err) + } + }() + } + queue, err := setupQueue(c) if err != nil { return err @@ -121,8 +133,8 @@ func server(c *cli.Context) error { middleware.AllowlistSchedule(c.StringSlice("vela-schedule-allowlist")), middleware.ScheduleFrequency(c.Duration("schedule-minimum-frequency")), - // inject service middleware - otelgin.Middleware("vela-server", otelgin.WithTracerProvider(tp)), + // tracing middle, which is an empty handler when tracing is enabled + tracingMiddleware, ) addr, err := url.Parse(c.String("server-addr")) diff --git a/database/context.go b/database/context.go index 3809ba2d5..f2a945bc6 100644 --- a/database/context.go +++ b/database/context.go @@ -7,9 +7,9 @@ package database import ( "context" + "github.com/go-vela/server/tracing" "github.com/sirupsen/logrus" "github.com/urfave/cli/v2" - sdktrace "go.opentelemetry.io/otel/sdk/trace" ) const key = "database" @@ -43,7 +43,7 @@ func ToContext(c Setter, d Interface) { } // FromCLIContext creates and returns a database engine from the urfave/cli context. -func FromCLIContext(c *cli.Context, tp *sdktrace.TracerProvider) (Interface, error) { +func FromCLIContext(c *cli.Context, tc *tracing.TracingConfig) (Interface, error) { logrus.Debug("creating database engine from CLI configuration") return New( @@ -55,6 +55,6 @@ func FromCLIContext(c *cli.Context, tp *sdktrace.TracerProvider) (Interface, err WithDriver(c.String("database.driver")), WithEncryptionKey(c.String("database.encryption.key")), WithSkipCreation(c.Bool("database.skip_creation")), - WithTracerProvider(tp), + WithTracingConfig(tc), ) } diff --git a/database/database.go b/database/database.go index df558863a..68e6895ec 100644 --- a/database/database.go +++ b/database/database.go @@ -19,11 +19,11 @@ import ( "github.com/go-vela/server/database/step" "github.com/go-vela/server/database/user" "github.com/go-vela/server/database/worker" + "github.com/go-vela/server/tracing" "github.com/go-vela/types/constants" "github.com/sirupsen/logrus" "github.com/uptrace/opentelemetry-go-extra/otelgorm" - sdktrace "go.opentelemetry.io/otel/sdk/trace" "gorm.io/driver/postgres" "gorm.io/driver/sqlite" "gorm.io/gorm" @@ -48,8 +48,6 @@ type ( EncryptionKey string // specifies to skip creating tables and indexes for the database engine SkipCreation bool - - TracerProvider *sdktrace.TracerProvider } // engine represents the functionality that implements the Interface. @@ -60,6 +58,8 @@ type ( config *config // sirupsen/logrus logger used in database functions logger *logrus.Entry + // configurations related to telemetry/tracing + tracing *tracing.TracingConfig build.BuildInterface hook.HookInterface @@ -134,9 +134,12 @@ func New(opts ...EngineOpt) (Interface, error) { } logrus.Info("initializing gorm tracing") - otelPlugin := otelgorm.NewPlugin(otelgorm.WithTracerProvider(e.config.TracerProvider), otelgorm.WithoutQueryVariables()) - if err := e.client.Use(otelPlugin); err != nil { - return nil, err + + if e.tracing.EnableTracing { + otelPlugin := otelgorm.NewPlugin(otelgorm.WithTracerProvider(e.tracing.TracerProvider), otelgorm.WithoutQueryVariables()) + if err := e.client.Use(otelPlugin); err != nil { + return nil, err + } } // set the maximum amount of time a connection may be reused @@ -174,6 +177,5 @@ func NewTest() (Interface, error) { WithDriver("sqlite3"), WithEncryptionKey("A1B2C3D4E5G6H7I8J9K0LMNOPQRSTUVW"), WithSkipCreation(false), - WithTracerProvider(nil), ) } diff --git a/database/opts.go b/database/opts.go index 7d69aca24..698e769fb 100644 --- a/database/opts.go +++ b/database/opts.go @@ -7,7 +7,7 @@ package database import ( "time" - sdktrace "go.opentelemetry.io/otel/sdk/trace" + "github.com/go-vela/server/tracing" ) // EngineOpt represents a configuration option to initialize the database engine. @@ -93,10 +93,11 @@ func WithSkipCreation(skipCreation bool) EngineOpt { } } -func WithTracerProvider(tp *sdktrace.TracerProvider) EngineOpt { +// WithTracingConfig sets the shared tracing config in the database engine. +func WithTracingConfig(tracing *tracing.TracingConfig) EngineOpt { return func(e *engine) error { - // set to skip creating tables and indexes in the database engine - e.config.TracerProvider = tp + // set the tracing config + e.tracing = tracing return nil } diff --git a/database/schedule/create.go b/database/schedule/create.go index 333512b9e..c8fc234c1 100644 --- a/database/schedule/create.go +++ b/database/schedule/create.go @@ -12,19 +12,11 @@ import ( "github.com/go-vela/types/database" "github.com/go-vela/types/library" "github.com/sirupsen/logrus" - "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" ) // CreateSchedule creates a new schedule in the database. func (e *engine) CreateSchedule(ctx context.Context, s *library.Schedule) error { - span := trace.SpanFromContext(ctx) - if span.IsRecording() { - span.SetAttributes( - attribute.String("db", "do-operation"), - ) - } - e.logger.WithFields(logrus.Fields{ "schedule": s.GetName(), "span_id": trace.SpanFromContext(ctx).SpanContext().SpanID(), diff --git a/docker-compose.yml b/docker-compose.yml index 8f2020515..e85f40cb6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -22,6 +22,7 @@ services: networks: - vela environment: + VELA_ENABLE_TRACING: true OTEL_EXPORTER_OTLP_ENDPOINT: http://jaeger:4318 DATABASE_DRIVER: postgres DATABASE_ADDR: 'postgres://vela:zB7mrKDTZqNeNTD8z47yG4DHywspAh@postgres:5432/vela?sslmode=disable' diff --git a/cmd/vela-server/tracing.go b/tracing/tracing.go similarity index 63% rename from cmd/vela-server/tracing.go rename to tracing/tracing.go index 193da3032..d98f9c6a8 100644 --- a/cmd/vela-server/tracing.go +++ b/tracing/tracing.go @@ -1,7 +1,9 @@ -package main +package tracing import ( "context" + + "github.com/urfave/cli/v2" "go.opentelemetry.io/otel/sdk/resource" semconv "go.opentelemetry.io/otel/semconv/v1.20.0" @@ -12,7 +14,31 @@ import ( sdktrace "go.opentelemetry.io/otel/sdk/trace" ) -func tracerProvider() (*sdktrace.TracerProvider, error) { +type TracingConfig struct { + EnableTracing bool + TracerProvider *sdktrace.TracerProvider +} + +func DefaultTracingConfig() *TracingConfig { + return &TracingConfig{ + EnableTracing: false, + } +} + +func SetupTracing(c *cli.Context) (*TracingConfig, error) { + // initialize the shared tracing provider + tp, err := setupTracerProvider() + if err != nil { + return nil, err + } + + return &TracingConfig{ + EnableTracing: c.Bool("enable-tracing"), + TracerProvider: tp, + }, nil +} + +func setupTracerProvider() (*sdktrace.TracerProvider, error) { client := otlptracehttp.NewClient() exporter, err := otlp.New(context.TODO(), client) From 6f86666faea484e0a220362fd175fb316542d27a Mon Sep 17 00:00:00 2001 From: davidvader Date: Thu, 29 Jun 2023 12:58:05 -0500 Subject: [PATCH 12/59] refactor: move code into tracing package --- cmd/vela-server/main.go | 11 ++++------- cmd/vela-server/server.go | 18 ++++++++--------- database/database.go | 12 ++++++++---- tracing/config.go | 32 +++++++++++++++++++++++++++++++ tracing/flags.go | 23 ++++++++++++++++++++++ tracing/{tracing.go => tracer.go} | 31 ++++-------------------------- 6 files changed, 80 insertions(+), 47 deletions(-) create mode 100644 tracing/config.go create mode 100644 tracing/flags.go rename tracing/{tracing.go => tracer.go} (60%) diff --git a/cmd/vela-server/main.go b/cmd/vela-server/main.go index 7c6ef8652..66cb66880 100644 --- a/cmd/vela-server/main.go +++ b/cmd/vela-server/main.go @@ -16,6 +16,7 @@ import ( "github.com/go-vela/server/queue" "github.com/go-vela/server/scm" "github.com/go-vela/server/secret" + "github.com/go-vela/server/tracing" "github.com/go-vela/server/version" "github.com/sirupsen/logrus" "github.com/urfave/cli/v2" @@ -222,13 +223,6 @@ func main() { Usage: "limit which repos can be utilize the schedule feature within the system", Value: &cli.StringSlice{}, }, - // tracing flags - &cli.BoolFlag{ - EnvVars: []string{"VELA_ENABLE_TRACING", "ENABLE_TRACING"}, - Name: "enable-tracing", - Value: true, - Usage: "enable otel tracing", - }, } // Add Database Flags app.Flags = append(app.Flags, database.Flags...) @@ -242,6 +236,9 @@ func main() { // Add Source Flags app.Flags = append(app.Flags, scm.Flags...) + // Add Tracing Flags + app.Flags = append(app.Flags, tracing.Flags...) + // set logrus to log in JSON format logrus.SetFormatter(&logrus.JSONFormatter{}) diff --git a/cmd/vela-server/server.go b/cmd/vela-server/server.go index cf38b7879..39f031205 100644 --- a/cmd/vela-server/server.go +++ b/cmd/vela-server/server.go @@ -65,22 +65,17 @@ func server(c *cli.Context) error { } // create the tracing config - tc, err := tracing.SetupTracing(c) - if err != nil { - return err - } - - database, err := database.FromCLIContext(c, tc) + tc, err := tracing.New(c) if err != nil { return err } // opt-in for telemetry - // todo: does this work for gorm by inheriting context tracingMiddleware := func(ctx *gin.Context) {} if tc.EnableTracing { // initialize tracing middleware - tracingMiddleware = otelgin.Middleware("vela-server", otelgin.WithTracerProvider(tc.TracerProvider)) + tracingMiddleware = otelgin.Middleware(c.String("tracing.service.name"), + otelgin.WithTracerProvider(tc.TracerProvider)) // shutdown tracing when server closes defer func() { @@ -91,6 +86,11 @@ func server(c *cli.Context) error { }() } + database, err := database.FromCLIContext(c, tc) + if err != nil { + return err + } + queue, err := setupQueue(c) if err != nil { return err @@ -133,7 +133,7 @@ func server(c *cli.Context) error { middleware.AllowlistSchedule(c.StringSlice("vela-schedule-allowlist")), middleware.ScheduleFrequency(c.Duration("schedule-minimum-frequency")), - // tracing middle, which is an empty handler when tracing is enabled + // tracing middle, which is an empty handler when tracing is disabled tracingMiddleware, ) diff --git a/database/database.go b/database/database.go index 68e6895ec..061117596 100644 --- a/database/database.go +++ b/database/database.go @@ -133,11 +133,15 @@ func New(opts ...EngineOpt) (Interface, error) { return nil, err } - logrus.Info("initializing gorm tracing") - + // initialize otel tracing if enabled if e.tracing.EnableTracing { - otelPlugin := otelgorm.NewPlugin(otelgorm.WithTracerProvider(e.tracing.TracerProvider), otelgorm.WithoutQueryVariables()) - if err := e.client.Use(otelPlugin); err != nil { + otelPlugin := otelgorm.NewPlugin( + otelgorm.WithTracerProvider(e.tracing.TracerProvider), + otelgorm.WithoutQueryVariables(), + ) + + err := e.client.Use(otelPlugin) + if err != nil { return nil, err } } diff --git a/tracing/config.go b/tracing/config.go new file mode 100644 index 000000000..c37a98f5d --- /dev/null +++ b/tracing/config.go @@ -0,0 +1,32 @@ +package tracing + +import ( + "github.com/urfave/cli/v2" + + sdktrace "go.opentelemetry.io/otel/sdk/trace" +) + +// TracingConfig represents the configurations for otel tracing +type TracingConfig struct { + EnableTracing bool + ServiceName string + TracerProvider *sdktrace.TracerProvider +} + +// New takes cli context and returns a tracing config to supply to traceable services +func New(c *cli.Context) (*TracingConfig, error) { + enable := c.Bool("tracing.enable") + serviceName := c.String("tracing.service.name") + + // could skip creating the tracer if tracing is disabled + tracerProvider, err := initTracer(c) + if err != nil { + return nil, err + } + + return &TracingConfig{ + EnableTracing: enable, + ServiceName: serviceName, + TracerProvider: tracerProvider, + }, nil +} diff --git a/tracing/flags.go b/tracing/flags.go new file mode 100644 index 000000000..7ca5d539e --- /dev/null +++ b/tracing/flags.go @@ -0,0 +1,23 @@ +package tracing + +import ( + "github.com/urfave/cli/v2" +) + +var Flags = []cli.Flag{ + // Tracing Flags + + &cli.BoolFlag{ + EnvVars: []string{"VELA_ENABLE_TRACING", "TRACING_ENABLE"}, + Name: "tracing.enable", + Usage: "enable otel tracing", + Value: false, + }, + + &cli.StringFlag{ + EnvVars: []string{"VELA_TRACING_SERVICE_NAME", "TRACING_SERVICE_NAME"}, + Name: "tracing.service.name", + Usage: "set otel tracing service name", + Value: "vela-server", + }, +} diff --git a/tracing/tracing.go b/tracing/tracer.go similarity index 60% rename from tracing/tracing.go rename to tracing/tracer.go index d98f9c6a8..c8874c7b8 100644 --- a/tracing/tracing.go +++ b/tracing/tracer.go @@ -14,41 +14,18 @@ import ( sdktrace "go.opentelemetry.io/otel/sdk/trace" ) -type TracingConfig struct { - EnableTracing bool - TracerProvider *sdktrace.TracerProvider -} - -func DefaultTracingConfig() *TracingConfig { - return &TracingConfig{ - EnableTracing: false, - } -} - -func SetupTracing(c *cli.Context) (*TracingConfig, error) { - // initialize the shared tracing provider - tp, err := setupTracerProvider() - if err != nil { - return nil, err - } - - return &TracingConfig{ - EnableTracing: c.Bool("enable-tracing"), - TracerProvider: tp, - }, nil -} - -func setupTracerProvider() (*sdktrace.TracerProvider, error) { +// initTracer returns the tracer provider supplied to the tracing config +func initTracer(c *cli.Context) (*sdktrace.TracerProvider, error) { client := otlptracehttp.NewClient() + // TODO: inject actual context exporter, err := otlp.New(context.TODO(), client) if err != nil { return nil, err } - // TODO: make the serviceName configurable res, err := resource.New(context.TODO(), resource.WithAttributes( - semconv.ServiceName("vela-server"), + semconv.ServiceName(c.String("tracing.service.name")), )) if err != nil { return nil, err From 2b7db12497bb60317b2b82a4a73cd2323dc7c414 Mon Sep 17 00:00:00 2001 From: davidvader Date: Thu, 29 Jun 2023 13:14:25 -0500 Subject: [PATCH 13/59] tweak: order of vars and flags --- docker-compose.yml | 5 +++-- tracing/flags.go | 1 - 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index e85f40cb6..db8b99059 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -22,8 +22,6 @@ services: networks: - vela environment: - VELA_ENABLE_TRACING: true - OTEL_EXPORTER_OTLP_ENDPOINT: http://jaeger:4318 DATABASE_DRIVER: postgres DATABASE_ADDR: 'postgres://vela:zB7mrKDTZqNeNTD8z47yG4DHywspAh@postgres:5432/vela?sslmode=disable' DATABASE_COMPRESSION_LEVEL: 3 @@ -46,6 +44,9 @@ services: VELA_ENABLE_SECURE_COOKIE: 'false' VELA_REPO_ALLOWLIST: '*' VELA_SCHEDULE_ALLOWLIST: '*' + VELA_ENABLE_TRACING: true + VELA_TRACING_SERVICE_NAME: 'vela-server' + OTEL_EXPORTER_OTLP_ENDPOINT: http://jaeger:4318 env_file: - .env restart: always diff --git a/tracing/flags.go b/tracing/flags.go index 7ca5d539e..ea6f7cc7f 100644 --- a/tracing/flags.go +++ b/tracing/flags.go @@ -13,7 +13,6 @@ var Flags = []cli.Flag{ Usage: "enable otel tracing", Value: false, }, - &cli.StringFlag{ EnvVars: []string{"VELA_TRACING_SERVICE_NAME", "TRACING_SERVICE_NAME"}, Name: "tracing.service.name", From 570d14859397e9fbbde141ca9260c3ebebfac6a5 Mon Sep 17 00:00:00 2001 From: Jordan Sussman Date: Thu, 29 Jun 2023 13:48:11 -0500 Subject: [PATCH 14/59] fix depends on --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index db8b99059..b4432e27b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -55,7 +55,7 @@ services: depends_on: - postgres - redis - # - vault + - vault # The `worker` compose service hosts the Vela build daemon. # From 120a5e74ad36a31044fc49a6a5bc480af14ffbd5 Mon Sep 17 00:00:00 2001 From: Jordan Sussman Date: Thu, 29 Jun 2023 14:02:07 -0500 Subject: [PATCH 15/59] make linter happy --- database/context.go | 2 +- database/database.go | 2 +- database/opts.go | 2 +- scm/github/repo.go | 3 ++- tracing/config.go | 14 +++++++++----- tracing/flags.go | 4 ++++ tracing/tracer.go | 7 ++++++- 7 files changed, 24 insertions(+), 10 deletions(-) diff --git a/database/context.go b/database/context.go index f2a945bc6..7249729a9 100644 --- a/database/context.go +++ b/database/context.go @@ -43,7 +43,7 @@ func ToContext(c Setter, d Interface) { } // FromCLIContext creates and returns a database engine from the urfave/cli context. -func FromCLIContext(c *cli.Context, tc *tracing.TracingConfig) (Interface, error) { +func FromCLIContext(c *cli.Context, tc *tracing.Config) (Interface, error) { logrus.Debug("creating database engine from CLI configuration") return New( diff --git a/database/database.go b/database/database.go index 061117596..9ffe82b8e 100644 --- a/database/database.go +++ b/database/database.go @@ -59,7 +59,7 @@ type ( // sirupsen/logrus logger used in database functions logger *logrus.Entry // configurations related to telemetry/tracing - tracing *tracing.TracingConfig + tracing *tracing.Config build.BuildInterface hook.HookInterface diff --git a/database/opts.go b/database/opts.go index 698e769fb..a99bf5f7b 100644 --- a/database/opts.go +++ b/database/opts.go @@ -94,7 +94,7 @@ func WithSkipCreation(skipCreation bool) EngineOpt { } // WithTracingConfig sets the shared tracing config in the database engine. -func WithTracingConfig(tracing *tracing.TracingConfig) EngineOpt { +func WithTracingConfig(tracing *tracing.Config) EngineOpt { return func(e *engine) error { // set the tracing config e.tracing = tracing diff --git a/scm/github/repo.go b/scm/github/repo.go index 269283854..7052cf953 100644 --- a/scm/github/repo.go +++ b/scm/github/repo.go @@ -7,12 +7,13 @@ package github import ( "context" "fmt" - "go.opentelemetry.io/otel/trace" "net/http" "strconv" "strings" "time" + "go.opentelemetry.io/otel/trace" + "github.com/sirupsen/logrus" "github.com/go-vela/types/constants" diff --git a/tracing/config.go b/tracing/config.go index c37a98f5d..0b45961e3 100644 --- a/tracing/config.go +++ b/tracing/config.go @@ -1,3 +1,7 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + package tracing import ( @@ -6,15 +10,15 @@ import ( sdktrace "go.opentelemetry.io/otel/sdk/trace" ) -// TracingConfig represents the configurations for otel tracing -type TracingConfig struct { +// Config represents the configurations for otel tracing. +type Config struct { EnableTracing bool ServiceName string TracerProvider *sdktrace.TracerProvider } -// New takes cli context and returns a tracing config to supply to traceable services -func New(c *cli.Context) (*TracingConfig, error) { +// New takes cli context and returns a tracing config to supply to traceable services. +func New(c *cli.Context) (*Config, error) { enable := c.Bool("tracing.enable") serviceName := c.String("tracing.service.name") @@ -24,7 +28,7 @@ func New(c *cli.Context) (*TracingConfig, error) { return nil, err } - return &TracingConfig{ + return &Config{ EnableTracing: enable, ServiceName: serviceName, TracerProvider: tracerProvider, diff --git a/tracing/flags.go b/tracing/flags.go index ea6f7cc7f..e9d82fe1e 100644 --- a/tracing/flags.go +++ b/tracing/flags.go @@ -1,3 +1,7 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + package tracing import ( diff --git a/tracing/tracer.go b/tracing/tracer.go index c8874c7b8..de89e2359 100644 --- a/tracing/tracer.go +++ b/tracing/tracer.go @@ -1,3 +1,7 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + package tracing import ( @@ -14,7 +18,7 @@ import ( sdktrace "go.opentelemetry.io/otel/sdk/trace" ) -// initTracer returns the tracer provider supplied to the tracing config +// initTracer returns the tracer provider supplied to the tracing config. func initTracer(c *cli.Context) (*sdktrace.TracerProvider, error) { client := otlptracehttp.NewClient() @@ -39,5 +43,6 @@ func initTracer(c *cli.Context) (*sdktrace.TracerProvider, error) { otel.SetTracerProvider(tp) otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(propagation.TraceContext{}, propagation.Baggage{})) + return tp, nil } From 541025d7f05e39b91ce17de8e976f83248577709 Mon Sep 17 00:00:00 2001 From: Jordan Sussman Date: Thu, 29 Jun 2023 14:30:47 -0500 Subject: [PATCH 16/59] fix tests --- database/context_test.go | 3 ++- database/database.go | 1 + database/database_test.go | 2 ++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/database/context_test.go b/database/context_test.go index 8513ca4ab..e0827e4ec 100644 --- a/database/context_test.go +++ b/database/context_test.go @@ -6,6 +6,7 @@ package database import ( "flag" + "github.com/go-vela/server/tracing" "reflect" "testing" "time" @@ -134,7 +135,7 @@ func TestDatabase_FromCLIContext(t *testing.T) { // run tests for _, test := range tests { t.Run(test.name, func(t *testing.T) { - _, err := FromCLIContext(test.context, nil) + _, err := FromCLIContext(test.context, &tracing.Config{EnableTracing: false}) if test.failure { if err == nil { diff --git a/database/database.go b/database/database.go index 9ffe82b8e..805a1ddce 100644 --- a/database/database.go +++ b/database/database.go @@ -181,5 +181,6 @@ func NewTest() (Interface, error) { WithDriver("sqlite3"), WithEncryptionKey("A1B2C3D4E5G6H7I8J9K0LMNOPQRSTUVW"), WithSkipCreation(false), + WithTracingConfig(&tracing.Config{EnableTracing: false}), ) } diff --git a/database/database_test.go b/database/database_test.go index a1cfecc8a..91ba70f47 100644 --- a/database/database_test.go +++ b/database/database_test.go @@ -5,6 +5,7 @@ package database import ( + "github.com/go-vela/server/tracing" "testing" "time" @@ -93,6 +94,7 @@ func TestDatabase_New(t *testing.T) { WithDriver(test.config.Driver), WithEncryptionKey(test.config.EncryptionKey), WithSkipCreation(test.config.SkipCreation), + WithTracingConfig(&tracing.Config{EnableTracing: false}), ) if test.failure { From b6955993ef2459887daeca17ce8022a668c1fc2f Mon Sep 17 00:00:00 2001 From: davidvader Date: Mon, 18 Sep 2023 16:14:01 -0500 Subject: [PATCH 17/59] enhance: add context to scm --- api/auth/get_token.go | 4 +- api/auth/post_token.go | 2 +- api/build/create.go | 10 ++-- api/build/get_id.go | 2 +- api/build/list_org.go | 2 +- api/build/restart.go | 10 ++-- api/build/update.go | 2 +- api/deployment/create.go | 3 +- api/deployment/get.go | 3 +- api/deployment/list.go | 4 +- api/pipeline/template.go | 2 +- api/repo/create.go | 4 +- api/repo/delete.go | 2 +- api/repo/list_org.go | 2 +- api/repo/repair.go | 4 +- api/repo/update.go | 2 +- api/scm/sync.go | 6 +-- api/scm/sync_org.go | 6 +-- api/secret/create.go | 4 +- api/secret/list.go | 2 +- api/user/get_source.go | 2 +- api/webhook/post.go | 18 ++++---- cmd/vela-server/schedule.go | 6 +-- router/middleware/perm/perm.go | 21 +++++---- router/middleware/pipeline/pipeline.go | 2 +- scm/github/access.go | 9 ++-- scm/github/access_test.go | 23 +++++----- scm/github/authentication.go | 12 ++--- scm/github/authentication_test.go | 26 ++++++----- scm/github/changeset.go | 5 +- scm/github/changeset_test.go | 5 +- scm/github/deployment.go | 9 ++-- scm/github/deployment_test.go | 9 ++-- scm/github/github.go | 2 - scm/github/org.go | 3 +- scm/github/org_test.go | 7 +-- scm/github/repo.go | 27 +++++------ scm/github/repo_test.go | 63 +++++++++++++------------- scm/github/webhook.go | 4 +- scm/github/webhook_test.go | 42 ++++++++--------- scm/service.go | 58 ++++++++++++------------ 41 files changed, 221 insertions(+), 208 deletions(-) diff --git a/api/auth/get_token.go b/api/auth/get_token.go index 81f632f4d..37ee028e7 100644 --- a/api/auth/get_token.go +++ b/api/auth/get_token.go @@ -72,7 +72,7 @@ func GetAuthToken(c *gin.Context) { code := c.Request.FormValue("code") if len(code) == 0 { // start the initial OAuth workflow - oAuthState, err = scm.FromContext(c).Login(c.Writer, c.Request) + oAuthState, err = scm.FromContext(c).Login(ctx, c.Writer, c.Request) if err != nil { retErr := fmt.Errorf("unable to login user: %w", err) @@ -83,7 +83,7 @@ func GetAuthToken(c *gin.Context) { } // complete the OAuth workflow and authenticates the user - newUser, err := scm.FromContext(c).Authenticate(c.Writer, c.Request, oAuthState) + newUser, err := scm.FromContext(c).Authenticate(ctx, c.Writer, c.Request, oAuthState) if err != nil { retErr := fmt.Errorf("unable to authenticate user: %w", err) diff --git a/api/auth/post_token.go b/api/auth/post_token.go index 55fe47fdf..4db7dd019 100644 --- a/api/auth/post_token.go +++ b/api/auth/post_token.go @@ -53,7 +53,7 @@ func PostAuthToken(c *gin.Context) { ctx := c.Request.Context() // attempt to get user from source - u, err := scm.FromContext(c).AuthenticateToken(c.Request) + u, err := scm.FromContext(c).AuthenticateToken(ctx, c.Request) if err != nil { retErr := fmt.Errorf("unable to authenticate user: %w", err) diff --git a/api/build/create.go b/api/build/create.go index 5398c0d91..c2885f096 100644 --- a/api/build/create.go +++ b/api/build/create.go @@ -187,7 +187,7 @@ func CreateBuild(c *gin.Context) { if !strings.EqualFold(input.GetEvent(), constants.EventComment) && !strings.EqualFold(input.GetEvent(), constants.EventPull) { // send API call to capture list of files changed for the commit - files, err = scm.FromContext(c).Changeset(u, r, input.GetCommit()) + files, err = scm.FromContext(c).Changeset(ctx, u, r, input.GetCommit()) if err != nil { retErr := fmt.Errorf("unable to create new build: failed to get changeset for %s: %w", r.GetFullName(), err) @@ -210,7 +210,7 @@ func CreateBuild(c *gin.Context) { } // send API call to capture list of files changed for the pull request - files, err = scm.FromContext(c).ChangesetPR(u, r, number) + files, err = scm.FromContext(c).ChangesetPR(ctx, u, r, number) if err != nil { retErr := fmt.Errorf("unable to create new build: failed to get changeset for %s: %w", r.GetFullName(), err) @@ -235,7 +235,7 @@ func CreateBuild(c *gin.Context) { pipeline, err = database.FromContext(c).GetPipelineForRepo(ctx, input.GetCommit(), r) if err != nil { // assume the pipeline doesn't exist in the database yet // send API call to capture the pipeline configuration file - config, err = scm.FromContext(c).ConfigBackoff(u, r, input.GetCommit()) + config, err = scm.FromContext(c).ConfigBackoff(ctx, u, r, input.GetCommit()) if err != nil { retErr := fmt.Errorf("unable to create new build: failed to get pipeline configuration for %s: %w", r.GetFullName(), err) @@ -289,7 +289,7 @@ func CreateBuild(c *gin.Context) { input.SetStatus(constants.StatusSuccess) // send API call to set the status on the commit - err = scm.FromContext(c).Status(u, input, r.GetOrg(), r.GetName()) + err = scm.FromContext(c).Status(ctx, u, input, r.GetOrg(), r.GetName()) if err != nil { logger.Errorf("unable to set commit status for %s/%d: %v", r.GetFullName(), input.GetNumber(), err) } @@ -345,7 +345,7 @@ func CreateBuild(c *gin.Context) { c.JSON(http.StatusCreated, input) // send API call to set the status on the commit - err = scm.FromContext(c).Status(u, input, r.GetOrg(), r.GetName()) + err = scm.FromContext(c).Status(ctx, u, input, r.GetOrg(), r.GetName()) if err != nil { logger.Errorf("unable to set commit status for build %s/%d: %v", r.GetFullName(), input.GetNumber(), err) } diff --git a/api/build/get_id.go b/api/build/get_id.go index 439cd9b81..7a122c8f5 100644 --- a/api/build/get_id.go +++ b/api/build/get_id.go @@ -101,7 +101,7 @@ func GetBuildByID(c *gin.Context) { // Capture user access from SCM. We do this in order to ensure user has access and is not // just retrieving any build using a random id number. - perm, err := scm.FromContext(c).RepoAccess(u, u.GetToken(), r.GetOrg(), r.GetName()) + perm, err := scm.FromContext(c).RepoAccess(ctx, u, u.GetToken(), r.GetOrg(), r.GetName()) if err != nil { logrus.Errorf("unable to get user %s access level for repo %s", u.GetName(), r.GetFullName()) } diff --git a/api/build/list_org.go b/api/build/list_org.go index 3ab3bf9d3..214b7294b 100644 --- a/api/build/list_org.go +++ b/api/build/list_org.go @@ -190,7 +190,7 @@ func ListBuildsForOrg(c *gin.Context) { perPage = util.MaxInt(1, util.MinInt(100, perPage)) // See if the user is an org admin to bypass individual permission checks - perm, err := scm.FromContext(c).OrgAccess(u, o) + perm, err := scm.FromContext(c).OrgAccess(ctx, u, o) if err != nil { logrus.Errorf("unable to get user %s access level for org %s", u.GetName(), o) } diff --git a/api/build/restart.go b/api/build/restart.go index 8fa21c0ca..6ee1e7fe9 100644 --- a/api/build/restart.go +++ b/api/build/restart.go @@ -177,7 +177,7 @@ func RestartBuild(c *gin.Context) { if !strings.EqualFold(b.GetEvent(), constants.EventComment) && !strings.EqualFold(b.GetEvent(), constants.EventPull) { // send API call to capture list of files changed for the commit - files, err = scm.FromContext(c).Changeset(u, r, b.GetCommit()) + files, err = scm.FromContext(c).Changeset(ctx, u, r, b.GetCommit()) if err != nil { retErr := fmt.Errorf("unable to restart build: failed to get changeset for %s: %w", r.GetFullName(), err) @@ -200,7 +200,7 @@ func RestartBuild(c *gin.Context) { } // send API call to capture list of files changed for the pull request - files, err = scm.FromContext(c).ChangesetPR(u, r, number) + files, err = scm.FromContext(c).ChangesetPR(ctx, u, r, number) if err != nil { retErr := fmt.Errorf("unable to restart build: failed to get changeset for %s: %w", r.GetFullName(), err) @@ -226,7 +226,7 @@ func RestartBuild(c *gin.Context) { pipeline, err = database.FromContext(c).GetPipelineForRepo(ctx, b.GetCommit(), r) if err != nil { // assume the pipeline doesn't exist in the database yet (before pipeline support was added) // send API call to capture the pipeline configuration file - config, err = scm.FromContext(c).ConfigBackoff(u, r, b.GetCommit()) + config, err = scm.FromContext(c).ConfigBackoff(ctx, u, r, b.GetCommit()) if err != nil { retErr := fmt.Errorf("unable to get pipeline configuration for %s: %w", r.GetFullName(), err) @@ -281,7 +281,7 @@ func RestartBuild(c *gin.Context) { b.SetStatus(constants.StatusSkipped) // send API call to set the status on the commit - err = scm.FromContext(c).Status(u, b, r.GetOrg(), r.GetName()) + err = scm.FromContext(c).Status(ctx, u, b, r.GetOrg(), r.GetName()) if err != nil { logrus.Errorf("unable to set commit status for %s/%d: %v", r.GetFullName(), b.GetNumber(), err) } @@ -336,7 +336,7 @@ func RestartBuild(c *gin.Context) { c.JSON(http.StatusCreated, b) // send API call to set the status on the commit - err = scm.FromContext(c).Status(u, b, r.GetOrg(), r.GetName()) + err = scm.FromContext(c).Status(ctx, u, b, r.GetOrg(), r.GetName()) if err != nil { logger.Errorf("unable to set commit status for build %s: %v", entry, err) } diff --git a/api/build/update.go b/api/build/update.go index 4a6ab2004..9f4aabbc0 100644 --- a/api/build/update.go +++ b/api/build/update.go @@ -176,7 +176,7 @@ func UpdateBuild(c *gin.Context) { } // send API call to set the status on the commit - err = scm.FromContext(c).Status(u, b, r.GetOrg(), r.GetName()) + err = scm.FromContext(c).Status(ctx, u, b, r.GetOrg(), r.GetName()) if err != nil { logrus.Errorf("unable to set commit status for build %s: %v", entry, err) } diff --git a/api/deployment/create.go b/api/deployment/create.go index 2075e65da..5e388d8d0 100644 --- a/api/deployment/create.go +++ b/api/deployment/create.go @@ -59,6 +59,7 @@ func CreateDeployment(c *gin.Context) { o := org.Retrieve(c) r := repo.Retrieve(c) u := user.Retrieve(c) + ctx := c.Request.Context() // update engine logger with API metadata // @@ -99,7 +100,7 @@ func CreateDeployment(c *gin.Context) { } // send API call to create the deployment - err = scm.FromContext(c).CreateDeployment(u, r, input) + err = scm.FromContext(c).CreateDeployment(ctx, u, r, input) if err != nil { retErr := fmt.Errorf("unable to create new deployment for %s: %w", r.GetFullName(), err) diff --git a/api/deployment/get.go b/api/deployment/get.go index ff6827d3a..cb1ecfceb 100644 --- a/api/deployment/get.go +++ b/api/deployment/get.go @@ -65,6 +65,7 @@ func GetDeployment(c *gin.Context) { r := repo.Retrieve(c) u := user.Retrieve(c) deployment := util.PathParameter(c, "deployment") + ctx := c.Request.Context() entry := fmt.Sprintf("%s/%s", r.GetFullName(), deployment) @@ -87,7 +88,7 @@ func GetDeployment(c *gin.Context) { } // send API call to capture the deployment - d, err := scm.FromContext(c).GetDeployment(u, r, int64(number)) + d, err := scm.FromContext(c).GetDeployment(ctx, u, r, int64(number)) if err != nil { retErr := fmt.Errorf("unable to get deployment %s: %w", entry, err) diff --git a/api/deployment/list.go b/api/deployment/list.go index cb2ec2010..08ed1853d 100644 --- a/api/deployment/list.go +++ b/api/deployment/list.go @@ -117,7 +117,7 @@ func ListDeployments(c *gin.Context) { perPage = util.MaxInt(1, util.MinInt(100, perPage)) // send API call to capture the total number of deployments for the repo - t, err := scm.FromContext(c).GetDeploymentCount(u, r) + t, err := scm.FromContext(c).GetDeploymentCount(ctx, u, r) if err != nil { retErr := fmt.Errorf("unable to get deployment count for %s: %w", r.GetFullName(), err) @@ -127,7 +127,7 @@ func ListDeployments(c *gin.Context) { } // send API call to capture the list of deployments for the repo - d, err := scm.FromContext(c).GetDeploymentList(u, r, page, perPage) + d, err := scm.FromContext(c).GetDeploymentList(ctx, u, r, page, perPage) if err != nil { retErr := fmt.Errorf("unable to get deployments for %s: %w", r.GetFullName(), err) diff --git a/api/pipeline/template.go b/api/pipeline/template.go index 7b00abb5d..596ff77f3 100644 --- a/api/pipeline/template.go +++ b/api/pipeline/template.go @@ -146,7 +146,7 @@ func GetTemplates(c *gin.Context) { } // retrieve link to template file from github - link, err := scm.FromContext(c).GetHTMLURL(user, src.Org, src.Repo, src.Name, src.Ref) + link, err := scm.FromContext(c).GetHTMLURL(ctx, user, src.Org, src.Repo, src.Name, src.Ref) if err != nil { util.HandleError(c, http.StatusBadRequest, fmt.Errorf("%s: unable to get html url for %s/%s/%s/@%s: %w", baseErr, src.Org, src.Repo, src.Name, src.Ref, err)) diff --git a/api/repo/create.go b/api/repo/create.go index 50d2383a0..efa1611df 100644 --- a/api/repo/create.go +++ b/api/repo/create.go @@ -99,7 +99,7 @@ func CreateRepo(c *gin.Context) { }).Infof("creating new repo %s", input.GetFullName()) // get repo information from the source - r, err := scm.FromContext(c).GetRepo(u, input) + r, err := scm.FromContext(c).GetRepo(ctx, u, input) if err != nil { retErr := fmt.Errorf("unable to retrieve repo info for %s from source: %w", r.GetFullName(), err) @@ -255,7 +255,7 @@ func CreateRepo(c *gin.Context) { // check if we should create the webhook if c.Value("webhookvalidation").(bool) { // send API call to create the webhook - h, _, err = scm.FromContext(c).Enable(u, r, h) + h, _, err = scm.FromContext(c).Enable(ctx, u, r, h) if err != nil { retErr := fmt.Errorf("unable to create webhook for %s: %w", r.GetFullName(), err) diff --git a/api/repo/delete.go b/api/repo/delete.go index 4ddaf11f7..cf6d64fbf 100644 --- a/api/repo/delete.go +++ b/api/repo/delete.go @@ -71,7 +71,7 @@ func DeleteRepo(c *gin.Context) { }).Infof("deleting repo %s", r.GetFullName()) // send API call to remove the webhook - err := scm.FromContext(c).Disable(u, r.GetOrg(), r.GetName()) + err := scm.FromContext(c).Disable(ctx, u, r.GetOrg(), r.GetName()) if err != nil { retErr := fmt.Errorf("unable to delete webhook for %s: %w", r.GetFullName(), err) diff --git a/api/repo/list_org.go b/api/repo/list_org.go index 727b91b5d..899a615a9 100644 --- a/api/repo/list_org.go +++ b/api/repo/list_org.go @@ -132,7 +132,7 @@ func ListReposForOrg(c *gin.Context) { } // See if the user is an org admin to bypass individual permission checks - perm, err := scm.FromContext(c).OrgAccess(u, o) + perm, err := scm.FromContext(c).OrgAccess(ctx, u, o) if err != nil { logrus.Errorf("unable to get user %s access level for org %s", u.GetName(), o) } diff --git a/api/repo/repair.go b/api/repo/repair.go index 1f8ffeea8..50f15c407 100644 --- a/api/repo/repair.go +++ b/api/repo/repair.go @@ -69,7 +69,7 @@ func RepairRepo(c *gin.Context) { // check if we should create the webhook if c.Value("webhookvalidation").(bool) { // send API call to remove the webhook - err := scm.FromContext(c).Disable(u, r.GetOrg(), r.GetName()) + err := scm.FromContext(c).Disable(ctx, u, r.GetOrg(), r.GetName()) if err != nil { retErr := fmt.Errorf("unable to delete webhook for %s: %w", r.GetFullName(), err) @@ -88,7 +88,7 @@ func RepairRepo(c *gin.Context) { } // send API call to create the webhook - hook, _, err = scm.FromContext(c).Enable(u, r, hook) + hook, _, err = scm.FromContext(c).Enable(ctx, u, r, hook) if err != nil { retErr := fmt.Errorf("unable to create webhook for %s: %w", r.GetFullName(), err) diff --git a/api/repo/update.go b/api/repo/update.go index 2d6b166e5..162c78ca9 100644 --- a/api/repo/update.go +++ b/api/repo/update.go @@ -285,7 +285,7 @@ func UpdateRepo(c *gin.Context) { }).Infof("platform admin %s updating repo webhook events for repo %s", admn, r.GetFullName()) } // update webhook with new events - _, err = scm.FromContext(c).Update(u, r, lastHook.GetWebhookID()) + _, err = scm.FromContext(c).Update(ctx, u, r, lastHook.GetWebhookID()) if err != nil { retErr := fmt.Errorf("unable to update repo webhook for %s: %w", r.GetFullName(), err) diff --git a/api/scm/sync.go b/api/scm/sync.go index db645f623..c462b6730 100644 --- a/api/scm/sync.go +++ b/api/scm/sync.go @@ -73,7 +73,7 @@ func SyncRepo(c *gin.Context) { logger.Infof("syncing repo %s", r.GetFullName()) // retrieve repo from source code manager service - _, err := scm.FromContext(c).GetRepo(u, r) + _, err := scm.FromContext(c).GetRepo(ctx, u, r) // if there is an error retrieving repo, we know it is deleted: set to inactive if err != nil { @@ -98,7 +98,7 @@ func SyncRepo(c *gin.Context) { // verify the user is an admin of the repo // we cannot use our normal permissions check due to the possibility the repo was deleted - perm, err := scm.FromContext(c).RepoAccess(u, u.GetToken(), o, r.GetName()) + perm, err := scm.FromContext(c).RepoAccess(ctx, u, u.GetToken(), o, r.GetName()) if err != nil { logger.Errorf("unable to get user %s access level for org %s", u.GetName(), o) } @@ -125,7 +125,7 @@ func SyncRepo(c *gin.Context) { } // update webhook - webhookExists, err := scm.FromContext(c).Update(u, r, lastHook.GetWebhookID()) + webhookExists, err := scm.FromContext(c).Update(ctx, u, r, lastHook.GetWebhookID()) if err != nil { // if webhook has been manually deleted from GitHub, diff --git a/api/scm/sync_org.go b/api/scm/sync_org.go index 6ae0b28df..ba469e8a0 100644 --- a/api/scm/sync_org.go +++ b/api/scm/sync_org.go @@ -65,7 +65,7 @@ func SyncReposForOrg(c *gin.Context) { logger.Infof("syncing repos for org %s", o) // see if the user is an org admin - perm, err := scm.FromContext(c).OrgAccess(u, o) + perm, err := scm.FromContext(c).OrgAccess(ctx, u, o) if err != nil { logger.Errorf("unable to get user %s access level for org %s", u.GetName(), o) } @@ -109,7 +109,7 @@ func SyncReposForOrg(c *gin.Context) { // iterate through captured repos and check if they are in GitHub for _, repo := range repos { - _, err := scm.FromContext(c).GetRepo(u, repo) + _, err := scm.FromContext(c).GetRepo(ctx, u, repo) // if repo cannot be captured from GitHub, set to inactive in database if err != nil { repo.SetActive(false) @@ -138,7 +138,7 @@ func SyncReposForOrg(c *gin.Context) { } // update webhook - webhookExists, err := scm.FromContext(c).Update(u, repo, lastHook.GetWebhookID()) + webhookExists, err := scm.FromContext(c).Update(ctx, u, repo, lastHook.GetWebhookID()) if err != nil { // if webhook has been manually deleted from GitHub, diff --git a/api/secret/create.go b/api/secret/create.go index fcfeac266..d1d6695d6 100644 --- a/api/secret/create.go +++ b/api/secret/create.go @@ -118,7 +118,7 @@ func CreateSecret(c *gin.Context) { // but Org/Repo != org/repo in Vela. So this check ensures that // what a user inputs matches the casing we expect in Vela since // the SCM will have the source of truth for casing. - org, err := scm.FromContext(c).GetOrgName(u, o) + org, err := scm.FromContext(c).GetOrgName(ctx, u, o) if err != nil { retErr := fmt.Errorf("unable to retrieve organization %s", o) @@ -141,7 +141,7 @@ func CreateSecret(c *gin.Context) { // retrieve org and repo name from SCM // // same story as org secret. SCM has accurate casing. - scmOrg, scmRepo, err := scm.FromContext(c).GetOrgAndRepoName(u, o, n) + scmOrg, scmRepo, err := scm.FromContext(c).GetOrgAndRepoName(ctx, u, o, n) if err != nil { retErr := fmt.Errorf("unable to retrieve repository %s/%s", o, n) diff --git a/api/secret/list.go b/api/secret/list.go index e69f4fdd5..a510294a9 100644 --- a/api/secret/list.go +++ b/api/secret/list.go @@ -106,7 +106,7 @@ func ListSecrets(c *gin.Context) { if t == constants.SecretShared && n == "*" { var err error - teams, err = scm.FromContext(c).ListUsersTeamsForOrg(u, o) + teams, err = scm.FromContext(c).ListUsersTeamsForOrg(ctx, u, o) if err != nil { retErr := fmt.Errorf("unable to list users %s teams for org %s: %w", u.GetName(), o, err) diff --git a/api/user/get_source.go b/api/user/get_source.go index 9892f319e..0ac333f38 100644 --- a/api/user/get_source.go +++ b/api/user/get_source.go @@ -55,7 +55,7 @@ func GetSourceRepos(c *gin.Context) { output := make(map[string][]library.Repo) // send API call to capture the list of repos for the user - srcRepos, err := scm.FromContext(c).ListUserRepos(u) + srcRepos, err := scm.FromContext(c).ListUserRepos(ctx, u) if err != nil { retErr := fmt.Errorf("unable to get SCM repos for user %s: %w", u.GetName(), err) diff --git a/api/webhook/post.go b/api/webhook/post.go index f869bf31d..120f7ac95 100644 --- a/api/webhook/post.go +++ b/api/webhook/post.go @@ -115,7 +115,7 @@ func PostWebhook(c *gin.Context) { // process the webhook from the source control provider // // populate build, hook, repo resources as well as PR Number / PR Comment if necessary - webhook, err := scm.FromContext(c).ProcessWebhook(c.Request) + webhook, err := scm.FromContext(c).ProcessWebhook(ctx, c.Request) if err != nil { retErr := fmt.Errorf("unable to parse webhook: %w", err) util.HandleError(c, http.StatusBadRequest, retErr) @@ -234,7 +234,7 @@ func PostWebhook(c *gin.Context) { // verify the webhook from the source control provider if c.Value("webhookvalidation").(bool) { - err = scm.FromContext(c).VerifyWebhook(dupRequest, repo) + err = scm.FromContext(c).VerifyWebhook(ctx, dupRequest, repo) if err != nil { retErr := fmt.Errorf("unable to verify webhook: %w", err) util.HandleError(c, http.StatusUnauthorized, retErr) @@ -298,7 +298,7 @@ func PostWebhook(c *gin.Context) { } // confirm current repo owner has at least write access to repo (needed for status update later) - _, err = scm.FromContext(c).RepoAccess(u, u.GetToken(), r.GetOrg(), r.GetName()) + _, err = scm.FromContext(c).RepoAccess(ctx, u, u.GetToken(), r.GetOrg(), r.GetName()) if err != nil { retErr := fmt.Errorf("unable to publish build to queue: repository owner %s no longer has write access to repository %s", u.GetName(), r.GetFullName()) util.HandleError(c, http.StatusUnauthorized, retErr) @@ -349,7 +349,7 @@ func PostWebhook(c *gin.Context) { // if the event is issue_comment and the issue is a pull request, // call SCM for more data not provided in webhook payload if strings.EqualFold(b.GetEvent(), constants.EventComment) && webhook.PRNumber > 0 { - commit, branch, baseref, headref, err := scm.FromContext(c).GetPullRequest(u, repo, webhook.PRNumber) + commit, branch, baseref, headref, err := scm.FromContext(c).GetPullRequest(ctx, u, repo, webhook.PRNumber) if err != nil { retErr := fmt.Errorf("%s: failed to get pull request info for %s: %w", baseErr, repo.GetFullName(), err) util.HandleError(c, http.StatusInternalServerError, retErr) @@ -373,7 +373,7 @@ func PostWebhook(c *gin.Context) { if !strings.EqualFold(b.GetEvent(), constants.EventComment) && !strings.EqualFold(b.GetEvent(), constants.EventPull) { // send API call to capture list of files changed for the commit - files, err = scm.FromContext(c).Changeset(u, repo, b.GetCommit()) + files, err = scm.FromContext(c).Changeset(ctx, u, repo, b.GetCommit()) if err != nil { retErr := fmt.Errorf("%s: failed to get changeset for %s: %w", baseErr, repo.GetFullName(), err) util.HandleError(c, http.StatusInternalServerError, retErr) @@ -388,7 +388,7 @@ func PostWebhook(c *gin.Context) { // check if the build event is a pull_request if strings.EqualFold(b.GetEvent(), constants.EventPull) && webhook.PRNumber > 0 { // send API call to capture list of files changed for the pull request - files, err = scm.FromContext(c).ChangesetPR(u, repo, webhook.PRNumber) + files, err = scm.FromContext(c).ChangesetPR(ctx, u, repo, webhook.PRNumber) if err != nil { retErr := fmt.Errorf("%s: failed to get changeset for %s: %w", baseErr, repo.GetFullName(), err) util.HandleError(c, http.StatusInternalServerError, retErr) @@ -430,7 +430,7 @@ func PostWebhook(c *gin.Context) { pipeline, err = database.FromContext(c).GetPipelineForRepo(ctx, b.GetCommit(), repo) if err != nil { // assume the pipeline doesn't exist in the database yet // send API call to capture the pipeline configuration file - config, err = scm.FromContext(c).ConfigBackoff(u, repo, b.GetCommit()) + config, err = scm.FromContext(c).ConfigBackoff(ctx, u, repo, b.GetCommit()) if err != nil { retErr := fmt.Errorf("%s: unable to get pipeline configuration for %s: %w", baseErr, repo.GetFullName(), err) @@ -535,7 +535,7 @@ func PostWebhook(c *gin.Context) { b.SetStatus(constants.StatusSkipped) // send API call to set the status on the commit - err = scm.FromContext(c).Status(u, b, repo.GetOrg(), repo.GetName()) + err = scm.FromContext(c).Status(ctx, u, b, repo.GetOrg(), repo.GetName()) if err != nil { logrus.Errorf("unable to set commit status for %s/%d: %v", repo.GetFullName(), b.GetNumber(), err) } @@ -664,7 +664,7 @@ func PostWebhook(c *gin.Context) { c.JSON(http.StatusOK, b) // send API call to set the status on the commit - err = scm.FromContext(c).Status(u, b, repo.GetOrg(), repo.GetName()) + err = scm.FromContext(c).Status(ctx, u, b, repo.GetOrg(), repo.GetName()) if err != nil { logrus.Errorf("unable to set commit status for %s/%d: %v", repo.GetFullName(), b.GetNumber(), err) } diff --git a/cmd/vela-server/schedule.go b/cmd/vela-server/schedule.go index 9d6c77c7f..04aebb8df 100644 --- a/cmd/vela-server/schedule.go +++ b/cmd/vela-server/schedule.go @@ -162,7 +162,7 @@ func processSchedule(ctx context.Context, s *library.Schedule, compiler compiler } // send API call to confirm repo owner has at least write access to repo - _, err = scm.RepoAccess(u, u.GetToken(), r.GetOrg(), r.GetName()) + _, err = scm.RepoAccess(ctx, u, u.GetToken(), r.GetOrg(), r.GetName()) if err != nil { return fmt.Errorf("%s does not have at least write access for repo %s", u.GetName(), r.GetFullName()) } @@ -184,7 +184,7 @@ func processSchedule(ctx context.Context, s *library.Schedule, compiler compiler } // send API call to capture the commit sha for the branch - _, commit, err := scm.GetBranch(u, r, s.GetBranch()) + _, commit, err := scm.GetBranch(ctx, u, r, s.GetBranch()) if err != nil { return fmt.Errorf("failed to get commit for repo %s on %s branch: %w", r.GetFullName(), r.GetBranch(), err) } @@ -240,7 +240,7 @@ func processSchedule(ctx context.Context, s *library.Schedule, compiler compiler pipeline, err = database.GetPipelineForRepo(context.TODO(), b.GetCommit(), r) if err != nil { // assume the pipeline doesn't exist in the database yet // send API call to capture the pipeline configuration file - config, err = scm.ConfigBackoff(u, r, b.GetCommit()) + config, err = scm.ConfigBackoff(ctx, u, r, b.GetCommit()) if err != nil { return fmt.Errorf("unable to get pipeline config for %s/%s: %w", r.GetFullName(), b.GetCommit(), err) } diff --git a/router/middleware/perm/perm.go b/router/middleware/perm/perm.go index 3e02e33be..ff960a549 100644 --- a/router/middleware/perm/perm.go +++ b/router/middleware/perm/perm.go @@ -188,6 +188,7 @@ func MustSecretAdmin() gin.HandlerFunc { n := util.PathParameter(c, "name") s := util.PathParameter(c, "secret") m := c.Request.Method + ctx := c.Request.Context() // create log fields from API metadata fields := logrus.Fields{ @@ -262,7 +263,7 @@ func MustSecretAdmin() gin.HandlerFunc { case constants.SecretOrg: logger.Debugf("verifying user %s has 'admin' permissions for org %s", u.GetName(), o) - perm, err := scm.FromContext(c).OrgAccess(u, o) + perm, err := scm.FromContext(c).OrgAccess(ctx, u, o) if err != nil { logger.Errorf("unable to get user %s access level for org %s: %v", u.GetName(), o, err) } @@ -277,7 +278,7 @@ func MustSecretAdmin() gin.HandlerFunc { case constants.SecretRepo: logger.Debugf("verifying user %s has 'admin' permissions for repo %s/%s", u.GetName(), o, n) - perm, err := scm.FromContext(c).RepoAccess(u, u.GetToken(), o, n) + perm, err := scm.FromContext(c).RepoAccess(ctx, u, u.GetToken(), o, n) if err != nil { logger.Errorf("unable to get user %s access level for repo %s/%s: %v", u.GetName(), o, n, err) } @@ -303,7 +304,7 @@ func MustSecretAdmin() gin.HandlerFunc { logger.Debugf("gathering teams user %s is a member of in the org %s", u.GetName(), o) - teams, err := scm.FromContext(c).ListUsersTeamsForOrg(u, o) + teams, err := scm.FromContext(c).ListUsersTeamsForOrg(ctx, u, o) if err != nil { logger.Errorf("unable to get users %s teams for org %s: %v", u.GetName(), o, err) } @@ -318,7 +319,7 @@ func MustSecretAdmin() gin.HandlerFunc { } else { logger.Debugf("verifying user %s has 'admin' permissions for team %s/%s", u.GetName(), o, n) - perm, err := scm.FromContext(c).TeamAccess(u, o, n) + perm, err := scm.FromContext(c).TeamAccess(ctx, u, o, n) if err != nil { logger.Errorf("unable to get user %s access level for team %s/%s: %v", u.GetName(), o, n, err) } @@ -366,7 +367,7 @@ func MustAdmin() gin.HandlerFunc { } // query source to determine requesters permissions for the repo using the requester's token - perm, err := scm.FromContext(c).RepoAccess(u, u.GetToken(), r.GetOrg(), r.GetName()) + perm, err := scm.FromContext(c).RepoAccess(ctx, u, u.GetToken(), r.GetOrg(), r.GetName()) if err != nil { // requester may not have permissions to use the Github API endpoint (requires read access) // try again using the repo owner token @@ -381,7 +382,7 @@ func MustAdmin() gin.HandlerFunc { return } - perm, err = scm.FromContext(c).RepoAccess(u, ro.GetToken(), r.GetOrg(), r.GetName()) + perm, err = scm.FromContext(c).RepoAccess(ctx, u, ro.GetToken(), r.GetOrg(), r.GetName()) if err != nil { logger.Errorf("unable to get user %s access level for repo %s", u.GetName(), r.GetFullName()) } @@ -425,7 +426,7 @@ func MustWrite() gin.HandlerFunc { } // query source to determine requesters permissions for the repo using the requester's token - perm, err := scm.FromContext(c).RepoAccess(u, u.GetToken(), r.GetOrg(), r.GetName()) + perm, err := scm.FromContext(c).RepoAccess(ctx, u, u.GetToken(), r.GetOrg(), r.GetName()) if err != nil { // requester may not have permissions to use the Github API endpoint (requires read access) // try again using the repo owner token @@ -440,7 +441,7 @@ func MustWrite() gin.HandlerFunc { return } - perm, err = scm.FromContext(c).RepoAccess(u, ro.GetToken(), r.GetOrg(), r.GetName()) + perm, err = scm.FromContext(c).RepoAccess(ctx, u, ro.GetToken(), r.GetOrg(), r.GetName()) if err != nil { logger.Errorf("unable to get user %s access level for repo %s", u.GetName(), r.GetFullName()) } @@ -508,7 +509,7 @@ func MustRead() gin.HandlerFunc { } // query source to determine requesters permissions for the repo using the requester's token - perm, err := scm.FromContext(c).RepoAccess(u, u.GetToken(), r.GetOrg(), r.GetName()) + perm, err := scm.FromContext(c).RepoAccess(ctx, u, u.GetToken(), r.GetOrg(), r.GetName()) if err != nil { // requester may not have permissions to use the Github API endpoint (requires read access) // try again using the repo owner token @@ -523,7 +524,7 @@ func MustRead() gin.HandlerFunc { return } - perm, err = scm.FromContext(c).RepoAccess(u, ro.GetToken(), r.GetOrg(), r.GetName()) + perm, err = scm.FromContext(c).RepoAccess(ctx, u, ro.GetToken(), r.GetOrg(), r.GetName()) if err != nil { logger.Errorf("unable to get user %s access level for repo %s", u.GetName(), r.GetFullName()) } diff --git a/router/middleware/pipeline/pipeline.go b/router/middleware/pipeline/pipeline.go index b3f2e57af..0fcecf7f3 100644 --- a/router/middleware/pipeline/pipeline.go +++ b/router/middleware/pipeline/pipeline.go @@ -66,7 +66,7 @@ func Establish() gin.HandlerFunc { pipeline, err := database.FromContext(c).GetPipelineForRepo(ctx, p, r) if err != nil { // assume the pipeline doesn't exist in the database yet (before pipeline support was added) // send API call to capture the pipeline configuration file - config, err := scm.FromContext(c).ConfigBackoff(u, r, p) + config, err := scm.FromContext(c).ConfigBackoff(ctx, u, r, p) if err != nil { retErr := fmt.Errorf("unable to get pipeline configuration for %s: %w", entry, err) diff --git a/scm/github/access.go b/scm/github/access.go index f5696a1ed..8733a73a4 100644 --- a/scm/github/access.go +++ b/scm/github/access.go @@ -5,6 +5,7 @@ package github import ( + "context" "strings" "github.com/sirupsen/logrus" @@ -14,7 +15,7 @@ import ( ) // OrgAccess captures the user's access level for an org. -func (c *client) OrgAccess(u *library.User, org string) (string, error) { +func (c *client) OrgAccess(ctx context.Context, u *library.User, org string) (string, error) { c.Logger.WithFields(logrus.Fields{ "org": org, "user": u.GetName(), @@ -49,7 +50,7 @@ func (c *client) OrgAccess(u *library.User, org string) (string, error) { } // RepoAccess captures the user's access level for a repo. -func (c *client) RepoAccess(u *library.User, token, org, repo string) (string, error) { +func (c *client) RepoAccess(ctx context.Context, u *library.User, token, org, repo string) (string, error) { c.Logger.WithFields(logrus.Fields{ "org": org, "repo": repo, @@ -80,7 +81,7 @@ func (c *client) RepoAccess(u *library.User, token, org, repo string) (string, e } // TeamAccess captures the user's access level for a team. -func (c *client) TeamAccess(u *library.User, org, team string) (string, error) { +func (c *client) TeamAccess(ctx context.Context, u *library.User, org, team string) (string, error) { c.Logger.WithFields(logrus.Fields{ "org": org, "team": team, @@ -142,7 +143,7 @@ func (c *client) TeamAccess(u *library.User, org, team string) (string, error) { } // ListUsersTeamsForOrg captures the user's teams for an org. -func (c *client) ListUsersTeamsForOrg(u *library.User, org string) ([]string, error) { +func (c *client) ListUsersTeamsForOrg(ctx context.Context, u *library.User, org string) ([]string, error) { c.Logger.WithFields(logrus.Fields{ "org": org, "user": u.GetName(), diff --git a/scm/github/access_test.go b/scm/github/access_test.go index 0dcba9182..79648baa0 100644 --- a/scm/github/access_test.go +++ b/scm/github/access_test.go @@ -5,6 +5,7 @@ package github import ( + "context" "net/http" "net/http/httptest" "reflect" @@ -42,7 +43,7 @@ func TestGithub_OrgAccess_Admin(t *testing.T) { client, _ := NewTest(s.URL) // run test - got, err := client.OrgAccess(u, "github") + got, err := client.OrgAccess(context.TODO(), u, "github") if resp.Code != http.StatusOK { t.Errorf("OrgAccess returned %v, want %v", resp.Code, http.StatusOK) @@ -84,7 +85,7 @@ func TestGithub_OrgAccess_Member(t *testing.T) { client, _ := NewTest(s.URL) // run test - got, err := client.OrgAccess(u, "github") + got, err := client.OrgAccess(context.TODO(), u, "github") if resp.Code != http.StatusOK { t.Errorf("OrgAccess returned %v, want %v", resp.Code, http.StatusOK) @@ -114,7 +115,7 @@ func TestGithub_OrgAccess_NotFound(t *testing.T) { client, _ := NewTest(s.URL) // run test - got, err := client.OrgAccess(u, "github") + got, err := client.OrgAccess(context.TODO(), u, "github") if err == nil { t.Errorf("OrgAccess should have returned err") @@ -152,7 +153,7 @@ func TestGithub_OrgAccess_Pending(t *testing.T) { client, _ := NewTest(s.URL) // run test - got, err := client.OrgAccess(u, "github") + got, err := client.OrgAccess(context.TODO(), u, "github") if resp.Code != http.StatusOK { t.Errorf("OrgAccess returned %v, want %v", resp.Code, http.StatusOK) @@ -182,7 +183,7 @@ func TestGithub_OrgAccess_Personal(t *testing.T) { client, _ := NewTest(s.URL) // run test - got, err := client.OrgAccess(u, "foo") + got, err := client.OrgAccess(context.TODO(), u, "foo") if err != nil { t.Errorf("OrgAccess returned err: %v", err) @@ -220,7 +221,7 @@ func TestGithub_RepoAccess_Admin(t *testing.T) { client, _ := NewTest(s.URL) // run test - got, err := client.RepoAccess(u, u.GetToken(), "github", "octocat") + got, err := client.RepoAccess(context.TODO(), u, u.GetToken(), "github", "octocat") if resp.Code != http.StatusOK { t.Errorf("RepoAccess returned %v, want %v", resp.Code, http.StatusOK) @@ -250,7 +251,7 @@ func TestGithub_RepoAccess_NotFound(t *testing.T) { client, _ := NewTest(s.URL) // run test - got, err := client.RepoAccess(u, u.GetToken(), "github", "octocat") + got, err := client.RepoAccess(context.TODO(), u, u.GetToken(), "github", "octocat") if err == nil { t.Errorf("RepoAccess should have returned err") @@ -288,7 +289,7 @@ func TestGithub_TeamAccess_Admin(t *testing.T) { client, _ := NewTest(s.URL) // run test - got, err := client.TeamAccess(u, "github", "octocat") + got, err := client.TeamAccess(context.TODO(), u, "github", "octocat") if resp.Code != http.StatusOK { t.Errorf("TeamAccess returned %v, want %v", resp.Code, http.StatusOK) @@ -330,7 +331,7 @@ func TestGithub_TeamAccess_NoAccess(t *testing.T) { client, _ := NewTest(s.URL) // run test - got, err := client.TeamAccess(u, "github", "baz") + got, err := client.TeamAccess(context.TODO(), u, "github", "baz") if resp.Code != http.StatusOK { t.Errorf("TeamAccess returned %v, want %v", resp.Code, http.StatusOK) @@ -360,7 +361,7 @@ func TestGithub_TeamAccess_NotFound(t *testing.T) { client, _ := NewTest(s.URL) // run test - got, err := client.TeamAccess(u, "github", "octocat") + got, err := client.TeamAccess(context.TODO(), u, "github", "octocat") if err == nil { t.Errorf("TeamAccess should have returned err") @@ -398,7 +399,7 @@ func TestGithub_TeamList(t *testing.T) { client, _ := NewTest(s.URL) // run test - got, err := client.ListUsersTeamsForOrg(u, "github") + got, err := client.ListUsersTeamsForOrg(context.TODO(), u, "github") if resp.Code != http.StatusOK { t.Errorf("TeamAccess returned %v, want %v", resp.Code, http.StatusOK) diff --git a/scm/github/authentication.go b/scm/github/authentication.go index 7aa317cf1..fe9080497 100644 --- a/scm/github/authentication.go +++ b/scm/github/authentication.go @@ -18,7 +18,7 @@ import ( ) // Authorize uses the given access token to authorize the user. -func (c *client) Authorize(token string) (string, error) { +func (c *client) Authorize(ctx context.Context, token string) (string, error) { c.Logger.Trace("authorizing user with token") // create GitHub OAuth client with user's token @@ -34,7 +34,7 @@ func (c *client) Authorize(token string) (string, error) { } // Login begins the authentication workflow for the session. -func (c *client) Login(w http.ResponseWriter, r *http.Request) (string, error) { +func (c *client) Login(ctx context.Context, w http.ResponseWriter, r *http.Request) (string, error) { c.Logger.Trace("processing login request") // generate a random string for creating the OAuth state @@ -57,7 +57,7 @@ func (c *client) Login(w http.ResponseWriter, r *http.Request) (string, error) { // Authenticate completes the authentication workflow for the session // and returns the remote user details. -func (c *client) Authenticate(w http.ResponseWriter, r *http.Request, oAuthState string) (*library.User, error) { +func (c *client) Authenticate(ctx context.Context, w http.ResponseWriter, r *http.Request, oAuthState string) (*library.User, error) { c.Logger.Trace("authenticating user") // get the OAuth code @@ -85,7 +85,7 @@ func (c *client) Authenticate(w http.ResponseWriter, r *http.Request, oAuthState } // authorize the user for the token - u, err := c.Authorize(token.AccessToken) + u, err := c.Authorize(ctx, token.AccessToken) if err != nil { return nil, err } @@ -98,7 +98,7 @@ func (c *client) Authenticate(w http.ResponseWriter, r *http.Request, oAuthState // AuthenticateToken completes the authentication workflow // for the session and returns the remote user details. -func (c *client) AuthenticateToken(r *http.Request) (*library.User, error) { +func (c *client) AuthenticateToken(ctx context.Context, r *http.Request) (*library.User, error) { c.Logger.Trace("authenticating user via token") token := r.Header.Get("Token") @@ -151,7 +151,7 @@ func (c *client) AuthenticateToken(r *http.Request) (*library.User, error) { return nil, errors.New("token must not be created by vela") } - u, err := c.Authorize(token) + u, err := c.Authorize(ctx, token) if err != nil { return nil, err } diff --git a/scm/github/authentication_test.go b/scm/github/authentication_test.go index d1b369a1f..3965430eb 100644 --- a/scm/github/authentication_test.go +++ b/scm/github/authentication_test.go @@ -10,6 +10,8 @@ import ( "reflect" "testing" + _context "context" + "github.com/gin-gonic/gin" "github.com/go-vela/types/library" ) @@ -45,7 +47,7 @@ func TestGithub_Authenticate(t *testing.T) { client, _ := NewTest(s.URL) // run test - got, err := client.Authenticate(context.Writer, context.Request, "bar") + got, err := client.Authenticate(_context.TODO(), context.Writer, context.Request, "bar") if resp.Code != http.StatusOK { t.Errorf("Authenticate returned %v, want %v", resp.Code, http.StatusOK) @@ -80,7 +82,7 @@ func TestGithub_Authenticate_NoCode(t *testing.T) { client, _ := NewTest(s.URL) // run test - got, err := client.Authenticate(context.Writer, context.Request, "bar") + got, err := client.Authenticate(_context.TODO(), context.Writer, context.Request, "bar") if resp.Code != http.StatusOK { t.Errorf("Authenticate returned %v, want %v", resp.Code, http.StatusOK) @@ -115,7 +117,7 @@ func TestGithub_Authenticate_NoState(t *testing.T) { client, _ := NewTest(s.URL) // run test - got, err := client.Authenticate(context.Writer, context.Request, "bar") + got, err := client.Authenticate(_context.TODO(), context.Writer, context.Request, "bar") if resp.Code != http.StatusOK { t.Errorf("Authenticate returned %v, want %v", resp.Code, http.StatusOK) @@ -150,7 +152,7 @@ func TestGithub_Authenticate_BadToken(t *testing.T) { client, _ := NewTest(s.URL) // run test - got, err := client.Authenticate(context.Writer, context.Request, "bar") + got, err := client.Authenticate(_context.TODO(), context.Writer, context.Request, "bar") if resp.Code != http.StatusOK { t.Errorf("Authenticate returned %v, want %v", resp.Code, http.StatusOK) @@ -190,7 +192,7 @@ func TestGithub_Authenticate_NotFound(t *testing.T) { client, _ := NewTest(s.URL) // run test - got, err := client.Authenticate(context.Writer, context.Request, "bar") + got, err := client.Authenticate(_context.TODO(), context.Writer, context.Request, "bar") if resp.Code != http.StatusOK { t.Errorf("Authenticate returned %v, want %v", resp.Code, http.StatusOK) @@ -227,7 +229,7 @@ func TestGithub_Authorize(t *testing.T) { // run test want := "octocat" - got, err := client.Authorize("foobar") + got, err := client.Authorize(_context.TODO(), "foobar") if resp.Code != http.StatusOK { t.Errorf("Authorize returned %v, want %v", resp.Code, http.StatusOK) @@ -261,7 +263,7 @@ func TestGithub_Authorize_NotFound(t *testing.T) { client, _ := NewTest(s.URL) // run test - got, err := client.Authorize("foobar") + got, err := client.Authorize(_context.TODO(), "foobar") if resp.Code != http.StatusOK { t.Errorf("Authorize returned %v, want %v", resp.Code, http.StatusOK) @@ -296,7 +298,7 @@ func TestGithub_Login(t *testing.T) { client, _ := NewTest(s.URL) // run test - _, err := client.Login(context.Writer, context.Request) + _, err := client.Login(_context.TODO(), context.Writer, context.Request) if resp.Code != http.StatusTemporaryRedirect { t.Errorf("Login returned %v, want %v", resp.Code, http.StatusTemporaryRedirect) @@ -333,7 +335,7 @@ func TestGithub_AuthenticateToken(t *testing.T) { client, _ := NewTest(s.URL) // run test - got, err := client.AuthenticateToken(context.Request) + got, err := client.AuthenticateToken(_context.TODO(), context.Request) if resp.Code != http.StatusOK { t.Errorf("Authenticate returned %v, want %v", resp.Code, http.StatusOK) @@ -369,7 +371,7 @@ func TestGithub_AuthenticateToken_Invalid(t *testing.T) { client, _ := NewTest(s.URL) // run test - got, err := client.AuthenticateToken(context.Request) + got, err := client.AuthenticateToken(_context.TODO(), context.Request) if resp.Code != http.StatusOK { t.Errorf("Authenticate returned %v, want %v", resp.Code, http.StatusOK) @@ -410,7 +412,7 @@ func TestGithub_AuthenticateToken_Vela_OAuth(t *testing.T) { client, _ := NewTest(s.URL) // run test - _, err := client.AuthenticateToken(context.Request) + _, err := client.AuthenticateToken(_context.TODO(), context.Request) if resp.Code != http.StatusOK { t.Errorf("AuthenticateToken returned %v, want %v", resp.Code, http.StatusOK) @@ -441,7 +443,7 @@ func TestGithub_LoginWCreds(t *testing.T) { client, _ := NewTest(s.URL) // run test - _, err := client.Login(context.Writer, context.Request) + _, err := client.Login(_context.TODO(), context.Writer, context.Request) if resp.Code != http.StatusOK { t.Errorf("Enable returned %v, want %v", resp.Code, http.StatusOK) diff --git a/scm/github/changeset.go b/scm/github/changeset.go index 21e8f511c..22cb5326e 100644 --- a/scm/github/changeset.go +++ b/scm/github/changeset.go @@ -5,6 +5,7 @@ package github import ( + "context" "fmt" "github.com/sirupsen/logrus" @@ -14,7 +15,7 @@ import ( ) // Changeset captures the list of files changed for a commit. -func (c *client) Changeset(u *library.User, r *library.Repo, sha string) ([]string, error) { +func (c *client) Changeset(ctx context.Context, u *library.User, r *library.Repo, sha string) ([]string, error) { c.Logger.WithFields(logrus.Fields{ "org": r.GetOrg(), "repo": r.GetName(), @@ -43,7 +44,7 @@ func (c *client) Changeset(u *library.User, r *library.Repo, sha string) ([]stri } // ChangesetPR captures the list of files changed for a pull request. -func (c *client) ChangesetPR(u *library.User, r *library.Repo, number int) ([]string, error) { +func (c *client) ChangesetPR(ctx context.Context, u *library.User, r *library.Repo, number int) ([]string, error) { c.Logger.WithFields(logrus.Fields{ "org": r.GetOrg(), "repo": r.GetName(), diff --git a/scm/github/changeset_test.go b/scm/github/changeset_test.go index cd4fdbe57..ee0db6083 100644 --- a/scm/github/changeset_test.go +++ b/scm/github/changeset_test.go @@ -5,6 +5,7 @@ package github import ( + "context" "net/http" "net/http/httptest" "reflect" @@ -46,7 +47,7 @@ func TestGithub_Changeset(t *testing.T) { client, _ := NewTest(s.URL) // run test - got, err := client.Changeset(u, r, "6dcb09b5b57875f334f61aebed695e2e4193db5e") + got, err := client.Changeset(context.TODO(), u, r, "6dcb09b5b57875f334f61aebed695e2e4193db5e") if resp.Code != http.StatusOK { t.Errorf("Changeset returned %v, want %v", resp.Code, http.StatusOK) @@ -92,7 +93,7 @@ func TestGithub_ChangesetPR(t *testing.T) { client, _ := NewTest(s.URL) // run test - got, err := client.ChangesetPR(u, r, 1) + got, err := client.ChangesetPR(context.TODO(), u, r, 1) if resp.Code != http.StatusOK { t.Errorf("ChangesetPR returned %v, want %v", resp.Code, http.StatusOK) diff --git a/scm/github/deployment.go b/scm/github/deployment.go index 2d59edd42..a3717d765 100644 --- a/scm/github/deployment.go +++ b/scm/github/deployment.go @@ -5,6 +5,7 @@ package github import ( + "context" "encoding/json" "github.com/sirupsen/logrus" @@ -15,7 +16,7 @@ import ( ) // GetDeployment gets a deployment from the GitHub repo. -func (c *client) GetDeployment(u *library.User, r *library.Repo, id int64) (*library.Deployment, error) { +func (c *client) GetDeployment(ctx context.Context, u *library.User, r *library.Repo, id int64) (*library.Deployment, error) { c.Logger.WithFields(logrus.Fields{ "org": r.GetOrg(), "repo": r.GetName(), @@ -53,7 +54,7 @@ func (c *client) GetDeployment(u *library.User, r *library.Repo, id int64) (*lib } // GetDeploymentCount counts a list of deployments from the GitHub repo. -func (c *client) GetDeploymentCount(u *library.User, r *library.Repo) (int64, error) { +func (c *client) GetDeploymentCount(ctx context.Context, u *library.User, r *library.Repo) (int64, error) { c.Logger.WithFields(logrus.Fields{ "org": r.GetOrg(), "repo": r.GetName(), @@ -95,7 +96,7 @@ func (c *client) GetDeploymentCount(u *library.User, r *library.Repo) (int64, er } // GetDeploymentList gets a list of deployments from the GitHub repo. -func (c *client) GetDeploymentList(u *library.User, r *library.Repo, page, perPage int) ([]*library.Deployment, error) { +func (c *client) GetDeploymentList(ctx context.Context, u *library.User, r *library.Repo, page, perPage int) ([]*library.Deployment, error) { c.Logger.WithFields(logrus.Fields{ "org": r.GetOrg(), "repo": r.GetName(), @@ -149,7 +150,7 @@ func (c *client) GetDeploymentList(u *library.User, r *library.Repo, page, perPa } // CreateDeployment creates a new deployment for the GitHub repo. -func (c *client) CreateDeployment(u *library.User, r *library.Repo, d *library.Deployment) error { +func (c *client) CreateDeployment(ctx context.Context, u *library.User, r *library.Repo, d *library.Deployment) error { c.Logger.WithFields(logrus.Fields{ "org": r.GetOrg(), "repo": r.GetName(), diff --git a/scm/github/deployment_test.go b/scm/github/deployment_test.go index 2136c0da4..35b4415cb 100644 --- a/scm/github/deployment_test.go +++ b/scm/github/deployment_test.go @@ -5,6 +5,7 @@ package github import ( + "context" "net/http" "net/http/httptest" "reflect" @@ -59,7 +60,7 @@ func TestGithub_CreateDeployment(t *testing.T) { client, _ := NewTest(s.URL, "https://foo.bar.com") // run test - err := client.CreateDeployment(u, r, d) + err := client.CreateDeployment(context.TODO(), u, r, d) if resp.Code != http.StatusOK { t.Errorf("CreateDeployment returned %v, want %v", resp.Code, http.StatusOK) @@ -113,7 +114,7 @@ func TestGithub_GetDeployment(t *testing.T) { client, _ := NewTest(s.URL, "https://foo.bar.com") // run test - got, err := client.GetDeployment(u, r, 1) + got, err := client.GetDeployment(context.TODO(), u, r, 1) if resp.Code != http.StatusOK { t.Errorf("GetDeployment returned %v, want %v", resp.Code, http.StatusOK) @@ -161,7 +162,7 @@ func TestGithub_GetDeploymentCount(t *testing.T) { client, _ := NewTest(s.URL, "https://foo.bar.com") // run test - got, err := client.GetDeploymentCount(u, r) + got, err := client.GetDeploymentCount(context.TODO(), u, r) if resp.Code != http.StatusOK { t.Errorf("GetDeployment returned %v, want %v", resp.Code, http.StatusOK) @@ -233,7 +234,7 @@ func TestGithub_GetDeploymentList(t *testing.T) { client, _ := NewTest(s.URL, "https://foo.bar.com") // run test - got, err := client.GetDeploymentList(u, r, 1, 100) + got, err := client.GetDeploymentList(context.TODO(), u, r, 1, 100) if resp.Code != http.StatusOK { t.Errorf("GetDeployment returned %v, want %v", resp.Code, http.StatusOK) diff --git a/scm/github/github.go b/scm/github/github.go index 723584585..f516d15a1 100644 --- a/scm/github/github.go +++ b/scm/github/github.go @@ -28,8 +28,6 @@ const ( eventInitialize = "initialize" ) -var ctx = context.Background() - type config struct { // specifies the address to use for the GitHub client Address string diff --git a/scm/github/org.go b/scm/github/org.go index ef9e43ca4..2f9963720 100644 --- a/scm/github/org.go +++ b/scm/github/org.go @@ -5,6 +5,7 @@ package github import ( + "context" "net/http" "github.com/sirupsen/logrus" @@ -13,7 +14,7 @@ import ( ) // GetOrgName gets org name from Github. -func (c *client) GetOrgName(u *library.User, o string) (string, error) { +func (c *client) GetOrgName(ctx context.Context, u *library.User, o string) (string, error) { c.Logger.WithFields(logrus.Fields{ "org": o, "user": u.GetName(), diff --git a/scm/github/org_test.go b/scm/github/org_test.go index b0512f9b9..35492c7e3 100644 --- a/scm/github/org_test.go +++ b/scm/github/org_test.go @@ -5,6 +5,7 @@ package github import ( + "context" "net/http" "net/http/httptest" "reflect" @@ -42,7 +43,7 @@ func TestGithub_GetOrgName(t *testing.T) { client, _ := NewTest(s.URL) // run test - got, err := client.GetOrgName(u, "github") + got, err := client.GetOrgName(context.TODO(), u, "github") if resp.Code != http.StatusOK { t.Errorf("GetOrgName returned %v, want %v", resp.Code, http.StatusOK) @@ -84,7 +85,7 @@ func TestGithub_GetOrgName_Personal(t *testing.T) { client, _ := NewTest(s.URL) // run test - got, err := client.GetOrgName(u, "octocat") + got, err := client.GetOrgName(context.TODO(), u, "octocat") if resp.Code != http.StatusOK { t.Errorf("GetOrgName returned %v, want %v", resp.Code, http.StatusOK) @@ -123,7 +124,7 @@ func TestGithub_GetOrgName_Fail(t *testing.T) { client, _ := NewTest(s.URL) // run test - _, err := client.GetOrgName(u, "octocat") + _, err := client.GetOrgName(context.TODO(), u, "octocat") if err == nil { t.Error("GetOrgName should return error") diff --git a/scm/github/repo.go b/scm/github/repo.go index a3d578878..3280c9a2a 100644 --- a/scm/github/repo.go +++ b/scm/github/repo.go @@ -5,6 +5,7 @@ package github import ( + "context" "fmt" "net/http" "strconv" @@ -20,14 +21,14 @@ import ( // ConfigBackoff is a wrapper for Config that will retry five times if the function // fails to retrieve the yaml/yml file. -func (c *client) ConfigBackoff(u *library.User, r *library.Repo, ref string) (data []byte, err error) { +func (c *client) ConfigBackoff(ctx context.Context, u *library.User, r *library.Repo, ref string) (data []byte, err error) { // number of times to retry retryLimit := 5 for i := 0; i < retryLimit; i++ { logrus.Debugf("Fetching config file - Attempt %d", i+1) // attempt to fetch the config - data, err = c.Config(u, r, ref) + data, err = c.Config(ctx, u, r, ref) // return err if the last attempt returns error if err != nil && i == retryLimit-1 { @@ -48,7 +49,7 @@ func (c *client) ConfigBackoff(u *library.User, r *library.Repo, ref string) (da } // Config gets the pipeline configuration from the GitHub repo. -func (c *client) Config(u *library.User, r *library.Repo, ref string) ([]byte, error) { +func (c *client) Config(ctx context.Context, u *library.User, r *library.Repo, ref string) ([]byte, error) { c.Logger.WithFields(logrus.Fields{ "org": r.GetOrg(), "repo": r.GetName(), @@ -93,7 +94,7 @@ func (c *client) Config(u *library.User, r *library.Repo, ref string) ([]byte, e } // Disable deactivates a repo by deleting the webhook. -func (c *client) Disable(u *library.User, org, name string) error { +func (c *client) Disable(ctx context.Context, u *library.User, org, name string) error { c.Logger.WithFields(logrus.Fields{ "org": org, "repo": name, @@ -151,7 +152,7 @@ func (c *client) Disable(u *library.User, org, name string) error { } // Enable activates a repo by creating the webhook. -func (c *client) Enable(u *library.User, r *library.Repo, h *library.Hook) (*library.Hook, string, error) { +func (c *client) Enable(ctx context.Context, u *library.User, r *library.Repo, h *library.Hook) (*library.Hook, string, error) { c.Logger.WithFields(logrus.Fields{ "org": r.GetOrg(), "repo": r.GetName(), @@ -217,7 +218,7 @@ func (c *client) Enable(u *library.User, r *library.Repo, h *library.Hook) (*lib } // Update edits a repo webhook. -func (c *client) Update(u *library.User, r *library.Repo, hookID int64) (bool, error) { +func (c *client) Update(ctx context.Context, u *library.User, r *library.Repo, hookID int64) (bool, error) { c.Logger.WithFields(logrus.Fields{ "org": r.GetOrg(), "repo": r.GetName(), @@ -266,7 +267,7 @@ func (c *client) Update(u *library.User, r *library.Repo, hookID int64) (bool, e } // Status sends the commit status for the given SHA from the GitHub repo. -func (c *client) Status(u *library.User, b *library.Build, org, name string) error { +func (c *client) Status(ctx context.Context, u *library.User, b *library.Build, org, name string) error { c.Logger.WithFields(logrus.Fields{ "build": b.GetNumber(), "org": org, @@ -368,7 +369,7 @@ func (c *client) Status(u *library.User, b *library.Build, org, name string) err } // GetRepo gets repo information from Github. -func (c *client) GetRepo(u *library.User, r *library.Repo) (*library.Repo, error) { +func (c *client) GetRepo(ctx context.Context, u *library.User, r *library.Repo) (*library.Repo, error) { c.Logger.WithFields(logrus.Fields{ "org": r.GetOrg(), "repo": r.GetName(), @@ -388,7 +389,7 @@ func (c *client) GetRepo(u *library.User, r *library.Repo) (*library.Repo, error } // GetOrgAndRepoName returns the name of the org and the repository in the SCM. -func (c *client) GetOrgAndRepoName(u *library.User, o string, r string) (string, string, error) { +func (c *client) GetOrgAndRepoName(ctx context.Context, u *library.User, o string, r string) (string, string, error) { c.Logger.WithFields(logrus.Fields{ "org": o, "repo": r, @@ -408,7 +409,7 @@ func (c *client) GetOrgAndRepoName(u *library.User, o string, r string) (string, } // ListUserRepos returns a list of all repos the user has access to. -func (c *client) ListUserRepos(u *library.User) ([]*library.Repo, error) { +func (c *client) ListUserRepos(ctx context.Context, u *library.User) ([]*library.Repo, error) { c.Logger.WithFields(logrus.Fields{ "user": u.GetName(), }).Tracef("listing source repositories for %s", u.GetName()) @@ -486,7 +487,7 @@ func toLibraryRepo(gr github.Repository) *library.Repo { // GetPullRequest defines a function that retrieves // a pull request for a repo. -func (c *client) GetPullRequest(u *library.User, r *library.Repo, number int) (string, string, string, string, error) { +func (c *client) GetPullRequest(ctx context.Context, u *library.User, r *library.Repo, number int) (string, string, string, string, error) { c.Logger.WithFields(logrus.Fields{ "org": r.GetOrg(), "repo": r.GetName(), @@ -510,7 +511,7 @@ func (c *client) GetPullRequest(u *library.User, r *library.Repo, number int) (s } // GetHTMLURL retrieves the html_url from repository contents from the GitHub repo. -func (c *client) GetHTMLURL(u *library.User, org, repo, name, ref string) (string, error) { +func (c *client) GetHTMLURL(ctx context.Context, u *library.User, org, repo, name, ref string) (string, error) { c.Logger.WithFields(logrus.Fields{ "org": org, "repo": repo, @@ -546,7 +547,7 @@ func (c *client) GetHTMLURL(u *library.User, org, repo, name, ref string) (strin } // GetBranch defines a function that retrieves a branch for a repo. -func (c *client) GetBranch(u *library.User, r *library.Repo, branch string) (string, string, error) { +func (c *client) GetBranch(ctx context.Context, u *library.User, r *library.Repo, branch string) (string, string, error) { c.Logger.WithFields(logrus.Fields{ "org": r.GetOrg(), "repo": r.GetName(), diff --git a/scm/github/repo_test.go b/scm/github/repo_test.go index a23326337..80a97afbb 100644 --- a/scm/github/repo_test.go +++ b/scm/github/repo_test.go @@ -5,6 +5,7 @@ package github import ( + "context" "fmt" "net/http" "net/http/httptest" @@ -58,7 +59,7 @@ func TestGithub_Config_YML(t *testing.T) { client, _ := NewTest(s.URL) // run test - got, err := client.Config(u, r, "") + got, err := client.Config(context.TODO(), u, r, "") if resp.Code != http.StatusOK { t.Errorf("Config returned %v, want %v", resp.Code, http.StatusOK) @@ -112,7 +113,7 @@ func TestGithub_ConfigBackoff_YML(t *testing.T) { client, _ := NewTest(s.URL) // run test - got, err := client.Config(u, r, "") + got, err := client.Config(context.TODO(), u, r, "") if resp.Code != http.StatusOK { t.Errorf("Config returned %v, want %v", resp.Code, http.StatusOK) @@ -154,7 +155,7 @@ func TestGithub_Config_YML_BadRequest(t *testing.T) { client, _ := NewTest(s.URL) // run test - got, err := client.Config(u, r, "") + got, err := client.Config(context.TODO(), u, r, "") if resp.Code != http.StatusOK { t.Errorf("Config returned %v, want %v", resp.Code, http.StatusOK) @@ -208,7 +209,7 @@ func TestGithub_Config_YAML(t *testing.T) { client, _ := NewTest(s.URL) // run test - got, err := client.Config(u, r, "") + got, err := client.Config(context.TODO(), u, r, "") if resp.Code != http.StatusOK { t.Errorf("Config returned %v, want %v", resp.Code, http.StatusOK) @@ -263,7 +264,7 @@ func TestGithub_Config_Star(t *testing.T) { client, _ := NewTest(s.URL) // run test - got, err := client.Config(u, r, "") + got, err := client.Config(context.TODO(), u, r, "") if resp.Code != http.StatusOK { t.Errorf("Config returned %v, want %v", resp.Code, http.StatusOK) @@ -318,7 +319,7 @@ func TestGithub_Config_Py(t *testing.T) { client, _ := NewTest(s.URL) // run test - got, err := client.Config(u, r, "") + got, err := client.Config(context.TODO(), u, r, "") if resp.Code != http.StatusOK { t.Errorf("Config returned %v, want %v", resp.Code, http.StatusOK) @@ -365,7 +366,7 @@ func TestGithub_Config_YAML_BadRequest(t *testing.T) { client, _ := NewTest(s.URL) // run test - got, err := client.Config(u, r, "") + got, err := client.Config(context.TODO(), u, r, "") if resp.Code != http.StatusOK { t.Errorf("Config returned %v, want %v", resp.Code, http.StatusOK) @@ -407,7 +408,7 @@ func TestGithub_Config_NotFound(t *testing.T) { client, _ := NewTest(s.URL) // run test - got, err := client.Config(u, r, "") + got, err := client.Config(context.TODO(), u, r, "") if resp.Code != http.StatusOK { t.Errorf("Config returned %v, want %v", resp.Code, http.StatusOK) @@ -450,7 +451,7 @@ func TestGithub_Disable(t *testing.T) { client, _ := NewTest(s.URL, "https://foo.bar.com") // run test - err := client.Disable(u, "foo", "bar") + err := client.Disable(context.TODO(), u, "foo", "bar") if resp.Code != http.StatusOK { t.Errorf("Disable returned %v, want %v", resp.Code, http.StatusOK) @@ -484,7 +485,7 @@ func TestGithub_Disable_NotFoundHooks(t *testing.T) { client, _ := NewTest(s.URL, "https://foo.bar.com") // run test - err := client.Disable(u, "foo", "bar") + err := client.Disable(context.TODO(), u, "foo", "bar") if resp.Code != http.StatusOK { t.Errorf("Disable returned %v, want %v", resp.Code, http.StatusOK) @@ -523,7 +524,7 @@ func TestGithub_Disable_HooksButNotFound(t *testing.T) { client, _ := NewTest(s.URL, "https://foos.ball.com") // run test - err := client.Disable(u, "foo", "bar") + err := client.Disable(context.TODO(), u, "foo", "bar") if resp.Code != http.StatusOK { t.Errorf("Disable returned %v, want %v", resp.Code, http.StatusOK) @@ -565,7 +566,7 @@ func TestGithub_Disable_MultipleHooks(t *testing.T) { client, _ := NewTest(s.URL, "https://foo.bar.com") // run test - err := client.Disable(u, "foo", "bar") + err := client.Disable(context.TODO(), u, "foo", "bar") if count != wantCount { t.Errorf("Count returned %d, want %d", count, wantCount) @@ -622,7 +623,7 @@ func TestGithub_Enable(t *testing.T) { client, _ := NewTest(s.URL) // run test - got, _, err := client.Enable(u, r, new(library.Hook)) + got, _, err := client.Enable(context.TODO(), u, r, new(library.Hook)) if resp.Code != http.StatusOK { t.Errorf("Enable returned %v, want %v", resp.Code, http.StatusOK) @@ -673,7 +674,7 @@ func TestGithub_Update(t *testing.T) { client, _ := NewTest(s.URL) // run test - _, err := client.Update(u, r, hookID) + _, err := client.Update(context.TODO(), u, r, hookID) if resp.Code != http.StatusOK { t.Errorf("Update returned %v, want %v", resp.Code, http.StatusOK) @@ -710,7 +711,7 @@ func TestGithub_Update_webhookExists_True(t *testing.T) { client, _ := NewTest(s.URL) // run test - webhookExists, err := client.Update(u, r, 0) + webhookExists, err := client.Update(context.TODO(), u, r, 0) if !webhookExists { t.Errorf("Update returned %v, want %v", webhookExists, true) @@ -747,7 +748,7 @@ func TestGithub_Update_webhookExists_False(t *testing.T) { client, _ := NewTest(s.URL) // run test - webhookExists, err := client.Update(u, r, 0) + webhookExists, err := client.Update(context.TODO(), u, r, 0) if webhookExists { t.Errorf("Update returned %v, want %v", webhookExists, false) @@ -792,7 +793,7 @@ func TestGithub_Status_Deployment(t *testing.T) { client, _ := NewTest(s.URL) // run test - err := client.Status(u, b, "foo", "bar") + err := client.Status(context.TODO(), u, b, "foo", "bar") if resp.Code != http.StatusOK { t.Errorf("Status returned %v, want %v", resp.Code, http.StatusOK) @@ -836,7 +837,7 @@ func TestGithub_Status_Running(t *testing.T) { client, _ := NewTest(s.URL) // run test - err := client.Status(u, b, "foo", "bar") + err := client.Status(context.TODO(), u, b, "foo", "bar") if resp.Code != http.StatusOK { t.Errorf("Status returned %v, want %v", resp.Code, http.StatusOK) @@ -880,7 +881,7 @@ func TestGithub_Status_Success(t *testing.T) { client, _ := NewTest(s.URL) // run test - err := client.Status(u, b, "foo", "bar") + err := client.Status(context.TODO(), u, b, "foo", "bar") if resp.Code != http.StatusOK { t.Errorf("Status returned %v, want %v", resp.Code, http.StatusOK) @@ -924,7 +925,7 @@ func TestGithub_Status_Failure(t *testing.T) { client, _ := NewTest(s.URL) // run test - err := client.Status(u, b, "foo", "bar") + err := client.Status(context.TODO(), u, b, "foo", "bar") if resp.Code != http.StatusOK { t.Errorf("Status returned %v, want %v", resp.Code, http.StatusOK) @@ -968,7 +969,7 @@ func TestGithub_Status_Killed(t *testing.T) { client, _ := NewTest(s.URL) // run test - err := client.Status(u, b, "foo", "bar") + err := client.Status(context.TODO(), u, b, "foo", "bar") if resp.Code != http.StatusOK { t.Errorf("Status returned %v, want %v", resp.Code, http.StatusOK) @@ -1012,7 +1013,7 @@ func TestGithub_Status_Skipped(t *testing.T) { client, _ := NewTest(s.URL) // run test - err := client.Status(u, b, "foo", "bar") + err := client.Status(context.TODO(), u, b, "foo", "bar") if resp.Code != http.StatusOK { t.Errorf("Status returned %v, want %v", resp.Code, http.StatusOK) @@ -1056,7 +1057,7 @@ func TestGithub_Status_Error(t *testing.T) { client, _ := NewTest(s.URL) // run test - err := client.Status(u, b, "foo", "bar") + err := client.Status(context.TODO(), u, b, "foo", "bar") if resp.Code != http.StatusOK { t.Errorf("Status returned %v, want %v", resp.Code, http.StatusOK) @@ -1107,7 +1108,7 @@ func TestGithub_GetRepo(t *testing.T) { client, _ := NewTest(s.URL) // run test - got, err := client.GetRepo(u, r) + got, err := client.GetRepo(context.TODO(), u, r) if resp.Code != http.StatusOK { t.Errorf("GetRepo returned %v, want %v", resp.Code, http.StatusOK) @@ -1150,7 +1151,7 @@ func TestGithub_GetRepo_Fail(t *testing.T) { client, _ := NewTest(s.URL) // run test - _, err := client.GetRepo(u, r) + _, err := client.GetRepo(context.TODO(), u, r) if err == nil { t.Error("GetRepo should return error") @@ -1185,7 +1186,7 @@ func TestGithub_GetOrgAndRepoName(t *testing.T) { client, _ := NewTest(s.URL) // run test - gotOrg, gotRepo, err := client.GetOrgAndRepoName(u, "octocat", "Hello-World") + gotOrg, gotRepo, err := client.GetOrgAndRepoName(context.TODO(), u, "octocat", "Hello-World") if resp.Code != http.StatusOK { t.Errorf("GetRepoName returned %v, want %v", resp.Code, http.StatusOK) @@ -1228,7 +1229,7 @@ func TestGithub_GetOrgAndRepoName_Fail(t *testing.T) { client, _ := NewTest(s.URL) // run test - _, _, err := client.GetOrgAndRepoName(u, "octocat", "Hello-World") + _, _, err := client.GetOrgAndRepoName(context.TODO(), u, "octocat", "Hello-World") if err == nil { t.Error("GetRepoName should return error") @@ -1273,7 +1274,7 @@ func TestGithub_ListUserRepos(t *testing.T) { client, _ := NewTest(s.URL) // run test - got, err := client.ListUserRepos(u) + got, err := client.ListUserRepos(context.TODO(), u) if err != nil { t.Errorf("Status returned err: %v", err) @@ -1311,7 +1312,7 @@ func TestGithub_ListUserRepos_Ineligible(t *testing.T) { client, _ := NewTest(s.URL) // run test - got, err := client.ListUserRepos(u) + got, err := client.ListUserRepos(context.TODO(), u) if err != nil { t.Errorf("Status returned err: %v", err) @@ -1356,7 +1357,7 @@ func TestGithub_GetPullRequest(t *testing.T) { client, _ := NewTest(s.URL) // run test - gotCommit, gotBranch, gotBaseRef, gotHeadRef, err := client.GetPullRequest(u, r, 1) + gotCommit, gotBranch, gotBaseRef, gotHeadRef, err := client.GetPullRequest(context.TODO(), u, r, 1) if err != nil { t.Errorf("Status returned err: %v", err) @@ -1413,7 +1414,7 @@ func TestGithub_GetBranch(t *testing.T) { client, _ := NewTest(s.URL) // run test - gotBranch, gotCommit, err := client.GetBranch(u, r, "main") + gotBranch, gotCommit, err := client.GetBranch(context.TODO(), u, r, "main") if err != nil { t.Errorf("Status returned err: %v", err) diff --git a/scm/github/webhook.go b/scm/github/webhook.go index 3c7d1e0b3..e4fc94db1 100644 --- a/scm/github/webhook.go +++ b/scm/github/webhook.go @@ -26,7 +26,7 @@ import ( // ProcessWebhook parses the webhook from a repo. // //nolint:nilerr // ignore webhook returning nil -func (c *client) ProcessWebhook(request *http.Request) (*types.Webhook, error) { +func (c *client) ProcessWebhook(ctx context.Context, request *http.Request) (*types.Webhook, error) { c.Logger.Tracef("processing GitHub webhook") // create our own record of the hook and populate its fields @@ -85,7 +85,7 @@ func (c *client) ProcessWebhook(request *http.Request) (*types.Webhook, error) { } // VerifyWebhook verifies the webhook from a repo. -func (c *client) VerifyWebhook(request *http.Request, r *library.Repo) error { +func (c *client) VerifyWebhook(ctx context.Context, request *http.Request, r *library.Repo) error { c.Logger.WithFields(logrus.Fields{ "org": r.GetOrg(), "repo": r.GetName(), diff --git a/scm/github/webhook_test.go b/scm/github/webhook_test.go index bac0692be..05a346474 100644 --- a/scm/github/webhook_test.go +++ b/scm/github/webhook_test.go @@ -89,7 +89,7 @@ func TestGithub_ProcessWebhook_Push(t *testing.T) { Build: wantBuild, } - got, err := client.ProcessWebhook(request) + got, err := client.ProcessWebhook(context.TODO(), request) if err != nil { t.Errorf("ProcessWebhook returned err: %v", err) @@ -168,7 +168,7 @@ func TestGithub_ProcessWebhook_Push_NoSender(t *testing.T) { Build: wantBuild, } - got, err := client.ProcessWebhook(request) + got, err := client.ProcessWebhook(context.TODO(), request) if err != nil { t.Errorf("ProcessWebhook returned err: %v", err) @@ -250,7 +250,7 @@ func TestGithub_ProcessWebhook_PullRequest(t *testing.T) { Build: wantBuild, } - got, err := client.ProcessWebhook(request) + got, err := client.ProcessWebhook(context.TODO(), request) if err != nil { t.Errorf("ProcessWebhook returned err: %v", err) @@ -305,7 +305,7 @@ func TestGithub_ProcessWebhook_PullRequest_ClosedAction(t *testing.T) { Build: nil, } - got, err := client.ProcessWebhook(request) + got, err := client.ProcessWebhook(context.TODO(), request) if err != nil { t.Errorf("ProcessWebhook returned err: %v", err) @@ -360,7 +360,7 @@ func TestGithub_ProcessWebhook_PullRequest_ClosedState(t *testing.T) { Build: nil, } - got, err := client.ProcessWebhook(request) + got, err := client.ProcessWebhook(context.TODO(), request) if err != nil { t.Errorf("ProcessWebhook returned err: %v", err) @@ -457,7 +457,7 @@ func TestGithub_ProcessWebhook_Deployment(t *testing.T) { Build: tt.args.build, } - got, err := client.ProcessWebhook(request) + got, err := client.ProcessWebhook(context.TODO(), request) if (err != nil) != tt.wantErr { t.Errorf("ProcessWebhook() error = %v, wantErr %v", err, tt.wantErr) return @@ -538,7 +538,7 @@ func TestGithub_ProcessWebhook_Deployment_Commit(t *testing.T) { Build: wantBuild, } - got, err := client.ProcessWebhook(request) + got, err := client.ProcessWebhook(context.TODO(), request) if err != nil { t.Errorf("ProcessWebhook returned err: %v", err) @@ -591,7 +591,7 @@ func TestGithub_ProcessWebhook_BadGithubEvent(t *testing.T) { Build: nil, } - got, err := client.ProcessWebhook(request) + got, err := client.ProcessWebhook(context.TODO(), request) if err != nil { t.Errorf("ProcessWebhook returned err: %v", err) @@ -644,7 +644,7 @@ func TestGithub_ProcessWebhook_BadContentType(t *testing.T) { Build: nil, } - got, err := client.ProcessWebhook(request) + got, err := client.ProcessWebhook(context.TODO(), request) if err != nil { t.Errorf("ProcessWebhook returned err: %v", err) @@ -681,7 +681,7 @@ func TestGithub_VerifyWebhook_EmptyRepo(t *testing.T) { client, _ := NewTest(s.URL) // run test - err = client.VerifyWebhook(request, new(library.Repo)) + err = client.VerifyWebhook(context.TODO(), request, new(library.Repo)) if err != nil { t.Errorf("VerifyWebhook should have returned err") } @@ -722,7 +722,7 @@ func TestGithub_VerifyWebhook_NoSecret(t *testing.T) { client, _ := NewTest(s.URL) // run test - err = client.VerifyWebhook(request, r) + err = client.VerifyWebhook(context.TODO(), request, r) if err != nil { t.Errorf("VerifyWebhook should have returned err") } @@ -794,7 +794,7 @@ func TestGithub_ProcessWebhook_IssueComment_PR(t *testing.T) { Build: wantBuild, } - got, err := client.ProcessWebhook(request) + got, err := client.ProcessWebhook(context.TODO(), request) if err != nil { t.Errorf("ProcessWebhook returned err: %v", err) @@ -871,7 +871,7 @@ func TestGithub_ProcessWebhook_IssueComment_Created(t *testing.T) { Build: wantBuild, } - got, err := client.ProcessWebhook(request) + got, err := client.ProcessWebhook(context.TODO(), request) if err != nil { t.Errorf("ProcessWebhook returned err: %v", err) @@ -926,7 +926,7 @@ func TestGithub_ProcessWebhook_IssueComment_Deleted(t *testing.T) { Build: nil, } - got, err := client.ProcessWebhook(request) + got, err := client.ProcessWebhook(context.TODO(), request) if err != nil { t.Errorf("ProcessWebhook returned err: %v", err) @@ -990,7 +990,7 @@ func TestGitHub_ProcessWebhook_RepositoryRename(t *testing.T) { Repo: wantRepo, } - got, err := client.ProcessWebhook(request) + got, err := client.ProcessWebhook(context.TODO(), request) if err != nil { t.Errorf("ProcessWebhook returned err: %v", err) @@ -1054,7 +1054,7 @@ func TestGitHub_ProcessWebhook_RepositoryTransfer(t *testing.T) { Repo: wantRepo, } - got, err := client.ProcessWebhook(request) + got, err := client.ProcessWebhook(context.TODO(), request) if err != nil { t.Errorf("ProcessWebhook returned err: %v", err) @@ -1118,7 +1118,7 @@ func TestGitHub_ProcessWebhook_RepositoryArchived(t *testing.T) { Repo: wantRepo, } - got, err := client.ProcessWebhook(request) + got, err := client.ProcessWebhook(context.TODO(), request) if err != nil { t.Errorf("ProcessWebhook returned err: %v", err) @@ -1182,7 +1182,7 @@ func TestGitHub_ProcessWebhook_RepositoryEdited(t *testing.T) { Repo: wantRepo, } - got, err := client.ProcessWebhook(request) + got, err := client.ProcessWebhook(context.TODO(), request) if err != nil { t.Errorf("ProcessWebhook returned err: %v", err) @@ -1246,7 +1246,7 @@ func TestGitHub_ProcessWebhook_Repository(t *testing.T) { Repo: wantRepo, } - got, err := client.ProcessWebhook(request) + got, err := client.ProcessWebhook(context.TODO(), request) if err != nil { t.Errorf("ProcessWebhook returned err: %v", err) @@ -1300,7 +1300,7 @@ func TestGithub_Redeliver_Webhook(t *testing.T) { client, _ := NewTest(s.URL, "https://foo.bar.com") // run test - err := client.RedeliverWebhook(ctx, u, _repo, _hook) + err := client.RedeliverWebhook(context.TODO(), u, _repo, _hook) if err != nil { t.Errorf("RedeliverWebhook returned err: %v", err) @@ -1348,7 +1348,7 @@ func TestGithub_GetDeliveryID(t *testing.T) { ghClient := client.newClientToken(*u.Token) // run test - got, err := client.getDeliveryID(ctx, ghClient, _repo, _hook) + got, err := client.getDeliveryID(context.TODO(), ghClient, _repo, _hook) if err != nil { t.Errorf("RedeliverWebhook returned err: %v", err) diff --git a/scm/service.go b/scm/service.go index ae9488724..7d015dca2 100644 --- a/scm/service.go +++ b/scm/service.go @@ -25,36 +25,36 @@ type Service interface { // Authorize defines a function that uses the // given access token to authorize the user. - Authorize(string) (string, error) + Authorize(context.Context, string) (string, error) // Authenticate defines a function that completes // the OAuth workflow for the session. - Authenticate(http.ResponseWriter, *http.Request, string) (*library.User, error) + Authenticate(context.Context, http.ResponseWriter, *http.Request, string) (*library.User, error) // AuthenticateToken defines a function that completes // the OAuth workflow for the session using PAT Token - AuthenticateToken(*http.Request) (*library.User, error) + AuthenticateToken(context.Context, *http.Request) (*library.User, error) // Login defines a function that begins // the OAuth workflow for the session. - Login(http.ResponseWriter, *http.Request) (string, error) + Login(context.Context, http.ResponseWriter, *http.Request) (string, error) // Access SCM Interface Functions // OrgAccess defines a function that captures // the user's access level for an org. - OrgAccess(*library.User, string) (string, error) + OrgAccess(context.Context, *library.User, string) (string, error) // RepoAccess defines a function that captures // the user's access level for a repo. - RepoAccess(*library.User, string, string, string) (string, error) + RepoAccess(context.Context, *library.User, string, string, string) (string, error) // TeamAccess defines a function that captures // the user's access level for a team. - TeamAccess(*library.User, string, string) (string, error) + TeamAccess(context.Context, *library.User, string, string) (string, error) // Teams SCM Interface Functions // ListUsersTeamsForOrg defines a function that captures // the user's teams for an org - ListUsersTeamsForOrg(*library.User, string) ([]string, error) + ListUsersTeamsForOrg(context.Context, *library.User, string) ([]string, error) // Changeset SCM Interface Functions @@ -62,79 +62,79 @@ type Service interface { // of files changed for a commit. // // https://en.wikipedia.org/wiki/Changeset. - Changeset(*library.User, *library.Repo, string) ([]string, error) + Changeset(context.Context, *library.User, *library.Repo, string) ([]string, error) // ChangesetPR defines a function that captures the list // of files changed for a pull request. // // https://en.wikipedia.org/wiki/Changeset. - ChangesetPR(*library.User, *library.Repo, int) ([]string, error) + ChangesetPR(context.Context, *library.User, *library.Repo, int) ([]string, error) // Deployment SCM Interface Functions // GetDeployment defines a function that // gets a deployment by number and repo. - GetDeployment(*library.User, *library.Repo, int64) (*library.Deployment, error) + GetDeployment(context.Context, *library.User, *library.Repo, int64) (*library.Deployment, error) // GetDeploymentCount defines a function that // counts a list of all deployment for a repo. - GetDeploymentCount(*library.User, *library.Repo) (int64, error) + GetDeploymentCount(context.Context, *library.User, *library.Repo) (int64, error) // GetDeploymentList defines a function that gets // a list of all deployments for a repo. - GetDeploymentList(*library.User, *library.Repo, int, int) ([]*library.Deployment, error) + GetDeploymentList(context.Context, *library.User, *library.Repo, int, int) ([]*library.Deployment, error) // CreateDeployment defines a function that // creates a new deployment. - CreateDeployment(*library.User, *library.Repo, *library.Deployment) error + CreateDeployment(context.Context, *library.User, *library.Repo, *library.Deployment) error // Repo SCM Interface Functions // Config defines a function that captures // the pipeline configuration from a repo. - Config(*library.User, *library.Repo, string) ([]byte, error) + Config(context.Context, *library.User, *library.Repo, string) ([]byte, error) // ConfigBackoff is a truncated constant backoff wrapper for Config. // Retry again in five seconds if Config fails to retrieve yaml/yml file. // Will return an error after five failed attempts. - ConfigBackoff(*library.User, *library.Repo, string) ([]byte, error) + ConfigBackoff(context.Context, *library.User, *library.Repo, string) ([]byte, error) // Disable defines a function that deactivates // a repo by destroying the webhook. - Disable(*library.User, string, string) error + Disable(context.Context, *library.User, string, string) error // Enable defines a function that activates // a repo by creating the webhook. - Enable(*library.User, *library.Repo, *library.Hook) (*library.Hook, string, error) + Enable(context.Context, *library.User, *library.Repo, *library.Hook) (*library.Hook, string, error) // Update defines a function that updates // a webhook for a specified repo. - Update(*library.User, *library.Repo, int64) (bool, error) + Update(context.Context, *library.User, *library.Repo, int64) (bool, error) // Status defines a function that sends the // commit status for the given SHA from a repo. - Status(*library.User, *library.Build, string, string) error + Status(context.Context, *library.User, *library.Build, string, string) error // ListUserRepos defines a function that retrieves // all repos with admin rights for the user. - ListUserRepos(*library.User) ([]*library.Repo, error) + ListUserRepos(context.Context, *library.User) ([]*library.Repo, error) // GetBranch defines a function that retrieves // a branch for a repo. - GetBranch(*library.User, *library.Repo, string) (string, string, error) + GetBranch(context.Context, *library.User, *library.Repo, string) (string, string, error) // GetPullRequest defines a function that retrieves // a pull request for a repo. - GetPullRequest(*library.User, *library.Repo, int) (string, string, string, string, error) + GetPullRequest(context.Context, *library.User, *library.Repo, int) (string, string, string, string, error) // GetRepo defines a function that retrieves // details for a repo. - GetRepo(*library.User, *library.Repo) (*library.Repo, error) + GetRepo(context.Context, *library.User, *library.Repo) (*library.Repo, error) // GetOrgAndRepoName defines a function that retrieves // the name of the org and repo in the SCM. - GetOrgAndRepoName(*library.User, string, string) (string, string, error) + GetOrgAndRepoName(context.Context, *library.User, string, string) (string, string, error) // GetOrg defines a function that retrieves // the name for an org in the SCM. - GetOrgName(*library.User, string) (string, error) + GetOrgName(context.Context, *library.User, string) (string, error) // GetHTMLURL defines a function that retrieves // a repository file's html_url. - GetHTMLURL(*library.User, string, string, string, string) (string, error) + GetHTMLURL(context.Context, *library.User, string, string, string, string) (string, error) // Webhook SCM Interface Functions // ProcessWebhook defines a function that // parses the webhook from a repo. - ProcessWebhook(*http.Request) (*types.Webhook, error) + ProcessWebhook(context.Context, *http.Request) (*types.Webhook, error) // VerifyWebhook defines a function that // verifies the webhook from a repo. - VerifyWebhook(*http.Request, *library.Repo) error + VerifyWebhook(context.Context, *http.Request, *library.Repo) error // RedeliverWebhook defines a function that // redelivers the webhook from the SCM. RedeliverWebhook(context.Context, *library.User, *library.Repo, *library.Hook) error From 4d424137707a23546b705ec866b51193dd048043 Mon Sep 17 00:00:00 2001 From: davidvader Date: Mon, 18 Sep 2023 16:52:17 -0500 Subject: [PATCH 18/59] chore: replace TODO with ctx --- cmd/vela-server/schedule.go | 12 ++++++------ database/build/list_deployment.go | 2 +- database/pipeline/list_repo.go | 2 +- database/pipeline/pipeline.go | 1 - 4 files changed, 8 insertions(+), 9 deletions(-) diff --git a/cmd/vela-server/schedule.go b/cmd/vela-server/schedule.go index 04aebb8df..0ca6024c4 100644 --- a/cmd/vela-server/schedule.go +++ b/cmd/vela-server/schedule.go @@ -173,7 +173,7 @@ func processSchedule(ctx context.Context, s *library.Schedule, compiler compiler } // send API call to capture the number of pending or running builds for the repo - builds, err := database.CountBuildsForRepo(context.TODO(), r, filters) + builds, err := database.CountBuildsForRepo(ctx, r, filters) if err != nil { return fmt.Errorf("unable to get count of builds for repo %s: %w", r.GetFullName(), err) } @@ -237,7 +237,7 @@ func processSchedule(ctx context.Context, s *library.Schedule, compiler compiler } // send API call to attempt to capture the pipeline - pipeline, err = database.GetPipelineForRepo(context.TODO(), b.GetCommit(), r) + pipeline, err = database.GetPipelineForRepo(ctx, b.GetCommit(), r) if err != nil { // assume the pipeline doesn't exist in the database yet // send API call to capture the pipeline configuration file config, err = scm.ConfigBackoff(ctx, u, r, b.GetCommit()) @@ -326,7 +326,7 @@ func processSchedule(ctx context.Context, s *library.Schedule, compiler compiler pipeline.SetRef(b.GetRef()) // send API call to create the pipeline - pipeline, err = database.CreatePipeline(context.TODO(), pipeline) + pipeline, err = database.CreatePipeline(ctx, pipeline) if err != nil { err = fmt.Errorf("failed to create pipeline for %s: %w", r.GetFullName(), err) @@ -351,7 +351,7 @@ func processSchedule(ctx context.Context, s *library.Schedule, compiler compiler // using the same Number and thus create a constraint // conflict; consider deleting the partially created // build object in the database - err = build.PlanBuild(context.TODO(), database, p, b, r) + err = build.PlanBuild(ctx, database, p, b, r) if err != nil { // check if the retry limit has been exceeded if i < retryLimit-1 { @@ -380,14 +380,14 @@ func processSchedule(ctx context.Context, s *library.Schedule, compiler compiler } // send API call to capture the triggered build - b, err = database.GetBuildForRepo(context.TODO(), r, b.GetNumber()) + b, err = database.GetBuildForRepo(ctx, r, b.GetNumber()) if err != nil { return fmt.Errorf("unable to get new build %s/%d: %w", r.GetFullName(), b.GetNumber(), err) } // publish the build to the queue go build.PublishToQueue( - context.TODO(), + ctx, queue, database, p, diff --git a/database/build/list_deployment.go b/database/build/list_deployment.go index 0d931fae7..e8f44f4b0 100644 --- a/database/build/list_deployment.go +++ b/database/build/list_deployment.go @@ -27,7 +27,7 @@ func (e *engine) ListBuildsForDeployment(ctx context.Context, d *library.Deploym builds := []*library.Build{} // count the results - count, err := e.CountBuildsForDeployment(context.TODO(), d, filters) + count, err := e.CountBuildsForDeployment(ctx, d, filters) if err != nil { return builds, 0, err } diff --git a/database/pipeline/list_repo.go b/database/pipeline/list_repo.go index e8c38afab..ce3983d23 100644 --- a/database/pipeline/list_repo.go +++ b/database/pipeline/list_repo.go @@ -28,7 +28,7 @@ func (e *engine) ListPipelinesForRepo(ctx context.Context, r *library.Repo, page pipelines := []*library.Pipeline{} // count the results - count, err := e.CountPipelinesForRepo(context.TODO(), r) + count, err := e.CountPipelinesForRepo(ctx, r) if err != nil { return pipelines, 0, err } diff --git a/database/pipeline/pipeline.go b/database/pipeline/pipeline.go index 7c9f29d8f..34852650f 100644 --- a/database/pipeline/pipeline.go +++ b/database/pipeline/pipeline.go @@ -53,7 +53,6 @@ func New(opts ...EngineOpt) (*engine, error) { e.client = new(gorm.DB) e.config = new(config) e.logger = new(logrus.Entry) - e.ctx = context.TODO() // apply all provided configuration options for _, opt := range opts { From 36a7b605a96437513dfb36bd7e7f1367700695d0 Mon Sep 17 00:00:00 2001 From: davidvader Date: Tue, 19 Sep 2023 08:23:29 -0500 Subject: [PATCH 19/59] chore: merge with main --- api/repo/create.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/api/repo/create.go b/api/repo/create.go index a0b70f3a4..efa1611df 100644 --- a/api/repo/create.go +++ b/api/repo/create.go @@ -255,11 +255,7 @@ func CreateRepo(c *gin.Context) { // check if we should create the webhook if c.Value("webhookvalidation").(bool) { // send API call to create the webhook -<<<<<<< HEAD - h, _, err = scm.FromContext(c).Enable(c.Request.Context(), u, r, h) -======= h, _, err = scm.FromContext(c).Enable(ctx, u, r, h) ->>>>>>> 4d424137707a23546b705ec866b51193dd048043 if err != nil { retErr := fmt.Errorf("unable to create webhook for %s: %w", r.GetFullName(), err) From d43a49db3323dc47008aadb18d3b6f13348bf334 Mon Sep 17 00:00:00 2001 From: davidvader Date: Tue, 19 Sep 2023 09:38:02 -0500 Subject: [PATCH 20/59] chore: merge with main --- database/database.go | 2 +- scm/github/access.go | 8 ++++---- scm/github/authentication.go | 2 +- scm/github/changeset.go | 4 ++-- scm/github/deployment.go | 8 ++++---- scm/github/org.go | 2 +- scm/github/repo.go | 14 +++++++------- scm/github/webhook.go | 2 +- tracing/tracer.go | 5 +++-- 9 files changed, 24 insertions(+), 23 deletions(-) diff --git a/database/database.go b/database/database.go index 08cd62d5c..145c6a409 100644 --- a/database/database.go +++ b/database/database.go @@ -94,7 +94,7 @@ func New(opts ...EngineOpt) (Interface, error) { e.client = new(gorm.DB) e.config = new(config) e.logger = new(logrus.Entry) - e.ctx = context.TODO() + e.ctx = context.Background() // apply all provided configuration options for _, opt := range opts { diff --git a/scm/github/access.go b/scm/github/access.go index d58fc6bdc..2b2a44281 100644 --- a/scm/github/access.go +++ b/scm/github/access.go @@ -33,7 +33,7 @@ func (c *client) OrgAccess(ctx context.Context, u *library.User, org string) (st } // create GitHub OAuth client with user's token - client := c.newClientToken(context.TODO(), *u.Token) + client := c.newClientToken(ctx, *u.Token) // send API call to capture org access level for user membership, _, err := client.Organizations.GetOrgMembership(ctx, *u.Name, org) @@ -69,7 +69,7 @@ func (c *client) RepoAccess(ctx context.Context, u *library.User, token, org, re } // create github oauth client with the given token - client := c.newClientToken(context.TODO(), token) + client := c.newClientToken(ctx, token) // send API call to capture repo access level for user perm, _, err := client.Repositories.GetPermissionLevel(ctx, org, repo, u.GetName()) @@ -100,7 +100,7 @@ func (c *client) TeamAccess(ctx context.Context, u *library.User, org, team stri } // create GitHub OAuth client with user's token - client := c.newClientToken(context.TODO(), u.GetToken()) + client := c.newClientToken(ctx, u.GetToken()) teams := []*github.Team{} // set the max per page for the options to capture the list of repos @@ -150,7 +150,7 @@ func (c *client) ListUsersTeamsForOrg(ctx context.Context, u *library.User, org }).Tracef("capturing %s team membership for org %s", u.GetName(), org) // create GitHub OAuth client with user's token - client := c.newClientToken(context.TODO(), u.GetToken()) + client := c.newClientToken(ctx, u.GetToken()) teams := []*github.Team{} // set the max per page for the options to capture the list of repos diff --git a/scm/github/authentication.go b/scm/github/authentication.go index cbcbb592b..0634a57f5 100644 --- a/scm/github/authentication.go +++ b/scm/github/authentication.go @@ -22,7 +22,7 @@ func (c *client) Authorize(ctx context.Context, token string) (string, error) { c.Logger.Trace("authorizing user with token") // create GitHub OAuth client with user's token - client := c.newClientToken(context.TODO(), token) + client := c.newClientToken(ctx, token) // send API call to capture the current user making the call u, _, err := client.Users.Get(ctx, "") diff --git a/scm/github/changeset.go b/scm/github/changeset.go index 088d28448..31b431c51 100644 --- a/scm/github/changeset.go +++ b/scm/github/changeset.go @@ -23,7 +23,7 @@ func (c *client) Changeset(ctx context.Context, u *library.User, r *library.Repo }).Tracef("capturing commit changeset for %s/commit/%s", r.GetFullName(), sha) // create GitHub OAuth client with user's token - client := c.newClientToken(context.TODO(), u.GetToken()) + client := c.newClientToken(ctx, u.GetToken()) s := []string{} // set the max per page for the options to capture the commit @@ -52,7 +52,7 @@ func (c *client) ChangesetPR(ctx context.Context, u *library.User, r *library.Re }).Tracef("capturing pull request changeset for %s/pull/%d", r.GetFullName(), number) // create GitHub OAuth client with user's token - client := c.newClientToken(context.TODO(), u.GetToken()) + client := c.newClientToken(ctx, u.GetToken()) s := []string{} f := []*github.CommitFile{} diff --git a/scm/github/deployment.go b/scm/github/deployment.go index b4f70a9ce..8bc97b2ef 100644 --- a/scm/github/deployment.go +++ b/scm/github/deployment.go @@ -24,7 +24,7 @@ func (c *client) GetDeployment(ctx context.Context, u *library.User, r *library. }).Tracef("capturing deployment %d for repo %s", id, r.GetFullName()) // create GitHub OAuth client with user's token - client := c.newClientToken(context.TODO(), *u.Token) + client := c.newClientToken(ctx, *u.Token) // send API call to capture the deployment deployment, _, err := client.Repositories.GetDeployment(ctx, r.GetOrg(), r.GetName(), id) @@ -62,7 +62,7 @@ func (c *client) GetDeploymentCount(ctx context.Context, u *library.User, r *lib }).Tracef("counting deployments for repo %s", r.GetFullName()) // create GitHub OAuth client with user's token - client := c.newClientToken(context.TODO(), *u.Token) + client := c.newClientToken(ctx, *u.Token) // create variable to track the deployments deployments := []*github.Deployment{} @@ -104,7 +104,7 @@ func (c *client) GetDeploymentList(ctx context.Context, u *library.User, r *libr }).Tracef("listing deployments for repo %s", r.GetFullName()) // create GitHub OAuth client with user's token - client := c.newClientToken(context.TODO(), *u.Token) + client := c.newClientToken(ctx, *u.Token) // set pagination options for listing deployments opts := &github.DeploymentsListOptions{ @@ -158,7 +158,7 @@ func (c *client) CreateDeployment(ctx context.Context, u *library.User, r *libra }).Tracef("creating deployment for repo %s", r.GetFullName()) // create GitHub OAuth client with user's token - client := c.newClientToken(context.TODO(), *u.Token) + client := c.newClientToken(ctx, *u.Token) var payload interface{} if d.Payload == nil { diff --git a/scm/github/org.go b/scm/github/org.go index 4fa373995..6b7f7b67d 100644 --- a/scm/github/org.go +++ b/scm/github/org.go @@ -21,7 +21,7 @@ func (c *client) GetOrgName(ctx context.Context, u *library.User, o string) (str }).Tracef("retrieving org information for %s", o) // create GitHub OAuth client with user's token - client := c.newClientToken(context.TODO(), u.GetToken()) + client := c.newClientToken(ctx, u.GetToken()) // send an API call to get the org info orgInfo, resp, err := client.Organizations.Get(ctx, o) diff --git a/scm/github/repo.go b/scm/github/repo.go index cba6a7b90..1528f0280 100644 --- a/scm/github/repo.go +++ b/scm/github/repo.go @@ -59,7 +59,7 @@ func (c *client) Config(ctx context.Context, u *library.User, r *library.Repo, r }).Tracef("capturing configuration file for %s/commit/%s", r.GetFullName(), ref) // create GitHub OAuth client with user's token - client := c.newClientToken(context.TODO(), *u.Token) + client := c.newClientToken(ctx, *u.Token) files := []string{".vela.yml", ".vela.yaml"} @@ -228,7 +228,7 @@ func (c *client) Update(ctx context.Context, u *library.User, r *library.Repo, h }).Tracef("updating repository webhook for %s/%s", r.GetOrg(), r.GetName()) // create GitHub OAuth client with user's token - client := c.newClientToken(context.TODO(), *u.Token) + client := c.newClientToken(ctx, *u.Token) // always listen to repository events in case of repo name change events := []string{eventRepository} @@ -278,7 +278,7 @@ func (c *client) Status(ctx context.Context, u *library.User, b *library.Build, }).Tracef("setting commit status for %s/%s/%d @ %s", org, name, b.GetNumber(), b.GetCommit()) // create GitHub OAuth client with user's token - client := c.newClientToken(context.TODO(), *u.Token) + client := c.newClientToken(ctx, *u.Token) context := fmt.Sprintf("%s/%s", c.config.StatusContext, b.GetEvent()) url := fmt.Sprintf("%s/%s/%s/%d", c.config.WebUIAddress, org, name, b.GetNumber()) @@ -399,7 +399,7 @@ func (c *client) GetOrgAndRepoName(ctx context.Context, u *library.User, o strin }).Tracef("retrieving repository information for %s/%s", o, r) // create GitHub OAuth client with user's token - client := c.newClientToken(context.TODO(), u.GetToken()) + client := c.newClientToken(ctx, u.GetToken()) // send an API call to get the repo info repo, _, err := client.Repositories.Get(ctx, o, r) @@ -499,7 +499,7 @@ func (c *client) GetPullRequest(ctx context.Context, u *library.User, r *library }).Tracef("retrieving pull request %d for repo %s", number, r.GetFullName()) // create GitHub OAuth client with user's token - client := c.newClientToken(context.TODO(), u.GetToken()) + client := c.newClientToken(ctx, u.GetToken()) pull, _, err := client.PullRequests.Get(ctx, r.GetOrg(), r.GetName(), number) if err != nil { @@ -523,7 +523,7 @@ func (c *client) GetHTMLURL(ctx context.Context, u *library.User, org, repo, nam }).Tracef("capturing html_url for %s/%s/%s@%s", org, repo, name, ref) // create GitHub OAuth client with user's token - client := c.newClientToken(context.TODO(), *u.Token) + client := c.newClientToken(ctx, *u.Token) // set the reference for the options to capture the repository contents opts := &github.RepositoryContentGetOptions{ @@ -559,7 +559,7 @@ func (c *client) GetBranch(ctx context.Context, u *library.User, r *library.Repo }).Tracef("retrieving branch %s for repo %s", branch, r.GetFullName()) // create GitHub OAuth client with user's token - client := c.newClientToken(context.TODO(), u.GetToken()) + client := c.newClientToken(ctx, u.GetToken()) data, _, err := client.Repositories.GetBranch(ctx, r.GetOrg(), r.GetName(), branch, true) if err != nil { diff --git a/scm/github/webhook.go b/scm/github/webhook.go index 72cabea7e..31de92dc5 100644 --- a/scm/github/webhook.go +++ b/scm/github/webhook.go @@ -103,7 +103,7 @@ func (c *client) VerifyWebhook(ctx context.Context, request *http.Request, r *li func (c *client) RedeliverWebhook(ctx context.Context, u *library.User, r *library.Repo, h *library.Hook) error { // create GitHub OAuth client with user's token //nolint:contextcheck // do not need to pass context in this instance - client := c.newClientToken(context.TODO(), *u.Token) + client := c.newClientToken(ctx, *u.Token) // capture the delivery ID of the hook using GitHub API deliveryID, err := c.getDeliveryID(ctx, client, r, h) diff --git a/tracing/tracer.go b/tracing/tracer.go index de89e2359..98f177a2b 100644 --- a/tracing/tracer.go +++ b/tracing/tracer.go @@ -21,14 +21,15 @@ import ( // initTracer returns the tracer provider supplied to the tracing config. func initTracer(c *cli.Context) (*sdktrace.TracerProvider, error) { client := otlptracehttp.NewClient() + ctx := context.Background() // TODO: inject actual context - exporter, err := otlp.New(context.TODO(), client) + exporter, err := otlp.New(ctx, client) if err != nil { return nil, err } - res, err := resource.New(context.TODO(), resource.WithAttributes( + res, err := resource.New(ctx, resource.WithAttributes( semconv.ServiceName(c.String("tracing.service.name")), )) if err != nil { From f597009afff2f1f8acbed6738ee0a0da6d3e0bb1 Mon Sep 17 00:00:00 2001 From: davidvader Date: Thu, 1 Aug 2024 13:44:55 -0500 Subject: [PATCH 21/59] chore: rm launch.json --- .vscode/launch.json | 15 --------------- 1 file changed, 15 deletions(-) delete mode 100644 .vscode/launch.json diff --git a/.vscode/launch.json b/.vscode/launch.json deleted file mode 100644 index 608d3c699..000000000 --- a/.vscode/launch.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - // Use IntelliSense to learn about possible attributes. - // Hover to view descriptions of existing attributes. - // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 - "version": "0.2.0", - "configurations": [ - { - "name": "Launch Package", - "type": "go", - "request": "launch", - "mode": "auto", - "program": "${fileDirname}" - } - ] -} \ No newline at end of file From d9da5a3aaea6c6ee4749dc04561b8d0efc706d34 Mon Sep 17 00:00:00 2001 From: davidvader Date: Thu, 1 Aug 2024 13:47:24 -0500 Subject: [PATCH 22/59] chore: headers and misc cleanup --- database/context.go | 2 -- tracing/config.go | 4 +--- tracing/flags.go | 4 +--- tracing/tracer.go | 4 +--- 4 files changed, 3 insertions(+), 11 deletions(-) diff --git a/database/context.go b/database/context.go index ca4ca464c..9b65c53a1 100644 --- a/database/context.go +++ b/database/context.go @@ -29,8 +29,6 @@ func FromContext(c context.Context) Interface { return nil } - // assign as inflight property - return d } diff --git a/tracing/config.go b/tracing/config.go index 0b45961e3..436a15b7c 100644 --- a/tracing/config.go +++ b/tracing/config.go @@ -1,6 +1,4 @@ -// Copyright (c) 2023 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. +// SPDX-License-Identifier: Apache-2.0 package tracing diff --git a/tracing/flags.go b/tracing/flags.go index e9d82fe1e..98e0a628b 100644 --- a/tracing/flags.go +++ b/tracing/flags.go @@ -1,6 +1,4 @@ -// Copyright (c) 2023 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. +// SPDX-License-Identifier: Apache-2.0 package tracing diff --git a/tracing/tracer.go b/tracing/tracer.go index 98f177a2b..1c4b5492a 100644 --- a/tracing/tracer.go +++ b/tracing/tracer.go @@ -1,6 +1,4 @@ -// Copyright (c) 2023 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. +// SPDX-License-Identifier: Apache-2.0 package tracing From 7b4e110e202e56c84f70dde2d11de3f3c0665ecd Mon Sep 17 00:00:00 2001 From: davidvader Date: Thu, 1 Aug 2024 15:34:49 -0500 Subject: [PATCH 23/59] chore: golangci --- cmd/vela-server/server.go | 5 +---- database/context.go | 3 ++- database/context_test.go | 3 ++- database/database.go | 12 ++++++------ database/database_test.go | 3 ++- database/opts.go | 3 +-- scm/github/github.go | 3 +-- scm/github/repo.go | 3 +-- scm/github/webhook.go | 2 +- something.json | 0 tracing/config.go | 1 - tracing/tracer.go | 5 ++--- 12 files changed, 19 insertions(+), 24 deletions(-) create mode 100644 something.json diff --git a/cmd/vela-server/server.go b/cmd/vela-server/server.go index 8182ada0a..0e9283f71 100644 --- a/cmd/vela-server/server.go +++ b/cmd/vela-server/server.go @@ -14,10 +14,6 @@ import ( "time" "github.com/gin-gonic/gin" - "github.com/go-vela/server/database" - "github.com/go-vela/server/router" - "github.com/go-vela/server/router/middleware" - "github.com/go-vela/server/tracing" "github.com/sirupsen/logrus" "github.com/urfave/cli/v2" "go.opentelemetry.io/contrib/instrumentation/github.com/gin-gonic/gin/otelgin" @@ -31,6 +27,7 @@ import ( "github.com/go-vela/server/queue" "github.com/go-vela/server/router" "github.com/go-vela/server/router/middleware" + "github.com/go-vela/server/tracing" ) //nolint:funlen,gocyclo // ignore function length and cyclomatic complexity diff --git a/database/context.go b/database/context.go index 9b65c53a1..54d0b9f0f 100644 --- a/database/context.go +++ b/database/context.go @@ -5,9 +5,10 @@ package database import ( "context" - "github.com/go-vela/server/tracing" "github.com/sirupsen/logrus" "github.com/urfave/cli/v2" + + "github.com/go-vela/server/tracing" ) const key = "database" diff --git a/database/context_test.go b/database/context_test.go index 303a7c986..80e26ea58 100644 --- a/database/context_test.go +++ b/database/context_test.go @@ -4,13 +4,14 @@ package database import ( "flag" - "github.com/go-vela/server/tracing" "reflect" "testing" "time" "github.com/gin-gonic/gin" "github.com/urfave/cli/v2" + + "github.com/go-vela/server/tracing" ) func TestDatabase_FromContext(t *testing.T) { diff --git a/database/database.go b/database/database.go index b197f77ca..757482962 100644 --- a/database/database.go +++ b/database/database.go @@ -7,6 +7,12 @@ import ( "fmt" "time" + "github.com/sirupsen/logrus" + "github.com/uptrace/opentelemetry-go-extra/otelgorm" + "gorm.io/driver/postgres" + "gorm.io/driver/sqlite" + "gorm.io/gorm" + "github.com/go-vela/server/database/build" "github.com/go-vela/server/database/dashboard" "github.com/go-vela/server/database/deployment" @@ -25,12 +31,6 @@ import ( "github.com/go-vela/server/database/worker" "github.com/go-vela/server/tracing" "github.com/go-vela/types/constants" - "github.com/sirupsen/logrus" - "github.com/uptrace/opentelemetry-go-extra/otelgorm" - - "gorm.io/driver/postgres" - "gorm.io/driver/sqlite" - "gorm.io/gorm" ) type ( diff --git a/database/database_test.go b/database/database_test.go index 2abbe49fb..6ea4fe92d 100644 --- a/database/database_test.go +++ b/database/database_test.go @@ -3,7 +3,6 @@ package database import ( - "github.com/go-vela/server/tracing" "testing" "time" @@ -12,6 +11,8 @@ import ( "gorm.io/driver/postgres" "gorm.io/driver/sqlite" "gorm.io/gorm" + + "github.com/go-vela/server/tracing" ) func TestDatabase_New(t *testing.T) { diff --git a/database/opts.go b/database/opts.go index 187c929a4..5158b8382 100644 --- a/database/opts.go +++ b/database/opts.go @@ -3,9 +3,8 @@ package database import ( - "time" - "context" + "time" "github.com/go-vela/server/tracing" ) diff --git a/scm/github/github.go b/scm/github/github.go index 7d748325e..ac84e3a73 100644 --- a/scm/github/github.go +++ b/scm/github/github.go @@ -10,10 +10,9 @@ import ( "github.com/google/go-github/v63/github" "github.com/sirupsen/logrus" - "golang.org/x/oauth2" - "go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace" "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" + "golang.org/x/oauth2" ) const ( diff --git a/scm/github/repo.go b/scm/github/repo.go index d4905e2dd..d19d6afc9 100644 --- a/scm/github/repo.go +++ b/scm/github/repo.go @@ -10,10 +10,9 @@ import ( "strings" "time" - "go.opentelemetry.io/otel/trace" - "github.com/google/go-github/v63/github" "github.com/sirupsen/logrus" + "go.opentelemetry.io/otel/trace" api "github.com/go-vela/server/api/types" "github.com/go-vela/types/constants" diff --git a/scm/github/webhook.go b/scm/github/webhook.go index c3e9845c8..92d7e0df6 100644 --- a/scm/github/webhook.go +++ b/scm/github/webhook.go @@ -101,7 +101,7 @@ func (c *client) VerifyWebhook(ctx context.Context, request *http.Request, r *ap // RedeliverWebhook redelivers webhooks from GitHub. func (c *client) RedeliverWebhook(ctx context.Context, u *api.User, r *api.Repo, h *library.Hook) error { // create GitHub OAuth client with user's token - //nolint:contextcheck // do not need to pass context in this instance + client := c.newClientToken(ctx, *u.Token) // capture the delivery ID of the hook using GitHub API diff --git a/something.json b/something.json new file mode 100644 index 000000000..e69de29bb diff --git a/tracing/config.go b/tracing/config.go index 436a15b7c..e61561c5d 100644 --- a/tracing/config.go +++ b/tracing/config.go @@ -4,7 +4,6 @@ package tracing import ( "github.com/urfave/cli/v2" - sdktrace "go.opentelemetry.io/otel/sdk/trace" ) diff --git a/tracing/tracer.go b/tracing/tracer.go index 1c4b5492a..f87229c2a 100644 --- a/tracing/tracer.go +++ b/tracing/tracer.go @@ -6,14 +6,13 @@ import ( "context" "github.com/urfave/cli/v2" - "go.opentelemetry.io/otel/sdk/resource" - semconv "go.opentelemetry.io/otel/semconv/v1.20.0" - "go.opentelemetry.io/otel" otlp "go.opentelemetry.io/otel/exporters/otlp/otlptrace" "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" "go.opentelemetry.io/otel/propagation" + "go.opentelemetry.io/otel/sdk/resource" sdktrace "go.opentelemetry.io/otel/sdk/trace" + semconv "go.opentelemetry.io/otel/semconv/v1.20.0" ) // initTracer returns the tracer provider supplied to the tracing config. From da924d597700323a7b1cfd67ea70db98da414569 Mon Sep 17 00:00:00 2001 From: davidvader Date: Tue, 13 Aug 2024 09:37:50 -0500 Subject: [PATCH 24/59] enhance: supply parent context to all db calls --- database/build/clean.go | 1 + database/build/count.go | 1 + database/build/count_deployment.go | 1 + database/build/count_org.go | 1 + database/build/count_repo.go | 1 + database/build/count_status.go | 1 + database/build/create.go | 5 ++++- database/build/delete.go | 1 + database/build/get.go | 1 + database/build/get_repo.go | 1 + database/build/index.go | 16 ++++++++++++---- database/build/last_repo.go | 1 + database/build/list.go | 1 + database/build/list_dashboard.go | 5 ++++- database/build/list_org.go | 1 + database/build/list_pending_running.go | 1 + database/build/list_pending_running_repo.go | 1 + database/build/list_repo.go | 1 + database/build/table.go | 8 ++++++-- database/build/update.go | 5 ++++- database/dashboard/create.go | 5 ++++- database/dashboard/delete.go | 1 + database/dashboard/get.go | 1 + database/dashboard/table.go | 8 ++++++-- database/dashboard/update.go | 5 ++++- database/deployment/count.go | 1 + database/deployment/count_repo.go | 1 + database/deployment/create.go | 5 ++++- database/deployment/delete.go | 1 + database/deployment/get.go | 2 ++ database/deployment/get_repo.go | 2 ++ database/deployment/index.go | 4 +++- database/deployment/list.go | 2 ++ database/deployment/list_repo.go | 2 ++ database/deployment/table.go | 8 ++++++-- database/deployment/update.go | 5 ++++- database/executable/clean.go | 8 ++++++-- database/executable/create.go | 1 + database/executable/pop.go | 3 +++ database/executable/table.go | 8 ++++++-- database/hook/count.go | 1 + database/hook/count_repo.go | 1 + database/hook/create.go | 5 ++++- database/hook/delete.go | 1 + database/hook/get.go | 1 + database/hook/get_repo.go | 1 + database/hook/get_webhook.go | 1 + database/hook/index.go | 4 +++- database/hook/last_repo.go | 1 + database/hook/list.go | 1 + database/hook/list_repo.go | 1 + database/hook/table.go | 8 ++++++-- database/hook/update.go | 5 ++++- database/jwk/create.go | 7 +++++-- database/jwk/get.go | 3 ++- database/jwk/list.go | 3 ++- database/jwk/rotate.go | 4 +++- database/jwk/table.go | 8 ++++++-- database/log/count.go | 1 + database/log/count_build.go | 1 + database/log/create.go | 1 + database/log/delete.go | 1 + database/log/get.go | 1 + database/log/get_service.go | 1 + database/log/get_step.go | 1 + database/log/index.go | 4 +++- database/log/list.go | 1 + database/log/list_build.go | 1 + database/log/table.go | 8 ++++++-- database/log/update.go | 1 + database/pipeline/count.go | 1 + database/pipeline/count_repo.go | 1 + database/pipeline/create.go | 5 ++++- database/pipeline/delete.go | 1 + database/pipeline/get.go | 1 + database/pipeline/get_repo.go | 1 + database/pipeline/index.go | 4 +++- database/pipeline/list.go | 1 + database/pipeline/list_repo.go | 1 + database/pipeline/table.go | 8 ++++++-- database/pipeline/update.go | 5 ++++- database/repo/count.go | 1 + database/repo/count_org.go | 1 + database/repo/count_user.go | 1 + database/repo/create.go | 5 ++++- database/repo/delete.go | 1 + database/repo/get.go | 1 + database/repo/get_org.go | 1 + database/repo/index.go | 4 +++- database/repo/list.go | 1 + database/repo/list_org.go | 3 +++ database/repo/list_user.go | 3 +++ database/repo/table.go | 8 ++++++-- database/repo/update.go | 5 ++++- database/schedule/count.go | 1 + database/schedule/count_active.go | 1 + database/schedule/count_repo.go | 1 + database/schedule/create.go | 5 ++++- database/schedule/delete.go | 1 + database/schedule/get.go | 1 + database/schedule/get_repo.go | 1 + database/schedule/index.go | 4 +++- database/schedule/list.go | 1 + database/schedule/list_active.go | 1 + database/schedule/list_repo.go | 1 + database/schedule/table.go | 8 ++++++-- database/schedule/update.go | 11 +++++++++-- database/secret/count.go | 1 + database/secret/count_org.go | 1 + database/secret/count_repo.go | 1 + database/secret/count_team.go | 2 ++ database/secret/create.go | 5 ++++- database/secret/delete.go | 1 + database/secret/get.go | 1 + database/secret/get_org.go | 1 + database/secret/get_repo.go | 1 + database/secret/get_team.go | 1 + database/secret/index.go | 12 +++++++++--- database/secret/list.go | 1 + database/secret/list_org.go | 1 + database/secret/list_repo.go | 1 + database/secret/list_team.go | 2 ++ database/secret/table.go | 8 ++++++-- database/secret/update.go | 5 ++++- database/service/clean.go | 1 + database/service/count.go | 1 + database/service/count_build.go | 1 + database/service/create.go | 5 ++++- database/service/delete.go | 1 + database/service/get.go | 1 + database/service/get_build.go | 1 + database/service/list.go | 1 + database/service/list_build.go | 1 + database/service/list_image.go | 1 + database/service/list_status.go | 1 + database/service/table.go | 8 ++++++-- database/service/update.go | 5 ++++- database/settings/create.go | 7 +++++-- database/settings/get.go | 1 + database/settings/table.go | 10 +++++++--- database/settings/update.go | 7 +++++-- database/step/clean.go | 1 + database/step/count.go | 1 + database/step/count_build.go | 1 + database/step/create.go | 5 ++++- database/step/delete.go | 1 + database/step/get.go | 1 + database/step/get_build.go | 1 + database/step/list.go | 1 + database/step/list_build.go | 1 + database/step/list_image.go | 1 + database/step/list_status.go | 1 + database/step/table.go | 8 ++++++-- database/step/update.go | 5 ++++- database/user/count.go | 1 + database/user/create.go | 5 ++++- database/user/delete.go | 1 + database/user/get.go | 1 + database/user/get_name.go | 1 + database/user/index.go | 4 +++- database/user/list.go | 1 + database/user/list_lite.go | 1 + database/user/table.go | 8 ++++++-- database/user/update.go | 5 ++++- database/worker/count.go | 1 + database/worker/create.go | 5 ++++- database/worker/delete.go | 1 + database/worker/get.go | 1 + database/worker/get_hostname.go | 1 + database/worker/index.go | 4 +++- database/worker/list.go | 4 +++- database/worker/table.go | 8 ++++++-- database/worker/update.go | 5 ++++- 173 files changed, 404 insertions(+), 86 deletions(-) diff --git a/database/build/clean.go b/database/build/clean.go index b7d876d16..81a461469 100644 --- a/database/build/clean.go +++ b/database/build/clean.go @@ -26,6 +26,7 @@ func (e *engine) CleanBuilds(ctx context.Context, msg string, before int64) (int // send query to the database result := e.client. + WithContext(ctx). Table(constants.TableBuild). Where("created < ?", before). Where("status = 'running' OR status = 'pending'"). diff --git a/database/build/count.go b/database/build/count.go index 3e86611d9..6b26c3f28 100644 --- a/database/build/count.go +++ b/database/build/count.go @@ -17,6 +17,7 @@ func (e *engine) CountBuilds(ctx context.Context) (int64, error) { // send query to the database and store result in variable err := e.client. + WithContext(ctx). Table(constants.TableBuild). Count(&b). Error diff --git a/database/build/count_deployment.go b/database/build/count_deployment.go index 46c8170fc..e00151cac 100644 --- a/database/build/count_deployment.go +++ b/database/build/count_deployment.go @@ -22,6 +22,7 @@ func (e *engine) CountBuildsForDeployment(ctx context.Context, d *library.Deploy // send query to the database and store result in variable err := e.client. + WithContext(ctx). Table(constants.TableBuild). Where("source = ?", d.GetURL()). Where(filters). diff --git a/database/build/count_org.go b/database/build/count_org.go index d9f4105ed..5a349c730 100644 --- a/database/build/count_org.go +++ b/database/build/count_org.go @@ -21,6 +21,7 @@ func (e *engine) CountBuildsForOrg(ctx context.Context, org string, filters map[ // send query to the database and store result in variable err := e.client. + WithContext(ctx). Table(constants.TableBuild). Joins("JOIN repos ON builds.repo_id = repos.id"). Where("repos.org = ?", org). diff --git a/database/build/count_repo.go b/database/build/count_repo.go index 298420caa..138b4d6d4 100644 --- a/database/build/count_repo.go +++ b/database/build/count_repo.go @@ -23,6 +23,7 @@ func (e *engine) CountBuildsForRepo(ctx context.Context, r *api.Repo, filters ma // send query to the database and store result in variable err := e.client. + WithContext(ctx). Table(constants.TableBuild). Where("repo_id = ?", r.GetID()). Where(filters). diff --git a/database/build/count_status.go b/database/build/count_status.go index 524d27d76..8fb401ffb 100644 --- a/database/build/count_status.go +++ b/database/build/count_status.go @@ -17,6 +17,7 @@ func (e *engine) CountBuildsForStatus(ctx context.Context, status string, filter // send query to the database and store result in variable err := e.client. + WithContext(ctx). Table(constants.TableBuild). Where("status = ?", status). Where(filters). diff --git a/database/build/create.go b/database/build/create.go index 3a3684102..7789434cb 100644 --- a/database/build/create.go +++ b/database/build/create.go @@ -30,7 +30,10 @@ func (e *engine) CreateBuild(ctx context.Context, b *api.Build) (*api.Build, err build = build.Crop() // send query to the database - err = e.client.Table(constants.TableBuild).Create(build).Error + err = e.client. + WithContext(ctx). + Table(constants.TableBuild). + Create(build).Error if err != nil { return nil, err } diff --git a/database/build/delete.go b/database/build/delete.go index 2bcb282e4..f03387a5f 100644 --- a/database/build/delete.go +++ b/database/build/delete.go @@ -22,6 +22,7 @@ func (e *engine) DeleteBuild(ctx context.Context, b *api.Build) error { // send query to the database return e.client. + WithContext(ctx). Table(constants.TableBuild). Delete(build). Error diff --git a/database/build/get.go b/database/build/get.go index caaf1fe0c..dc7f0e2de 100644 --- a/database/build/get.go +++ b/database/build/get.go @@ -19,6 +19,7 @@ func (e *engine) GetBuild(ctx context.Context, id int64) (*api.Build, error) { // send query to the database and store result in variable err := e.client. + WithContext(ctx). Table(constants.TableBuild). Preload("Repo"). Preload("Repo.Owner"). diff --git a/database/build/get_repo.go b/database/build/get_repo.go index b144cf432..8ac935881 100644 --- a/database/build/get_repo.go +++ b/database/build/get_repo.go @@ -25,6 +25,7 @@ func (e *engine) GetBuildForRepo(ctx context.Context, r *api.Repo, number int) ( // send query to the database and store result in variable err := e.client. + WithContext(ctx). Table(constants.TableBuild). Preload("Repo"). Preload("Repo.Owner"). diff --git a/database/build/index.go b/database/build/index.go index 446395cc9..f224ccacc 100644 --- a/database/build/index.go +++ b/database/build/index.go @@ -47,23 +47,31 @@ func (e *engine) CreateBuildIndexes(ctx context.Context) error { e.logger.Tracef("creating indexes for builds table") // create the created column index for the builds table - err := e.client.Exec(CreateCreatedIndex).Error + err := e.client. + WithContext(ctx). + Exec(CreateCreatedIndex).Error if err != nil { return err } // create the repo_id column index for the builds table - err = e.client.Exec(CreateRepoIDIndex).Error + err = e.client. + WithContext(ctx). + Exec(CreateRepoIDIndex).Error if err != nil { return err } // create the source column index for the builds table - err = e.client.Exec(CreateSourceIndex).Error + err = e.client. + WithContext(ctx). + Exec(CreateSourceIndex).Error if err != nil { return err } // create the status column index for the builds table - return e.client.Exec(CreateStatusIndex).Error + return e.client. + WithContext(ctx). + Exec(CreateStatusIndex).Error } diff --git a/database/build/last_repo.go b/database/build/last_repo.go index fe04af158..ccade3cc8 100644 --- a/database/build/last_repo.go +++ b/database/build/last_repo.go @@ -26,6 +26,7 @@ func (e *engine) LastBuildForRepo(ctx context.Context, r *api.Repo, branch strin // send query to the database and store result in variable err := e.client. + WithContext(ctx). Table(constants.TableBuild). Preload("Repo"). Preload("Repo.Owner"). diff --git a/database/build/list.go b/database/build/list.go index 9e3e45c8c..4074618c7 100644 --- a/database/build/list.go +++ b/database/build/list.go @@ -32,6 +32,7 @@ func (e *engine) ListBuilds(ctx context.Context) ([]*api.Build, error) { // send query to the database and store result in variable err = e.client. + WithContext(ctx). Preload("Repo"). Preload("Repo.Owner"). Table(constants.TableBuild). diff --git a/database/build/list_dashboard.go b/database/build/list_dashboard.go index 8ddc6fa57..7ba0c379a 100644 --- a/database/build/list_dashboard.go +++ b/database/build/list_dashboard.go @@ -23,7 +23,10 @@ func (e *engine) ListBuildsForDashboardRepo(ctx context.Context, r *api.Repo, br b := new([]types.Build) builds := []*api.Build{} - query := e.client.Table(constants.TableBuild).Where("repo_id = ?", r.GetID()) + query := e.client. + WithContext(ctx). + Table(constants.TableBuild). + Where("repo_id = ?", r.GetID()) if len(branches) > 0 { query = query.Where("branch IN (?)", branches) diff --git a/database/build/list_org.go b/database/build/list_org.go index cfaa1992d..d1c33a7a7 100644 --- a/database/build/list_org.go +++ b/database/build/list_org.go @@ -40,6 +40,7 @@ func (e *engine) ListBuildsForOrg(ctx context.Context, org string, filters map[s offset := perPage * (page - 1) err = e.client. + WithContext(ctx). Table(constants.TableBuild). Preload("Repo"). Preload("Repo.Owner"). diff --git a/database/build/list_pending_running.go b/database/build/list_pending_running.go index 858b9c08e..6a08dcbac 100644 --- a/database/build/list_pending_running.go +++ b/database/build/list_pending_running.go @@ -20,6 +20,7 @@ func (e *engine) ListPendingAndRunningBuilds(ctx context.Context, after string) // send query to the database and store result in variable err := e.client. + WithContext(ctx). Table(constants.TableBuild). Select("builds.created, builds.number, builds.status, repos.full_name"). InnerJoins("INNER JOIN repos ON builds.repo_id = repos.id"). diff --git a/database/build/list_pending_running_repo.go b/database/build/list_pending_running_repo.go index cc8984265..f4028a57a 100644 --- a/database/build/list_pending_running_repo.go +++ b/database/build/list_pending_running_repo.go @@ -20,6 +20,7 @@ func (e *engine) ListPendingAndRunningBuildsForRepo(ctx context.Context, repo *a // send query to the database and store result in variable err := e.client. + WithContext(ctx). Table(constants.TableBuild). Preload("Repo"). Preload("Repo.Owner"). diff --git a/database/build/list_repo.go b/database/build/list_repo.go index 2552c2ea3..880ea8e8c 100644 --- a/database/build/list_repo.go +++ b/database/build/list_repo.go @@ -41,6 +41,7 @@ func (e *engine) ListBuildsForRepo(ctx context.Context, r *api.Repo, filters map offset := perPage * (page - 1) err = e.client. + WithContext(ctx). Table(constants.TableBuild). Preload("Repo"). Preload("Repo.Owner"). diff --git a/database/build/table.go b/database/build/table.go index 8a7ae172c..2a7af528c 100644 --- a/database/build/table.go +++ b/database/build/table.go @@ -108,11 +108,15 @@ func (e *engine) CreateBuildTable(ctx context.Context, driver string) error { switch driver { case constants.DriverPostgres: // create the builds table for Postgres - return e.client.Exec(CreatePostgresTable).Error + return e.client. + WithContext(ctx). + Exec(CreatePostgresTable).Error case constants.DriverSqlite: fallthrough default: // create the builds table for Sqlite - return e.client.Exec(CreateSqliteTable).Error + return e.client. + WithContext(ctx). + Exec(CreateSqliteTable).Error } } diff --git a/database/build/update.go b/database/build/update.go index e8ff1f274..07b029093 100644 --- a/database/build/update.go +++ b/database/build/update.go @@ -30,7 +30,10 @@ func (e *engine) UpdateBuild(ctx context.Context, b *api.Build) (*api.Build, err build = build.Crop() // send query to the database - err = e.client.Table(constants.TableBuild).Save(build).Error + err = e.client. + WithContext(ctx). + Table(constants.TableBuild). + Save(build).Error if err != nil { return nil, err } diff --git a/database/dashboard/create.go b/database/dashboard/create.go index 1af56728a..5d49d574f 100644 --- a/database/dashboard/create.go +++ b/database/dashboard/create.go @@ -26,7 +26,10 @@ func (e *engine) CreateDashboard(ctx context.Context, d *api.Dashboard) (*api.Da } // send query to the database - result := e.client.Table(constants.TableDashboard).Create(dashboard) + result := e.client. + WithContext(ctx). + Table(constants.TableDashboard). + Create(dashboard) return dashboard.ToAPI(), result.Error } diff --git a/database/dashboard/delete.go b/database/dashboard/delete.go index 2f49cca9f..1e8c40434 100644 --- a/database/dashboard/delete.go +++ b/database/dashboard/delete.go @@ -22,6 +22,7 @@ func (e *engine) DeleteDashboard(ctx context.Context, d *api.Dashboard) error { // send query to the database return e.client. + WithContext(ctx). Table(constants.TableDashboard). Delete(dashboard). Error diff --git a/database/dashboard/get.go b/database/dashboard/get.go index 7c29c987f..d6d8f1f21 100644 --- a/database/dashboard/get.go +++ b/database/dashboard/get.go @@ -19,6 +19,7 @@ func (e *engine) GetDashboard(ctx context.Context, id string) (*api.Dashboard, e // send query to the database and store result in variable err := e.client. + WithContext(ctx). Table(constants.TableDashboard). Where("id = ?", id). Take(d). diff --git a/database/dashboard/table.go b/database/dashboard/table.go index 1cedb30c3..cef302382 100644 --- a/database/dashboard/table.go +++ b/database/dashboard/table.go @@ -50,11 +50,15 @@ func (e *engine) CreateDashboardTable(ctx context.Context, driver string) error switch driver { case constants.DriverPostgres: // create the dashboards table for Postgres - return e.client.Exec(CreatePostgresTable).Error + return e.client. + WithContext(ctx). + Exec(CreatePostgresTable).Error case constants.DriverSqlite: fallthrough default: // create the dashboards table for Sqlite - return e.client.Exec(CreateSqliteTable).Error + return e.client. + WithContext(ctx). + Exec(CreateSqliteTable).Error } } diff --git a/database/dashboard/update.go b/database/dashboard/update.go index e13387cb3..18089ff86 100644 --- a/database/dashboard/update.go +++ b/database/dashboard/update.go @@ -26,7 +26,10 @@ func (e *engine) UpdateDashboard(ctx context.Context, d *api.Dashboard) (*api.Da } // send query to the database - err = e.client.Table(constants.TableDashboard).Save(dashboard).Error + err = e.client. + WithContext(ctx). + Table(constants.TableDashboard). + Save(dashboard).Error if err != nil { return nil, err } diff --git a/database/deployment/count.go b/database/deployment/count.go index 26189cf10..f1c9e5b14 100644 --- a/database/deployment/count.go +++ b/database/deployment/count.go @@ -17,6 +17,7 @@ func (e *engine) CountDeployments(ctx context.Context) (int64, error) { // send query to the database and store result in variable err := e.client. + WithContext(ctx). Table(constants.TableDeployment). Count(&d). Error diff --git a/database/deployment/count_repo.go b/database/deployment/count_repo.go index 94ed86558..8a2afd6fc 100644 --- a/database/deployment/count_repo.go +++ b/database/deployment/count_repo.go @@ -23,6 +23,7 @@ func (e *engine) CountDeploymentsForRepo(ctx context.Context, r *api.Repo) (int6 // send query to the database and store result in variable err := e.client. + WithContext(ctx). Table(constants.TableDeployment). Where("repo_id = ?", r.GetID()). Count(&d). diff --git a/database/deployment/create.go b/database/deployment/create.go index c766596e8..923625bb6 100644 --- a/database/deployment/create.go +++ b/database/deployment/create.go @@ -27,7 +27,10 @@ func (e *engine) CreateDeployment(ctx context.Context, d *library.Deployment) (* return nil, err } - result := e.client.Table(constants.TableDeployment).Create(deployment) + result := e.client. + WithContext(ctx). + Table(constants.TableDeployment). + Create(deployment) // send query to the database return deployment.ToLibrary(d.Builds), result.Error diff --git a/database/deployment/delete.go b/database/deployment/delete.go index 606ef8262..cdb98fe31 100644 --- a/database/deployment/delete.go +++ b/database/deployment/delete.go @@ -23,6 +23,7 @@ func (e *engine) DeleteDeployment(ctx context.Context, d *library.Deployment) er // send query to the database return e.client. + WithContext(ctx). Table(constants.TableDeployment). Delete(deployment). Error diff --git a/database/deployment/get.go b/database/deployment/get.go index 214dcf57f..0086a03a2 100644 --- a/database/deployment/get.go +++ b/database/deployment/get.go @@ -20,6 +20,7 @@ func (e *engine) GetDeployment(ctx context.Context, id int64) (*library.Deployme // send query to the database and store result in variable err := e.client. + WithContext(ctx). Table(constants.TableDeployment). Where("id = ?", id). Take(d). @@ -40,6 +41,7 @@ func (e *engine) GetDeployment(ctx context.Context, id int64) (*library.Deployme // send query to the database and store result in variable err = e.client. + WithContext(ctx). Table(constants.TableBuild). Where("id = ?", bID). Take(b). diff --git a/database/deployment/get_repo.go b/database/deployment/get_repo.go index 1558b0428..f50ef3aba 100644 --- a/database/deployment/get_repo.go +++ b/database/deployment/get_repo.go @@ -27,6 +27,7 @@ func (e *engine) GetDeploymentForRepo(ctx context.Context, r *api.Repo, number i // send query to the database and store result in variable err := e.client. + WithContext(ctx). Table(constants.TableDeployment). Where("repo_id = ?", r.GetID()). Where("number = ?", number). @@ -48,6 +49,7 @@ func (e *engine) GetDeploymentForRepo(ctx context.Context, r *api.Repo, number i // send query to the database and store result in variable err = e.client. + WithContext(ctx). Table(constants.TableBuild). Where("id = ?", bID). Take(b). diff --git a/database/deployment/index.go b/database/deployment/index.go index f9b6a3419..c695313d0 100644 --- a/database/deployment/index.go +++ b/database/deployment/index.go @@ -20,5 +20,7 @@ func (e *engine) CreateDeploymentIndexes(ctx context.Context) error { e.logger.Tracef("creating indexes for deployments table") // create the repo_id column index for the deployments table - return e.client.Exec(CreateRepoIDIndex).Error + return e.client. + WithContext(ctx). + Exec(CreateRepoIDIndex).Error } diff --git a/database/deployment/list.go b/database/deployment/list.go index f395f668b..7f942f859 100644 --- a/database/deployment/list.go +++ b/database/deployment/list.go @@ -21,6 +21,7 @@ func (e *engine) ListDeployments(ctx context.Context) ([]*library.Deployment, er // send query to the database and store result in variable err := e.client. + WithContext(ctx). Table(constants.TableDeployment). Find(&d). Error @@ -45,6 +46,7 @@ func (e *engine) ListDeployments(ctx context.Context) ([]*library.Deployment, er // send query to the database and store result in variable err = e.client. + WithContext(ctx). Table(constants.TableBuild). Where("id = ?", bID). Take(b). diff --git a/database/deployment/list_repo.go b/database/deployment/list_repo.go index 8f9b40eb4..4be4e071b 100644 --- a/database/deployment/list_repo.go +++ b/database/deployment/list_repo.go @@ -30,6 +30,7 @@ func (e *engine) ListDeploymentsForRepo(ctx context.Context, r *api.Repo, page, // send query to the database and store result in variable err := e.client. + WithContext(ctx). Table(constants.TableDeployment). Where("repo_id = ?", r.GetID()). Order("number DESC"). @@ -58,6 +59,7 @@ func (e *engine) ListDeploymentsForRepo(ctx context.Context, r *api.Repo, page, // send query to the database and store result in variable err = e.client. + WithContext(ctx). Table(constants.TableBuild). Where("id = ?", bID). Take(b). diff --git a/database/deployment/table.go b/database/deployment/table.go index b519c8abf..9baef0284 100644 --- a/database/deployment/table.go +++ b/database/deployment/table.go @@ -62,11 +62,15 @@ func (e *engine) CreateDeploymentTable(ctx context.Context, driver string) error switch driver { case constants.DriverPostgres: // create the deployments table for Postgres - return e.client.Exec(CreatePostgresTable).Error + return e.client. + WithContext(ctx). + Exec(CreatePostgresTable).Error case constants.DriverSqlite: fallthrough default: // create the deployments table for Sqlite - return e.client.Exec(CreateSqliteTable).Error + return e.client. + WithContext(ctx). + Exec(CreateSqliteTable).Error } } diff --git a/database/deployment/update.go b/database/deployment/update.go index c2bbecff0..810450d79 100644 --- a/database/deployment/update.go +++ b/database/deployment/update.go @@ -27,7 +27,10 @@ func (e *engine) UpdateDeployment(ctx context.Context, d *library.Deployment) (* return nil, err } - result := e.client.Table(constants.TableDeployment).Save(deployment) + result := e.client. + WithContext(ctx). + Table(constants.TableDeployment). + Save(deployment) // send query to the database return deployment.ToLibrary(d.Builds), result.Error diff --git a/database/executable/clean.go b/database/executable/clean.go index b6bd2580a..10a15db41 100644 --- a/database/executable/clean.go +++ b/database/executable/clean.go @@ -33,10 +33,14 @@ func (e *engine) CleanBuildExecutables(ctx context.Context) (int64, error) { switch e.config.Driver { case constants.DriverPostgres: - res := e.client.Exec(CleanExecutablesPostgres) + res := e.client. + WithContext(ctx). + Exec(CleanExecutablesPostgres) return res.RowsAffected, res.Error default: - res := e.client.Exec(CleanExecutablesSqlite) + res := e.client. + WithContext(ctx). + Exec(CleanExecutablesSqlite) return res.RowsAffected, res.Error } } diff --git a/database/executable/create.go b/database/executable/create.go index b619da068..e231f01fb 100644 --- a/database/executable/create.go +++ b/database/executable/create.go @@ -50,6 +50,7 @@ func (e *engine) CreateBuildExecutable(ctx context.Context, b *library.BuildExec // send query to the database return e.client. + WithContext(ctx). Table(constants.TableBuildExecutable). Create(executable). Error diff --git a/database/executable/pop.go b/database/executable/pop.go index 62dfde193..9625ad984 100644 --- a/database/executable/pop.go +++ b/database/executable/pop.go @@ -25,6 +25,7 @@ func (e *engine) PopBuildExecutable(ctx context.Context, id int64) (*library.Bui case constants.DriverPostgres: // send query to the database and store result in variable err := e.client. + WithContext(ctx). Table(constants.TableBuildExecutable). Clauses(clause.Returning{}). Where("build_id = ?", id). @@ -37,6 +38,7 @@ func (e *engine) PopBuildExecutable(ctx context.Context, id int64) (*library.Bui case constants.DriverSqlite: // send query to the database and store result in variable err := e.client. + WithContext(ctx). Table(constants.TableBuildExecutable). Where("id = ?", id). Take(b). @@ -47,6 +49,7 @@ func (e *engine) PopBuildExecutable(ctx context.Context, id int64) (*library.Bui // send query to the database to delete result just got err = e.client. + WithContext(ctx). Table(constants.TableBuildExecutable). Delete(b). Error diff --git a/database/executable/table.go b/database/executable/table.go index 9a683f245..0a7a3c147 100644 --- a/database/executable/table.go +++ b/database/executable/table.go @@ -42,11 +42,15 @@ func (e *engine) CreateBuildExecutableTable(ctx context.Context, driver string) switch driver { case constants.DriverPostgres: // create the build_executables table for Postgres - return e.client.Exec(CreatePostgresTable).Error + return e.client. + WithContext(ctx). + Exec(CreatePostgresTable).Error case constants.DriverSqlite: fallthrough default: // create the build_executables table for Sqlite - return e.client.Exec(CreateSqliteTable).Error + return e.client. + WithContext(ctx). + Exec(CreateSqliteTable).Error } } diff --git a/database/hook/count.go b/database/hook/count.go index ce02e6fa0..9f3e3c264 100644 --- a/database/hook/count.go +++ b/database/hook/count.go @@ -17,6 +17,7 @@ func (e *engine) CountHooks(ctx context.Context) (int64, error) { // send query to the database and store result in variable err := e.client. + WithContext(ctx). Table(constants.TableHook). Count(&h). Error diff --git a/database/hook/count_repo.go b/database/hook/count_repo.go index aa794b5f1..456019fc1 100644 --- a/database/hook/count_repo.go +++ b/database/hook/count_repo.go @@ -23,6 +23,7 @@ func (e *engine) CountHooksForRepo(ctx context.Context, r *api.Repo) (int64, err // send query to the database and store result in variable err := e.client. + WithContext(ctx). Table(constants.TableHook). Where("repo_id = ?", r.GetID()). Count(&h). diff --git a/database/hook/create.go b/database/hook/create.go index c260eb2b5..badb47e82 100644 --- a/database/hook/create.go +++ b/database/hook/create.go @@ -31,7 +31,10 @@ func (e *engine) CreateHook(ctx context.Context, h *library.Hook) (*library.Hook return nil, err } - result := e.client.Table(constants.TableHook).Create(hook) + result := e.client. + WithContext(ctx). + Table(constants.TableHook). + Create(hook) // send query to the database return hook.ToLibrary(), result.Error diff --git a/database/hook/delete.go b/database/hook/delete.go index 0245477af..6eab73780 100644 --- a/database/hook/delete.go +++ b/database/hook/delete.go @@ -25,6 +25,7 @@ func (e *engine) DeleteHook(ctx context.Context, h *library.Hook) error { // send query to the database return e.client. + WithContext(ctx). Table(constants.TableHook). Delete(hook). Error diff --git a/database/hook/get.go b/database/hook/get.go index 82e421dda..afe9b920d 100644 --- a/database/hook/get.go +++ b/database/hook/get.go @@ -19,6 +19,7 @@ func (e *engine) GetHook(ctx context.Context, id int64) (*library.Hook, error) { // send query to the database and store result in variable err := e.client. + WithContext(ctx). Table(constants.TableHook). Where("id = ?", id). Take(h). diff --git a/database/hook/get_repo.go b/database/hook/get_repo.go index a88d593df..61afe28b2 100644 --- a/database/hook/get_repo.go +++ b/database/hook/get_repo.go @@ -26,6 +26,7 @@ func (e *engine) GetHookForRepo(ctx context.Context, r *api.Repo, number int) (* // send query to the database and store result in variable err := e.client. + WithContext(ctx). Table(constants.TableHook). Where("repo_id = ?", r.GetID()). Where("number = ?", number). diff --git a/database/hook/get_webhook.go b/database/hook/get_webhook.go index 919092fd7..47ea57f6c 100644 --- a/database/hook/get_webhook.go +++ b/database/hook/get_webhook.go @@ -19,6 +19,7 @@ func (e *engine) GetHookByWebhookID(ctx context.Context, webhookID int64) (*libr // send query to the database and store result in variable err := e.client. + WithContext(ctx). Table(constants.TableHook). Where("webhook_id = ?", webhookID). Take(h). diff --git a/database/hook/index.go b/database/hook/index.go index b86c9ac39..7f813ca8c 100644 --- a/database/hook/index.go +++ b/database/hook/index.go @@ -20,5 +20,7 @@ func (e *engine) CreateHookIndexes(ctx context.Context) error { e.logger.Tracef("creating indexes for hooks table") // create the repo_id column index for the hooks table - return e.client.Exec(CreateRepoIDIndex).Error + return e.client. + WithContext(ctx). + Exec(CreateRepoIDIndex).Error } diff --git a/database/hook/last_repo.go b/database/hook/last_repo.go index 710053ad8..776f9b6c0 100644 --- a/database/hook/last_repo.go +++ b/database/hook/last_repo.go @@ -27,6 +27,7 @@ func (e *engine) LastHookForRepo(ctx context.Context, r *api.Repo) (*library.Hoo // send query to the database and store result in variable err := e.client. + WithContext(ctx). Table(constants.TableHook). Where("repo_id = ?", r.GetID()). Order("number DESC"). diff --git a/database/hook/list.go b/database/hook/list.go index fcfba1e28..3957f57f2 100644 --- a/database/hook/list.go +++ b/database/hook/list.go @@ -32,6 +32,7 @@ func (e *engine) ListHooks(ctx context.Context) ([]*library.Hook, error) { // send query to the database and store result in variable err = e.client. + WithContext(ctx). Table(constants.TableHook). Find(&h). Error diff --git a/database/hook/list_repo.go b/database/hook/list_repo.go index 28e80c03e..7d4487f63 100644 --- a/database/hook/list_repo.go +++ b/database/hook/list_repo.go @@ -41,6 +41,7 @@ func (e *engine) ListHooksForRepo(ctx context.Context, r *api.Repo, page, perPag // send query to the database and store result in variable err = e.client. + WithContext(ctx). Table(constants.TableHook). Where("repo_id = ?", r.GetID()). Order("id DESC"). diff --git a/database/hook/table.go b/database/hook/table.go index 81ef7157f..a5d8d46ea 100644 --- a/database/hook/table.go +++ b/database/hook/table.go @@ -64,11 +64,15 @@ func (e *engine) CreateHookTable(ctx context.Context, driver string) error { switch driver { case constants.DriverPostgres: // create the hooks table for Postgres - return e.client.Exec(CreatePostgresTable).Error + return e.client. + WithContext(ctx). + Exec(CreatePostgresTable).Error case constants.DriverSqlite: fallthrough default: // create the hooks table for Sqlite - return e.client.Exec(CreateSqliteTable).Error + return e.client. + WithContext(ctx). + Exec(CreateSqliteTable).Error } } diff --git a/database/hook/update.go b/database/hook/update.go index 16222a6d1..bd6fe93ef 100644 --- a/database/hook/update.go +++ b/database/hook/update.go @@ -31,7 +31,10 @@ func (e *engine) UpdateHook(ctx context.Context, h *library.Hook) (*library.Hook return nil, err } - result := e.client.Table(constants.TableHook).Save(hook) + result := e.client. + WithContext(ctx). + Table(constants.TableHook). + Save(hook) // send query to the database return hook.ToLibrary(), result.Error diff --git a/database/jwk/create.go b/database/jwk/create.go index 0ef8330b9..684f49b97 100644 --- a/database/jwk/create.go +++ b/database/jwk/create.go @@ -14,7 +14,7 @@ import ( ) // CreateJWK creates a new JWK in the database. -func (e *engine) CreateJWK(_ context.Context, j jwk.RSAPublicKey) error { +func (e *engine) CreateJWK(ctx context.Context, j jwk.RSAPublicKey) error { e.logger.WithFields(logrus.Fields{ "jwk": j.KeyID(), }).Tracef("creating key %s", j.KeyID()) @@ -23,5 +23,8 @@ func (e *engine) CreateJWK(_ context.Context, j jwk.RSAPublicKey) error { key.Active = sql.NullBool{Bool: true, Valid: true} // send query to the database - return e.client.Table(constants.TableJWK).Create(key).Error + return e.client. + WithContext(ctx). + Table(constants.TableJWK). + Create(key).Error } diff --git a/database/jwk/get.go b/database/jwk/get.go index b324ae3cc..a99dbbb63 100644 --- a/database/jwk/get.go +++ b/database/jwk/get.go @@ -12,7 +12,7 @@ import ( ) // GetActiveJWK gets a JWK by UUID (kid) from the database if active. -func (e *engine) GetActiveJWK(_ context.Context, id string) (jwk.RSAPublicKey, error) { +func (e *engine) GetActiveJWK(ctx context.Context, id string) (jwk.RSAPublicKey, error) { e.logger.Tracef("getting JWK key %s", id) // variable to store query results @@ -20,6 +20,7 @@ func (e *engine) GetActiveJWK(_ context.Context, id string) (jwk.RSAPublicKey, e // send query to the database and store result in variable err := e.client. + WithContext(ctx). Table(constants.TableJWK). Where("id = ?", id). Where("active = ?", true). diff --git a/database/jwk/list.go b/database/jwk/list.go index 4c29a4c5c..8de0fcf63 100644 --- a/database/jwk/list.go +++ b/database/jwk/list.go @@ -12,7 +12,7 @@ import ( ) // ListJWKs gets a list of all configured JWKs from the database. -func (e *engine) ListJWKs(_ context.Context) (jwk.Set, error) { +func (e *engine) ListJWKs(ctx context.Context) (jwk.Set, error) { e.logger.Trace("listing all JWKs") k := new([]types.JWK) @@ -20,6 +20,7 @@ func (e *engine) ListJWKs(_ context.Context) (jwk.Set, error) { // send query to the database and store result in variable err := e.client. + WithContext(ctx). Table(constants.TableJWK). Find(&k). Error diff --git a/database/jwk/rotate.go b/database/jwk/rotate.go index 793584037..bcd4462ce 100644 --- a/database/jwk/rotate.go +++ b/database/jwk/rotate.go @@ -11,13 +11,14 @@ import ( ) // RotateKeys removes all inactive keys and sets active keys to inactive. -func (e *engine) RotateKeys(_ context.Context) error { +func (e *engine) RotateKeys(ctx context.Context) error { e.logger.Trace("rotating jwks") k := types.JWK{} // remove inactive keys err := e.client. + WithContext(ctx). Table(constants.TableJWK). Where("active = ?", false). Delete(&k). @@ -28,6 +29,7 @@ func (e *engine) RotateKeys(_ context.Context) error { // set active keys to inactive err = e.client. + WithContext(ctx). Table(constants.TableJWK). Where("active = ?", true). Update("active", sql.NullBool{Bool: false, Valid: true}). diff --git a/database/jwk/table.go b/database/jwk/table.go index b2a40c844..ee5895fcb 100644 --- a/database/jwk/table.go +++ b/database/jwk/table.go @@ -40,11 +40,15 @@ func (e *engine) CreateJWKTable(ctx context.Context, driver string) error { switch driver { case constants.DriverPostgres: // create the jwks table for Postgres - return e.client.Exec(CreatePostgresTable).Error + return e.client. + WithContext(ctx). + Exec(CreatePostgresTable).Error case constants.DriverSqlite: fallthrough default: // create the jwks table for Sqlite - return e.client.Exec(CreateSqliteTable).Error + return e.client. + WithContext(ctx). + Exec(CreateSqliteTable).Error } } diff --git a/database/log/count.go b/database/log/count.go index e3b38b316..d31c37f62 100644 --- a/database/log/count.go +++ b/database/log/count.go @@ -17,6 +17,7 @@ func (e *engine) CountLogs(ctx context.Context) (int64, error) { // send query to the database and store result in variable err := e.client. + WithContext(ctx). Table(constants.TableLog). Count(&l). Error diff --git a/database/log/count_build.go b/database/log/count_build.go index 524ff2960..36cd2984f 100644 --- a/database/log/count_build.go +++ b/database/log/count_build.go @@ -18,6 +18,7 @@ func (e *engine) CountLogsForBuild(ctx context.Context, b *api.Build) (int64, er // send query to the database and store result in variable err := e.client. + WithContext(ctx). Table(constants.TableLog). Where("build_id = ?", b.GetID()). Count(&l). diff --git a/database/log/create.go b/database/log/create.go index fdabab63c..4d0ee3428 100644 --- a/database/log/create.go +++ b/database/log/create.go @@ -50,6 +50,7 @@ func (e *engine) CreateLog(ctx context.Context, l *library.Log) error { // send query to the database return e.client. + WithContext(ctx). Table(constants.TableLog). Create(log). Error diff --git a/database/log/delete.go b/database/log/delete.go index 31f4ea31e..394321159 100644 --- a/database/log/delete.go +++ b/database/log/delete.go @@ -27,6 +27,7 @@ func (e *engine) DeleteLog(ctx context.Context, l *library.Log) error { // send query to the database return e.client. + WithContext(ctx). Table(constants.TableLog). Delete(log). Error diff --git a/database/log/get.go b/database/log/get.go index 2a825a6da..cf66dbfd7 100644 --- a/database/log/get.go +++ b/database/log/get.go @@ -19,6 +19,7 @@ func (e *engine) GetLog(ctx context.Context, id int64) (*library.Log, error) { // send query to the database and store result in variable err := e.client. + WithContext(ctx). Table(constants.TableLog). Where("id = ?", id). Take(l). diff --git a/database/log/get_service.go b/database/log/get_service.go index aac501801..ad4156a8d 100644 --- a/database/log/get_service.go +++ b/database/log/get_service.go @@ -20,6 +20,7 @@ func (e *engine) GetLogForService(ctx context.Context, s *library.Service) (*lib // send query to the database and store result in variable err := e.client. + WithContext(ctx). Table(constants.TableLog). Where("service_id = ?", s.GetID()). Take(l). diff --git a/database/log/get_step.go b/database/log/get_step.go index f2ca4a3a1..6d0084712 100644 --- a/database/log/get_step.go +++ b/database/log/get_step.go @@ -20,6 +20,7 @@ func (e *engine) GetLogForStep(ctx context.Context, s *library.Step) (*library.L // send query to the database and store result in variable err := e.client. + WithContext(ctx). Table(constants.TableLog). Where("step_id = ?", s.GetID()). Take(l). diff --git a/database/log/index.go b/database/log/index.go index 230a10d8b..1c9ff5736 100644 --- a/database/log/index.go +++ b/database/log/index.go @@ -20,5 +20,7 @@ func (e *engine) CreateLogIndexes(ctx context.Context) error { e.logger.Tracef("creating indexes for logs table") // create the build_id column index for the logs table - return e.client.Exec(CreateBuildIDIndex).Error + return e.client. + WithContext(ctx). + Exec(CreateBuildIDIndex).Error } diff --git a/database/log/list.go b/database/log/list.go index 367d8c15e..ed6ecdcb7 100644 --- a/database/log/list.go +++ b/database/log/list.go @@ -32,6 +32,7 @@ func (e *engine) ListLogs(ctx context.Context) ([]*library.Log, error) { // send query to the database and store result in variable err = e.client. + WithContext(ctx). Table(constants.TableLog). Find(&l). Error diff --git a/database/log/list_build.go b/database/log/list_build.go index 43c117916..f27798b78 100644 --- a/database/log/list_build.go +++ b/database/log/list_build.go @@ -36,6 +36,7 @@ func (e *engine) ListLogsForBuild(ctx context.Context, b *api.Build, page, perPa // send query to the database and store result in variable err = e.client. + WithContext(ctx). Table(constants.TableLog). Where("build_id = ?", b.GetID()). Order("service_id ASC NULLS LAST"). diff --git a/database/log/table.go b/database/log/table.go index 94ec398ff..fbf74cd28 100644 --- a/database/log/table.go +++ b/database/log/table.go @@ -50,11 +50,15 @@ func (e *engine) CreateLogTable(ctx context.Context, driver string) error { switch driver { case constants.DriverPostgres: // create the logs table for Postgres - return e.client.Exec(CreatePostgresTable).Error + return e.client. + WithContext(ctx). + Exec(CreatePostgresTable).Error case constants.DriverSqlite: fallthrough default: // create the logs table for Sqlite - return e.client.Exec(CreateSqliteTable).Error + return e.client. + WithContext(ctx). + Exec(CreateSqliteTable).Error } } diff --git a/database/log/update.go b/database/log/update.go index f7a148aae..ae9819e4a 100644 --- a/database/log/update.go +++ b/database/log/update.go @@ -50,6 +50,7 @@ func (e *engine) UpdateLog(ctx context.Context, l *library.Log) error { // send query to the database return e.client. + WithContext(ctx). Table(constants.TableLog). Save(log). Error diff --git a/database/pipeline/count.go b/database/pipeline/count.go index 3252f5edd..818982b52 100644 --- a/database/pipeline/count.go +++ b/database/pipeline/count.go @@ -17,6 +17,7 @@ func (e *engine) CountPipelines(ctx context.Context) (int64, error) { // send query to the database and store result in variable err := e.client. + WithContext(ctx). Table(constants.TablePipeline). Count(&p). Error diff --git a/database/pipeline/count_repo.go b/database/pipeline/count_repo.go index 6adf46984..147ae552b 100644 --- a/database/pipeline/count_repo.go +++ b/database/pipeline/count_repo.go @@ -23,6 +23,7 @@ func (e *engine) CountPipelinesForRepo(ctx context.Context, r *api.Repo) (int64, // send query to the database and store result in variable err := e.client. + WithContext(ctx). Table(constants.TablePipeline). Where("repo_id = ?", r.GetID()). Count(&p). diff --git a/database/pipeline/create.go b/database/pipeline/create.go index 4ee770b52..e2910e6a1 100644 --- a/database/pipeline/create.go +++ b/database/pipeline/create.go @@ -40,7 +40,10 @@ func (e *engine) CreatePipeline(ctx context.Context, p *library.Pipeline) (*libr } // send query to the database - err = e.client.Table(constants.TablePipeline).Create(pipeline).Error + err = e.client. + WithContext(ctx). + Table(constants.TablePipeline). + Create(pipeline).Error if err != nil { return nil, err } diff --git a/database/pipeline/delete.go b/database/pipeline/delete.go index b35cdd7d3..ff49a8406 100644 --- a/database/pipeline/delete.go +++ b/database/pipeline/delete.go @@ -25,6 +25,7 @@ func (e *engine) DeletePipeline(ctx context.Context, p *library.Pipeline) error // send query to the database return e.client. + WithContext(ctx). Table(constants.TablePipeline). Delete(pipeline). Error diff --git a/database/pipeline/get.go b/database/pipeline/get.go index 8e1c54b13..e685a9178 100644 --- a/database/pipeline/get.go +++ b/database/pipeline/get.go @@ -19,6 +19,7 @@ func (e *engine) GetPipeline(ctx context.Context, id int64) (*library.Pipeline, // send query to the database and store result in variable err := e.client. + WithContext(ctx). Table(constants.TablePipeline). Where("id = ?", id). Take(p). diff --git a/database/pipeline/get_repo.go b/database/pipeline/get_repo.go index 0fc493003..5724dd0a7 100644 --- a/database/pipeline/get_repo.go +++ b/database/pipeline/get_repo.go @@ -26,6 +26,7 @@ func (e *engine) GetPipelineForRepo(ctx context.Context, commit string, r *api.R // send query to the database and store result in variable err := e.client. + WithContext(ctx). Table(constants.TablePipeline). Where("repo_id = ?", r.GetID()). Where("\"commit\" = ?", commit). diff --git a/database/pipeline/index.go b/database/pipeline/index.go index f49ecb58a..4f0095f70 100644 --- a/database/pipeline/index.go +++ b/database/pipeline/index.go @@ -20,5 +20,7 @@ func (e *engine) CreatePipelineIndexes(ctx context.Context) error { e.logger.Tracef("creating indexes for pipelines table in the database") // create the repo_id column index for the pipelines table - return e.client.Exec(CreateRepoIDIndex).Error + return e.client. + WithContext(ctx). + Exec(CreateRepoIDIndex).Error } diff --git a/database/pipeline/list.go b/database/pipeline/list.go index fd9e077e0..12fac0bf1 100644 --- a/database/pipeline/list.go +++ b/database/pipeline/list.go @@ -32,6 +32,7 @@ func (e *engine) ListPipelines(ctx context.Context) ([]*library.Pipeline, error) // send query to the database and store result in variable err = e.client. + WithContext(ctx). Table(constants.TablePipeline). Find(&p). Error diff --git a/database/pipeline/list_repo.go b/database/pipeline/list_repo.go index 4f855c746..22e9032cb 100644 --- a/database/pipeline/list_repo.go +++ b/database/pipeline/list_repo.go @@ -42,6 +42,7 @@ func (e *engine) ListPipelinesForRepo(ctx context.Context, r *api.Repo, page, pe offset := perPage * (page - 1) err = e.client. + WithContext(ctx). Table(constants.TablePipeline). Where("repo_id = ?", r.GetID()). Limit(perPage). diff --git a/database/pipeline/table.go b/database/pipeline/table.go index eb399772c..919b1b652 100644 --- a/database/pipeline/table.go +++ b/database/pipeline/table.go @@ -66,11 +66,15 @@ func (e *engine) CreatePipelineTable(ctx context.Context, driver string) error { switch driver { case constants.DriverPostgres: // create the pipelines table for Postgres - return e.client.Exec(CreatePostgresTable).Error + return e.client. + WithContext(ctx). + Exec(CreatePostgresTable).Error case constants.DriverSqlite: fallthrough default: // create the pipelines table for Sqlite - return e.client.Exec(CreateSqliteTable).Error + return e.client. + WithContext(ctx). + Exec(CreateSqliteTable).Error } } diff --git a/database/pipeline/update.go b/database/pipeline/update.go index 37353d853..93cff9351 100644 --- a/database/pipeline/update.go +++ b/database/pipeline/update.go @@ -40,7 +40,10 @@ func (e *engine) UpdatePipeline(ctx context.Context, p *library.Pipeline) (*libr } // send query to the database - err = e.client.Table(constants.TablePipeline).Save(pipeline).Error + err = e.client. + WithContext(ctx). + Table(constants.TablePipeline). + Save(pipeline).Error if err != nil { return nil, err } diff --git a/database/repo/count.go b/database/repo/count.go index 165bf0f0b..6db509b06 100644 --- a/database/repo/count.go +++ b/database/repo/count.go @@ -17,6 +17,7 @@ func (e *engine) CountRepos(ctx context.Context) (int64, error) { // send query to the database and store result in variable err := e.client. + WithContext(ctx). Table(constants.TableRepo). Count(&r). Error diff --git a/database/repo/count_org.go b/database/repo/count_org.go index 6e0ce6cb4..915a0d9a7 100644 --- a/database/repo/count_org.go +++ b/database/repo/count_org.go @@ -21,6 +21,7 @@ func (e *engine) CountReposForOrg(ctx context.Context, org string, filters map[s // send query to the database and store result in variable err := e.client. + WithContext(ctx). Table(constants.TableRepo). Where("org = ?", org). Where(filters). diff --git a/database/repo/count_user.go b/database/repo/count_user.go index dfc002d9b..57f3ecb3c 100644 --- a/database/repo/count_user.go +++ b/database/repo/count_user.go @@ -22,6 +22,7 @@ func (e *engine) CountReposForUser(ctx context.Context, u *api.User, filters map // send query to the database and store result in variable err := e.client. + WithContext(ctx). Table(constants.TableRepo). Where("user_id = ?", u.GetID()). Where(filters). diff --git a/database/repo/create.go b/database/repo/create.go index 166362de9..7500b4101 100644 --- a/database/repo/create.go +++ b/database/repo/create.go @@ -37,7 +37,10 @@ func (e *engine) CreateRepo(ctx context.Context, r *api.Repo) (*api.Repo, error) } // send query to the database - err = e.client.Table(constants.TableRepo).Create(repo).Error + err = e.client. + WithContext(ctx). + Table(constants.TableRepo). + Create(repo).Error if err != nil { return nil, err } diff --git a/database/repo/delete.go b/database/repo/delete.go index a12e2ec91..cfd6dffa3 100644 --- a/database/repo/delete.go +++ b/database/repo/delete.go @@ -24,6 +24,7 @@ func (e *engine) DeleteRepo(ctx context.Context, r *api.Repo) error { // send query to the database return e.client. + WithContext(ctx). Table(constants.TableRepo). Delete(repo). Error diff --git a/database/repo/get.go b/database/repo/get.go index d29ba16c8..15a301290 100644 --- a/database/repo/get.go +++ b/database/repo/get.go @@ -19,6 +19,7 @@ func (e *engine) GetRepo(ctx context.Context, id int64) (*api.Repo, error) { // send query to the database and store result in variable err := e.client. + WithContext(ctx). Table(constants.TableRepo). Preload("Owner"). Where("id = ?", id). diff --git a/database/repo/get_org.go b/database/repo/get_org.go index e66b97b37..38b32d4ca 100644 --- a/database/repo/get_org.go +++ b/database/repo/get_org.go @@ -24,6 +24,7 @@ func (e *engine) GetRepoForOrg(ctx context.Context, org, name string) (*api.Repo // send query to the database and store result in variable err := e.client. + WithContext(ctx). Table(constants.TableRepo). Preload("Owner"). Where("org = ?", org). diff --git a/database/repo/index.go b/database/repo/index.go index 9c97ad962..e4b236064 100644 --- a/database/repo/index.go +++ b/database/repo/index.go @@ -20,5 +20,7 @@ func (e *engine) CreateRepoIndexes(ctx context.Context) error { e.logger.Tracef("creating indexes for repos table") // create the org and name columns index for the repos table - return e.client.Exec(CreateOrgNameIndex).Error + return e.client. + WithContext(ctx). + Exec(CreateOrgNameIndex).Error } diff --git a/database/repo/list.go b/database/repo/list.go index abe27a434..980044d97 100644 --- a/database/repo/list.go +++ b/database/repo/list.go @@ -32,6 +32,7 @@ func (e *engine) ListRepos(ctx context.Context) ([]*api.Repo, error) { // send query to the database and store result in variable err = e.client. + WithContext(ctx). Table(constants.TableRepo). Preload("Owner"). Find(&r). diff --git a/database/repo/list_org.go b/database/repo/list_org.go index 016044744..1b39feea8 100644 --- a/database/repo/list_org.go +++ b/database/repo/list_org.go @@ -42,6 +42,7 @@ func (e *engine) ListReposForOrg(ctx context.Context, org, sortBy string, filter switch sortBy { case "latest": query := e.client. + WithContext(ctx). Table(constants.TableBuild). Select("repos.id, MAX(builds.created) AS latest_build"). Joins("INNER JOIN repos repos ON builds.repo_id = repos.id"). @@ -49,6 +50,7 @@ func (e *engine) ListReposForOrg(ctx context.Context, org, sortBy string, filter Group("repos.id") err = e.client. + WithContext(ctx). Table(constants.TableRepo). Preload("Owner"). Select("repos.*"). @@ -65,6 +67,7 @@ func (e *engine) ListReposForOrg(ctx context.Context, org, sortBy string, filter fallthrough default: err = e.client. + WithContext(ctx). Table(constants.TableRepo). Preload("Owner"). Where("org = ?", org). diff --git a/database/repo/list_user.go b/database/repo/list_user.go index f50374de1..997fe15f4 100644 --- a/database/repo/list_user.go +++ b/database/repo/list_user.go @@ -42,6 +42,7 @@ func (e *engine) ListReposForUser(ctx context.Context, u *api.User, sortBy strin switch sortBy { case "latest": query := e.client. + WithContext(ctx). Table(constants.TableBuild). Select("repos.id, MAX(builds.created) AS latest_build"). Joins("INNER JOIN repos repos ON builds.repo_id = repos.id"). @@ -49,6 +50,7 @@ func (e *engine) ListReposForUser(ctx context.Context, u *api.User, sortBy strin Group("repos.id") err = e.client. + WithContext(ctx). Table(constants.TableRepo). Preload("Owner"). Select("repos.*"). @@ -65,6 +67,7 @@ func (e *engine) ListReposForUser(ctx context.Context, u *api.User, sortBy strin fallthrough default: err = e.client. + WithContext(ctx). Table(constants.TableRepo). Preload("Owner"). Where("user_id = ?", u.GetID()). diff --git a/database/repo/table.go b/database/repo/table.go index 65bd21fb2..0b16bae94 100644 --- a/database/repo/table.go +++ b/database/repo/table.go @@ -78,11 +78,15 @@ func (e *engine) CreateRepoTable(ctx context.Context, driver string) error { switch driver { case constants.DriverPostgres: // create the repos table for Postgres - return e.client.Exec(CreatePostgresTable).Error + return e.client. + WithContext(ctx). + Exec(CreatePostgresTable).Error case constants.DriverSqlite: fallthrough default: // create the repos table for Sqlite - return e.client.Exec(CreateSqliteTable).Error + return e.client. + WithContext(ctx). + Exec(CreateSqliteTable).Error } } diff --git a/database/repo/update.go b/database/repo/update.go index 9a11b8010..05af67470 100644 --- a/database/repo/update.go +++ b/database/repo/update.go @@ -37,7 +37,10 @@ func (e *engine) UpdateRepo(ctx context.Context, r *api.Repo) (*api.Repo, error) } // send query to the database - err = e.client.Table(constants.TableRepo).Save(repo).Error + err = e.client. + WithContext(ctx). + Table(constants.TableRepo). + Save(repo).Error if err != nil { return nil, err } diff --git a/database/schedule/count.go b/database/schedule/count.go index 8da93e69f..e199e8ed0 100644 --- a/database/schedule/count.go +++ b/database/schedule/count.go @@ -17,6 +17,7 @@ func (e *engine) CountSchedules(ctx context.Context) (int64, error) { // send query to the database and store result in variable err := e.client. + WithContext(ctx). Table(constants.TableSchedule). Count(&s). Error diff --git a/database/schedule/count_active.go b/database/schedule/count_active.go index be2dd9b5e..5aa6e7209 100644 --- a/database/schedule/count_active.go +++ b/database/schedule/count_active.go @@ -17,6 +17,7 @@ func (e *engine) CountActiveSchedules(ctx context.Context) (int64, error) { // send query to the database and store result in variable err := e.client. + WithContext(ctx). Table(constants.TableSchedule). Where("active = ?", true). Count(&s). diff --git a/database/schedule/count_repo.go b/database/schedule/count_repo.go index 724ce19b3..41246b800 100644 --- a/database/schedule/count_repo.go +++ b/database/schedule/count_repo.go @@ -23,6 +23,7 @@ func (e *engine) CountSchedulesForRepo(ctx context.Context, r *api.Repo) (int64, // send query to the database and store result in variable err := e.client. + WithContext(ctx). Table(constants.TableSchedule). Where("repo_id = ?", r.GetID()). Count(&s). diff --git a/database/schedule/create.go b/database/schedule/create.go index 9b349a7d1..891237c26 100644 --- a/database/schedule/create.go +++ b/database/schedule/create.go @@ -28,7 +28,10 @@ func (e *engine) CreateSchedule(ctx context.Context, s *api.Schedule) (*api.Sche } // send query to the database - err = e.client.Table(constants.TableSchedule).Create(schedule).Error + err = e.client. + WithContext(ctx). + Table(constants.TableSchedule). + Create(schedule).Error if err != nil { return nil, err } diff --git a/database/schedule/delete.go b/database/schedule/delete.go index bc63c2f10..8f28c1173 100644 --- a/database/schedule/delete.go +++ b/database/schedule/delete.go @@ -23,6 +23,7 @@ func (e *engine) DeleteSchedule(ctx context.Context, s *api.Schedule) error { // send query to the database return e.client. + WithContext(ctx). Table(constants.TableSchedule). Delete(schedule). Error diff --git a/database/schedule/get.go b/database/schedule/get.go index 1bcdc37cb..887d4e002 100644 --- a/database/schedule/get.go +++ b/database/schedule/get.go @@ -19,6 +19,7 @@ func (e *engine) GetSchedule(ctx context.Context, id int64) (*api.Schedule, erro // send query to the database and store result in variable err := e.client. + WithContext(ctx). Table(constants.TableSchedule). Preload("Repo"). Preload("Repo.Owner"). diff --git a/database/schedule/get_repo.go b/database/schedule/get_repo.go index e940694a4..57c9f485a 100644 --- a/database/schedule/get_repo.go +++ b/database/schedule/get_repo.go @@ -25,6 +25,7 @@ func (e *engine) GetScheduleForRepo(ctx context.Context, r *api.Repo, name strin // send query to the database and store result in variable err := e.client. + WithContext(ctx). Table(constants.TableSchedule). Preload("Repo"). Preload("Repo.Owner"). diff --git a/database/schedule/index.go b/database/schedule/index.go index 11834bf4f..b02543b93 100644 --- a/database/schedule/index.go +++ b/database/schedule/index.go @@ -20,5 +20,7 @@ func (e *engine) CreateScheduleIndexes(ctx context.Context) error { e.logger.Tracef("creating indexes for schedules table in the database") // create the repo_id column index for the schedules table - return e.client.Exec(CreateRepoIDIndex).Error + return e.client. + WithContext(ctx). + Exec(CreateRepoIDIndex).Error } diff --git a/database/schedule/list.go b/database/schedule/list.go index 4282f554a..44f548352 100644 --- a/database/schedule/list.go +++ b/database/schedule/list.go @@ -32,6 +32,7 @@ func (e *engine) ListSchedules(ctx context.Context) ([]*api.Schedule, error) { // send query to the database and store result in variable err = e.client. + WithContext(ctx). Table(constants.TableSchedule). Preload("Repo"). Preload("Repo.Owner"). diff --git a/database/schedule/list_active.go b/database/schedule/list_active.go index 84e575b1a..8952de929 100644 --- a/database/schedule/list_active.go +++ b/database/schedule/list_active.go @@ -32,6 +32,7 @@ func (e *engine) ListActiveSchedules(ctx context.Context) ([]*api.Schedule, erro // send query to the database and store result in variable err = e.client. + WithContext(ctx). Table(constants.TableSchedule). Preload("Repo"). Preload("Repo.Owner"). diff --git a/database/schedule/list_repo.go b/database/schedule/list_repo.go index 9ec086589..38e0d1b3e 100644 --- a/database/schedule/list_repo.go +++ b/database/schedule/list_repo.go @@ -40,6 +40,7 @@ func (e *engine) ListSchedulesForRepo(ctx context.Context, r *api.Repo, page, pe // send query to the database and store result in variable err = e.client. + WithContext(ctx). Table(constants.TableSchedule). Preload("Repo"). Preload("Repo.Owner"). diff --git a/database/schedule/table.go b/database/schedule/table.go index 82ae8fdaf..86eda10bc 100644 --- a/database/schedule/table.go +++ b/database/schedule/table.go @@ -60,11 +60,15 @@ func (e *engine) CreateScheduleTable(ctx context.Context, driver string) error { switch driver { case constants.DriverPostgres: // create the schedules table for Postgres - return e.client.Exec(CreatePostgresTable).Error + return e.client. + WithContext(ctx). + Exec(CreatePostgresTable).Error case constants.DriverSqlite: fallthrough default: // create the schedules table for Sqlite - return e.client.Exec(CreateSqliteTable).Error + return e.client. + WithContext(ctx). + Exec(CreateSqliteTable).Error } } diff --git a/database/schedule/update.go b/database/schedule/update.go index 7dcc1256d..d2f9cbf33 100644 --- a/database/schedule/update.go +++ b/database/schedule/update.go @@ -32,9 +32,16 @@ func (e *engine) UpdateSchedule(ctx context.Context, s *api.Schedule, fields boo // we do this because Gorm will automatically set `updated_at` with the Save function // and the `updated_at` field should reflect the last time a user updated the record, rather than the scheduler if fields { - err = e.client.Table(constants.TableSchedule).Save(schedule).Error + err = e.client. + WithContext(ctx). + Table(constants.TableSchedule). + Save(schedule).Error } else { - err = e.client.Table(constants.TableSchedule).Model(schedule).UpdateColumn("scheduled_at", s.GetScheduledAt()).Error + err = e.client. + WithContext(ctx). + Table(constants.TableSchedule). + Model(schedule). + UpdateColumn("scheduled_at", s.GetScheduledAt()).Error } if err != nil { diff --git a/database/secret/count.go b/database/secret/count.go index bf91525fa..2b9b36ad0 100644 --- a/database/secret/count.go +++ b/database/secret/count.go @@ -17,6 +17,7 @@ func (e *engine) CountSecrets(ctx context.Context) (int64, error) { // send query to the database and store result in variable err := e.client. + WithContext(ctx). Table(constants.TableSecret). Count(&s). Error diff --git a/database/secret/count_org.go b/database/secret/count_org.go index 9b4a8c263..85fb17624 100644 --- a/database/secret/count_org.go +++ b/database/secret/count_org.go @@ -22,6 +22,7 @@ func (e *engine) CountSecretsForOrg(ctx context.Context, org string, filters map // send query to the database and store result in variable err := e.client. + WithContext(ctx). Table(constants.TableSecret). Where("type = ?", constants.SecretOrg). Where("org = ?", org). diff --git a/database/secret/count_repo.go b/database/secret/count_repo.go index b8edddcb0..6430f7fe9 100644 --- a/database/secret/count_repo.go +++ b/database/secret/count_repo.go @@ -24,6 +24,7 @@ func (e *engine) CountSecretsForRepo(ctx context.Context, r *api.Repo, filters m // send query to the database and store result in variable err := e.client. + WithContext(ctx). Table(constants.TableSecret). Where("type = ?", constants.SecretRepo). Where("org = ?", r.GetOrg()). diff --git a/database/secret/count_team.go b/database/secret/count_team.go index 1b5fc8361..af158dcbc 100644 --- a/database/secret/count_team.go +++ b/database/secret/count_team.go @@ -24,6 +24,7 @@ func (e *engine) CountSecretsForTeam(ctx context.Context, org, team string, filt // send query to the database and store result in variable err := e.client. + WithContext(ctx). Table(constants.TableSecret). Where("type = ?", constants.SecretShared). Where("org = ?", org). @@ -56,6 +57,7 @@ func (e *engine) CountSecretsForTeams(ctx context.Context, org string, teams []s // send query to the database and store result in variable err := e.client. + WithContext(ctx). Table(constants.TableSecret). Where("type = ?", constants.SecretShared). Where("org = ?", org). diff --git a/database/secret/create.go b/database/secret/create.go index be409d304..163e72776 100644 --- a/database/secret/create.go +++ b/database/secret/create.go @@ -61,7 +61,10 @@ func (e *engine) CreateSecret(ctx context.Context, s *library.Secret) (*library. } // create secret record - result := e.client.Table(constants.TableSecret).Create(secret.Nullify()) + result := e.client. + WithContext(ctx). + Table(constants.TableSecret). + Create(secret.Nullify()) if result.Error != nil { return nil, result.Error diff --git a/database/secret/delete.go b/database/secret/delete.go index bc3cba36f..aa87ba1ea 100644 --- a/database/secret/delete.go +++ b/database/secret/delete.go @@ -41,6 +41,7 @@ func (e *engine) DeleteSecret(ctx context.Context, s *library.Secret) error { // send query to the database return e.client. + WithContext(ctx). Table(constants.TableSecret). Delete(secret). Error diff --git a/database/secret/get.go b/database/secret/get.go index 6f5bc7bf9..4f4d04199 100644 --- a/database/secret/get.go +++ b/database/secret/get.go @@ -19,6 +19,7 @@ func (e *engine) GetSecret(ctx context.Context, id int64) (*library.Secret, erro // send query to the database and store result in variable err := e.client. + WithContext(ctx). Table(constants.TableSecret). Where("id = ?", id). Take(s). diff --git a/database/secret/get_org.go b/database/secret/get_org.go index 7d1766c88..473d5a1a6 100644 --- a/database/secret/get_org.go +++ b/database/secret/get_org.go @@ -25,6 +25,7 @@ func (e *engine) GetSecretForOrg(ctx context.Context, org, name string) (*librar // send query to the database and store result in variable err := e.client. + WithContext(ctx). Table(constants.TableSecret). Where("type = ?", constants.SecretOrg). Where("org = ?", org). diff --git a/database/secret/get_repo.go b/database/secret/get_repo.go index 690823c08..27a5f3cfb 100644 --- a/database/secret/get_repo.go +++ b/database/secret/get_repo.go @@ -27,6 +27,7 @@ func (e *engine) GetSecretForRepo(ctx context.Context, name string, r *api.Repo) // send query to the database and store result in variable err := e.client. + WithContext(ctx). Table(constants.TableSecret). Where("type = ?", constants.SecretRepo). Where("org = ?", r.GetOrg()). diff --git a/database/secret/get_team.go b/database/secret/get_team.go index e03845947..ee82409c6 100644 --- a/database/secret/get_team.go +++ b/database/secret/get_team.go @@ -26,6 +26,7 @@ func (e *engine) GetSecretForTeam(ctx context.Context, org, team, name string) ( // send query to the database and store result in variable err := e.client. + WithContext(ctx). Table(constants.TableSecret). Where("type = ?", constants.SecretShared). Where("org = ?", org). diff --git a/database/secret/index.go b/database/secret/index.go index e045031bb..786cad40a 100644 --- a/database/secret/index.go +++ b/database/secret/index.go @@ -36,17 +36,23 @@ func (e *engine) CreateSecretIndexes(ctx context.Context) error { e.logger.Tracef("creating indexes for secrets table") // create the type, org and repo columns index for the secrets table - err := e.client.Exec(CreateTypeOrgRepo).Error + err := e.client. + WithContext(ctx). + Exec(CreateTypeOrgRepo).Error if err != nil { return err } // create the type, org and team columns index for the secrets table - err = e.client.Exec(CreateTypeOrgTeam).Error + err = e.client. + WithContext(ctx). + Exec(CreateTypeOrgTeam).Error if err != nil { return err } // create the type and org columns index for the secrets table - return e.client.Exec(CreateTypeOrg).Error + return e.client. + WithContext(ctx). + Exec(CreateTypeOrg).Error } diff --git a/database/secret/list.go b/database/secret/list.go index a61e2088f..fe2031f89 100644 --- a/database/secret/list.go +++ b/database/secret/list.go @@ -32,6 +32,7 @@ func (e *engine) ListSecrets(ctx context.Context) ([]*library.Secret, error) { // send query to the database and store result in variable err = e.client. + WithContext(ctx). Table(constants.TableSecret). Find(&s). Error diff --git a/database/secret/list_org.go b/database/secret/list_org.go index d8c7ced5a..c75fa8f8d 100644 --- a/database/secret/list_org.go +++ b/database/secret/list_org.go @@ -42,6 +42,7 @@ func (e *engine) ListSecretsForOrg(ctx context.Context, org string, filters map[ // send query to the database and store result in variable err = e.client. + WithContext(ctx). Table(constants.TableSecret). Where("type = ?", constants.SecretOrg). Where("org = ?", org). diff --git a/database/secret/list_repo.go b/database/secret/list_repo.go index ef2fe546c..593139869 100644 --- a/database/secret/list_repo.go +++ b/database/secret/list_repo.go @@ -44,6 +44,7 @@ func (e *engine) ListSecretsForRepo(ctx context.Context, r *api.Repo, filters ma // send query to the database and store result in variable err = e.client. + WithContext(ctx). Table(constants.TableSecret). Where("type = ?", constants.SecretRepo). Where("org = ?", r.GetOrg()). diff --git a/database/secret/list_team.go b/database/secret/list_team.go index 93a5d6db1..2e2a5773c 100644 --- a/database/secret/list_team.go +++ b/database/secret/list_team.go @@ -44,6 +44,7 @@ func (e *engine) ListSecretsForTeam(ctx context.Context, org, team string, filte // send query to the database and store result in variable err = e.client. + WithContext(ctx). Table(constants.TableSecret). Where("type = ?", constants.SecretShared). Where("org = ?", org). @@ -120,6 +121,7 @@ func (e *engine) ListSecretsForTeams(ctx context.Context, org string, teams []st // send query to the database and store result in variable err = e.client. + WithContext(ctx). Table(constants.TableSecret). Where("type = ?", constants.SecretShared). Where("org = ?", org). diff --git a/database/secret/table.go b/database/secret/table.go index e696409c2..650a3f3f4 100644 --- a/database/secret/table.go +++ b/database/secret/table.go @@ -68,11 +68,15 @@ func (e *engine) CreateSecretTable(ctx context.Context, driver string) error { switch driver { case constants.DriverPostgres: // create the secrets table for Postgres - return e.client.Exec(CreatePostgresTable).Error + return e.client. + WithContext(ctx). + Exec(CreatePostgresTable).Error case constants.DriverSqlite: fallthrough default: // create the secrets table for Sqlite - return e.client.Exec(CreateSqliteTable).Error + return e.client. + WithContext(ctx). + Exec(CreateSqliteTable).Error } } diff --git a/database/secret/update.go b/database/secret/update.go index 0bb713e47..c046d18ed 100644 --- a/database/secret/update.go +++ b/database/secret/update.go @@ -60,7 +60,10 @@ func (e *engine) UpdateSecret(ctx context.Context, s *library.Secret) (*library. } } - err = e.client.Table(constants.TableSecret).Save(secret.Nullify()).Error + err = e.client. + WithContext(ctx). + Table(constants.TableSecret). + Save(secret.Nullify()).Error if err != nil { return nil, err } diff --git a/database/service/clean.go b/database/service/clean.go index b4becb0ad..d234b4478 100644 --- a/database/service/clean.go +++ b/database/service/clean.go @@ -26,6 +26,7 @@ func (e *engine) CleanServices(ctx context.Context, msg string, before int64) (i // send query to the database result := e.client. + WithContext(ctx). Table(constants.TableService). Where("created < ?", before). Where("status = 'running' OR status = 'pending'"). diff --git a/database/service/count.go b/database/service/count.go index f8bee25b1..86e76cfe2 100644 --- a/database/service/count.go +++ b/database/service/count.go @@ -17,6 +17,7 @@ func (e *engine) CountServices(ctx context.Context) (int64, error) { // send query to the database and store result in variable err := e.client. + WithContext(ctx). Table(constants.TableService). Count(&s). Error diff --git a/database/service/count_build.go b/database/service/count_build.go index 1e84819a2..0f00403e2 100644 --- a/database/service/count_build.go +++ b/database/service/count_build.go @@ -22,6 +22,7 @@ func (e *engine) CountServicesForBuild(ctx context.Context, b *api.Build, filter // send query to the database and store result in variable err := e.client. + WithContext(ctx). Table(constants.TableService). Where("build_id = ?", b.GetID()). Where(filters). diff --git a/database/service/create.go b/database/service/create.go index 6d2ec8f74..3ecb96e04 100644 --- a/database/service/create.go +++ b/database/service/create.go @@ -32,7 +32,10 @@ func (e *engine) CreateService(ctx context.Context, s *library.Service) (*librar } // send query to the database - result := e.client.Table(constants.TableService).Create(service) + result := e.client. + WithContext(ctx). + Table(constants.TableService). + Create(service) return service.ToLibrary(), result.Error } diff --git a/database/service/delete.go b/database/service/delete.go index 3f0bd331e..835a25992 100644 --- a/database/service/delete.go +++ b/database/service/delete.go @@ -25,6 +25,7 @@ func (e *engine) DeleteService(ctx context.Context, s *library.Service) error { // send query to the database return e.client. + WithContext(ctx). Table(constants.TableService). Delete(service). Error diff --git a/database/service/get.go b/database/service/get.go index eee921123..37cf9a5d0 100644 --- a/database/service/get.go +++ b/database/service/get.go @@ -19,6 +19,7 @@ func (e *engine) GetService(ctx context.Context, id int64) (*library.Service, er // send query to the database and store result in variable err := e.client. + WithContext(ctx). Table(constants.TableService). Where("id = ?", id). Take(s). diff --git a/database/service/get_build.go b/database/service/get_build.go index 81a95e0bc..e85ddc368 100644 --- a/database/service/get_build.go +++ b/database/service/get_build.go @@ -25,6 +25,7 @@ func (e *engine) GetServiceForBuild(ctx context.Context, b *api.Build, number in // send query to the database and store result in variable err := e.client. + WithContext(ctx). Table(constants.TableService). Where("build_id = ?", b.GetID()). Where("number = ?", number). diff --git a/database/service/list.go b/database/service/list.go index 656beee10..6dd70fdd1 100644 --- a/database/service/list.go +++ b/database/service/list.go @@ -32,6 +32,7 @@ func (e *engine) ListServices(ctx context.Context) ([]*library.Service, error) { // send query to the database and store result in variable err = e.client. + WithContext(ctx). Table(constants.TableService). Find(&w). Error diff --git a/database/service/list_build.go b/database/service/list_build.go index 047aba669..faf8c81fb 100644 --- a/database/service/list_build.go +++ b/database/service/list_build.go @@ -40,6 +40,7 @@ func (e *engine) ListServicesForBuild(ctx context.Context, b *api.Build, filters // send query to the database and store result in variable err = e.client. + WithContext(ctx). Table(constants.TableService). Where("build_id = ?", b.GetID()). Where(filters). diff --git a/database/service/list_image.go b/database/service/list_image.go index a94447e6c..dbff3913c 100644 --- a/database/service/list_image.go +++ b/database/service/list_image.go @@ -22,6 +22,7 @@ func (e *engine) ListServiceImageCount(ctx context.Context) (map[string]float64, // send query to the database and store result in variable err := e.client. + WithContext(ctx). Table(constants.TableService). Select("image", " count(image) as count"). Group("image"). diff --git a/database/service/list_status.go b/database/service/list_status.go index 9e5f106fd..b2570b55d 100644 --- a/database/service/list_status.go +++ b/database/service/list_status.go @@ -28,6 +28,7 @@ func (e *engine) ListServiceStatusCount(ctx context.Context) (map[string]float64 // send query to the database and store result in variable err := e.client. + WithContext(ctx). Table(constants.TableService). Select("status", " count(status) as count"). Group("status"). diff --git a/database/service/table.go b/database/service/table.go index 56e78b249..426eb2333 100644 --- a/database/service/table.go +++ b/database/service/table.go @@ -66,11 +66,15 @@ func (e *engine) CreateServiceTable(ctx context.Context, driver string) error { switch driver { case constants.DriverPostgres: // create the services table for Postgres - return e.client.Exec(CreatePostgresTable).Error + return e.client. + WithContext(ctx). + Exec(CreatePostgresTable).Error case constants.DriverSqlite: fallthrough default: // create the services table for Sqlite - return e.client.Exec(CreateSqliteTable).Error + return e.client. + WithContext(ctx). + Exec(CreateSqliteTable).Error } } diff --git a/database/service/update.go b/database/service/update.go index 6eb43a7e2..a53b9ae41 100644 --- a/database/service/update.go +++ b/database/service/update.go @@ -32,7 +32,10 @@ func (e *engine) UpdateService(ctx context.Context, s *library.Service) (*librar } // send query to the database - result := e.client.Table(constants.TableService).Save(service) + result := e.client. + WithContext(ctx). + Table(constants.TableService). + Save(service) return service.ToLibrary(), result.Error } diff --git a/database/settings/create.go b/database/settings/create.go index 8c5747320..1879483b1 100644 --- a/database/settings/create.go +++ b/database/settings/create.go @@ -10,7 +10,7 @@ import ( ) // CreateSettings creates a platform settings record in the database. -func (e *engine) CreateSettings(_ context.Context, s *settings.Platform) (*settings.Platform, error) { +func (e *engine) CreateSettings(ctx context.Context, s *settings.Platform) (*settings.Platform, error) { e.logger.Tracef("creating platform settings with %v", s.String()) // cast the api type to database type @@ -23,7 +23,10 @@ func (e *engine) CreateSettings(_ context.Context, s *settings.Platform) (*setti } // send query to the database - err = e.client.Table(TableSettings).Create(settings.Nullify()).Error + err = e.client. + WithContext(ctx). + Table(TableSettings). + Create(settings.Nullify()).Error if err != nil { return nil, err } diff --git a/database/settings/get.go b/database/settings/get.go index 96bc19ef1..160faf6a4 100644 --- a/database/settings/get.go +++ b/database/settings/get.go @@ -18,6 +18,7 @@ func (e *engine) GetSettings(ctx context.Context) (*settings.Platform, error) { // send query to the database and store result in variable err := e.client. + WithContext(ctx). Table(TableSettings). Where("id = ?", 1). Take(s). diff --git a/database/settings/table.go b/database/settings/table.go index 18a4207ab..3eba03cfe 100644 --- a/database/settings/table.go +++ b/database/settings/table.go @@ -43,18 +43,22 @@ settings ( ) // CreateSettingsTable creates the settings table in the database. -func (e *engine) CreateSettingsTable(_ context.Context, driver string) error { +func (e *engine) CreateSettingsTable(ctx context.Context, driver string) error { e.logger.Tracef("creating settings table") // handle the driver provided to create the table switch driver { case constants.DriverPostgres: // create the steps table for Postgres - return e.client.Exec(CreatePostgresTable).Error + return e.client. + WithContext(ctx). + Exec(CreatePostgresTable).Error case constants.DriverSqlite: fallthrough default: // create the steps table for Sqlite - return e.client.Exec(CreateSqliteTable).Error + return e.client. + WithContext(ctx). + Exec(CreateSqliteTable).Error } } diff --git a/database/settings/update.go b/database/settings/update.go index c8a3e1679..e4e2f7f55 100644 --- a/database/settings/update.go +++ b/database/settings/update.go @@ -10,7 +10,7 @@ import ( ) // UpdateSettings updates a platform settings in the database. -func (e *engine) UpdateSettings(_ context.Context, s *settings.Platform) (*settings.Platform, error) { +func (e *engine) UpdateSettings(ctx context.Context, s *settings.Platform) (*settings.Platform, error) { e.logger.Trace("updating platform settings in the database") // cast the api type to database type @@ -23,7 +23,10 @@ func (e *engine) UpdateSettings(_ context.Context, s *settings.Platform) (*setti } // send query to the database - err = e.client.Table(TableSettings).Save(dbS.Nullify()).Error + err = e.client. + WithContext(ctx). + Table(TableSettings). + Save(dbS.Nullify()).Error if err != nil { return nil, err } diff --git a/database/step/clean.go b/database/step/clean.go index 3e75c906b..b7d8cb8f6 100644 --- a/database/step/clean.go +++ b/database/step/clean.go @@ -26,6 +26,7 @@ func (e *engine) CleanSteps(ctx context.Context, msg string, before int64) (int6 // send query to the database result := e.client. + WithContext(ctx). Table(constants.TableStep). Where("created < ?", before). Where("status = 'running' OR status = 'pending'"). diff --git a/database/step/count.go b/database/step/count.go index d5c64ba34..bb9f0ce3a 100644 --- a/database/step/count.go +++ b/database/step/count.go @@ -17,6 +17,7 @@ func (e *engine) CountSteps(ctx context.Context) (int64, error) { // send query to the database and store result in variable err := e.client. + WithContext(ctx). Table(constants.TableStep). Count(&s). Error diff --git a/database/step/count_build.go b/database/step/count_build.go index 6b4d63fc4..4a0cefda5 100644 --- a/database/step/count_build.go +++ b/database/step/count_build.go @@ -22,6 +22,7 @@ func (e *engine) CountStepsForBuild(ctx context.Context, b *api.Build, filters m // send query to the database and store result in variable err := e.client. + WithContext(ctx). Table(constants.TableStep). Where("build_id = ?", b.GetID()). Where(filters). diff --git a/database/step/create.go b/database/step/create.go index 99c9987b4..9e46cf146 100644 --- a/database/step/create.go +++ b/database/step/create.go @@ -32,7 +32,10 @@ func (e *engine) CreateStep(ctx context.Context, s *library.Step) (*library.Step } // send query to the database - result := e.client.Table(constants.TableStep).Create(step) + result := e.client. + WithContext(ctx). + Table(constants.TableStep). + Create(step) return step.ToLibrary(), result.Error } diff --git a/database/step/delete.go b/database/step/delete.go index dfd2d5bfa..7c9139f6b 100644 --- a/database/step/delete.go +++ b/database/step/delete.go @@ -25,6 +25,7 @@ func (e *engine) DeleteStep(ctx context.Context, s *library.Step) error { // send query to the database return e.client. + WithContext(ctx). Table(constants.TableStep). Delete(step). Error diff --git a/database/step/get.go b/database/step/get.go index 7a4e18041..a05a56fce 100644 --- a/database/step/get.go +++ b/database/step/get.go @@ -19,6 +19,7 @@ func (e *engine) GetStep(ctx context.Context, id int64) (*library.Step, error) { // send query to the database and store result in variable err := e.client. + WithContext(ctx). Table(constants.TableStep). Where("id = ?", id). Take(s). diff --git a/database/step/get_build.go b/database/step/get_build.go index 5d32ba582..de847a769 100644 --- a/database/step/get_build.go +++ b/database/step/get_build.go @@ -25,6 +25,7 @@ func (e *engine) GetStepForBuild(ctx context.Context, b *api.Build, number int) // send query to the database and store result in variable err := e.client. + WithContext(ctx). Table(constants.TableStep). Where("build_id = ?", b.GetID()). Where("number = ?", number). diff --git a/database/step/list.go b/database/step/list.go index 0d53297bd..9eb72996c 100644 --- a/database/step/list.go +++ b/database/step/list.go @@ -32,6 +32,7 @@ func (e *engine) ListSteps(ctx context.Context) ([]*library.Step, error) { // send query to the database and store result in variable err = e.client. + WithContext(ctx). Table(constants.TableStep). Find(&w). Error diff --git a/database/step/list_build.go b/database/step/list_build.go index ac0a608d1..1ba16fdb0 100644 --- a/database/step/list_build.go +++ b/database/step/list_build.go @@ -40,6 +40,7 @@ func (e *engine) ListStepsForBuild(ctx context.Context, b *api.Build, filters ma // send query to the database and store result in variable err = e.client. + WithContext(ctx). Table(constants.TableStep). Where("build_id = ?", b.GetID()). Where(filters). diff --git a/database/step/list_image.go b/database/step/list_image.go index 5b0f46aec..c485ec741 100644 --- a/database/step/list_image.go +++ b/database/step/list_image.go @@ -22,6 +22,7 @@ func (e *engine) ListStepImageCount(ctx context.Context) (map[string]float64, er // send query to the database and store result in variable err := e.client. + WithContext(ctx). Table(constants.TableStep). Select("image", " count(image) as count"). Group("image"). diff --git a/database/step/list_status.go b/database/step/list_status.go index c74b8db3c..75c19a805 100644 --- a/database/step/list_status.go +++ b/database/step/list_status.go @@ -28,6 +28,7 @@ func (e *engine) ListStepStatusCount(ctx context.Context) (map[string]float64, e // send query to the database and store result in variable err := e.client. + WithContext(ctx). Table(constants.TableStep). Select("status", " count(status) as count"). Group("status"). diff --git a/database/step/table.go b/database/step/table.go index a7cd7eb74..c7560326d 100644 --- a/database/step/table.go +++ b/database/step/table.go @@ -70,11 +70,15 @@ func (e *engine) CreateStepTable(ctx context.Context, driver string) error { switch driver { case constants.DriverPostgres: // create the steps table for Postgres - return e.client.Exec(CreatePostgresTable).Error + return e.client. + WithContext(ctx). + Exec(CreatePostgresTable).Error case constants.DriverSqlite: fallthrough default: // create the steps table for Sqlite - return e.client.Exec(CreateSqliteTable).Error + return e.client. + WithContext(ctx). + Exec(CreateSqliteTable).Error } } diff --git a/database/step/update.go b/database/step/update.go index 57fee9e74..d467a2b65 100644 --- a/database/step/update.go +++ b/database/step/update.go @@ -32,7 +32,10 @@ func (e *engine) UpdateStep(ctx context.Context, s *library.Step) (*library.Step } // send query to the database - result := e.client.Table(constants.TableStep).Save(step) + result := e.client. + WithContext(ctx). + Table(constants.TableStep). + Save(step) return step.ToLibrary(), result.Error } diff --git a/database/user/count.go b/database/user/count.go index c48fa00ed..b6b686fba 100644 --- a/database/user/count.go +++ b/database/user/count.go @@ -17,6 +17,7 @@ func (e *engine) CountUsers(ctx context.Context) (int64, error) { // send query to the database and store result in variable err := e.client. + WithContext(ctx). Table(constants.TableUser). Count(&u). Error diff --git a/database/user/create.go b/database/user/create.go index e9cf0f2c6..c778349c4 100644 --- a/database/user/create.go +++ b/database/user/create.go @@ -40,7 +40,10 @@ func (e *engine) CreateUser(ctx context.Context, u *api.User) (*api.User, error) } // send query to the database - result := e.client.Table(constants.TableUser).Create(user) + result := e.client. + WithContext(ctx). + Table(constants.TableUser). + Create(user) // decrypt fields to return user err = user.Decrypt(e.config.EncryptionKey) diff --git a/database/user/delete.go b/database/user/delete.go index 3154461e2..28547c07c 100644 --- a/database/user/delete.go +++ b/database/user/delete.go @@ -23,6 +23,7 @@ func (e *engine) DeleteUser(ctx context.Context, u *api.User) error { // send query to the database return e.client. + WithContext(ctx). Table(constants.TableUser). Delete(user). Error diff --git a/database/user/get.go b/database/user/get.go index a2f7d8468..30901c2ff 100644 --- a/database/user/get.go +++ b/database/user/get.go @@ -19,6 +19,7 @@ func (e *engine) GetUser(ctx context.Context, id int64) (*api.User, error) { // send query to the database and store result in variable err := e.client. + WithContext(ctx). Table(constants.TableUser). Where("id = ?", id). Take(u). diff --git a/database/user/get_name.go b/database/user/get_name.go index 8d3633eb8..fb9a90090 100644 --- a/database/user/get_name.go +++ b/database/user/get_name.go @@ -23,6 +23,7 @@ func (e *engine) GetUserForName(ctx context.Context, name string) (*api.User, er // send query to the database and store result in variable err := e.client. + WithContext(ctx). Table(constants.TableUser). Where("name = ?", name). Take(u). diff --git a/database/user/index.go b/database/user/index.go index d35e1accd..37dc9476e 100644 --- a/database/user/index.go +++ b/database/user/index.go @@ -20,5 +20,7 @@ func (e *engine) CreateUserIndexes(ctx context.Context) error { e.logger.Tracef("creating indexes for users table") // create the refresh_token column index for the users table - return e.client.Exec(CreateUserRefreshIndex).Error + return e.client. + WithContext(ctx). + Exec(CreateUserRefreshIndex).Error } diff --git a/database/user/list.go b/database/user/list.go index 80c7f5681..035699489 100644 --- a/database/user/list.go +++ b/database/user/list.go @@ -32,6 +32,7 @@ func (e *engine) ListUsers(ctx context.Context) ([]*api.User, error) { // send query to the database and store result in variable err = e.client. + WithContext(ctx). Table(constants.TableUser). Find(&u). Error diff --git a/database/user/list_lite.go b/database/user/list_lite.go index 1a5c570f9..c961fe483 100644 --- a/database/user/list_lite.go +++ b/database/user/list_lite.go @@ -36,6 +36,7 @@ func (e *engine) ListLiteUsers(ctx context.Context, page, perPage int) ([]*api.U offset := perPage * (page - 1) err = e.client. + WithContext(ctx). Table(constants.TableUser). Select("id", "name"). Limit(perPage). diff --git a/database/user/table.go b/database/user/table.go index 183390b43..de1f964a7 100644 --- a/database/user/table.go +++ b/database/user/table.go @@ -52,11 +52,15 @@ func (e *engine) CreateUserTable(ctx context.Context, driver string) error { switch driver { case constants.DriverPostgres: // create the users table for Postgres - return e.client.Exec(CreatePostgresTable).Error + return e.client. + WithContext(ctx). + Exec(CreatePostgresTable).Error case constants.DriverSqlite: fallthrough default: // create the users table for Sqlite - return e.client.Exec(CreateSqliteTable).Error + return e.client. + WithContext(ctx). + Exec(CreateSqliteTable).Error } } diff --git a/database/user/update.go b/database/user/update.go index 19f653f8d..d860e3293 100644 --- a/database/user/update.go +++ b/database/user/update.go @@ -36,7 +36,10 @@ func (e *engine) UpdateUser(ctx context.Context, u *api.User) (*api.User, error) } // send query to the database - result := e.client.Table(constants.TableUser).Save(user) + result := e.client. + WithContext(ctx). + Table(constants.TableUser). + Save(user) // decrypt fields to return user err = user.Decrypt(e.config.EncryptionKey) diff --git a/database/worker/count.go b/database/worker/count.go index 0d992cdca..6c440637c 100644 --- a/database/worker/count.go +++ b/database/worker/count.go @@ -17,6 +17,7 @@ func (e *engine) CountWorkers(ctx context.Context) (int64, error) { // send query to the database and store result in variable err := e.client. + WithContext(ctx). Table(constants.TableWorker). Count(&w). Error diff --git a/database/worker/create.go b/database/worker/create.go index 445669deb..180ee7ef6 100644 --- a/database/worker/create.go +++ b/database/worker/create.go @@ -32,7 +32,10 @@ func (e *engine) CreateWorker(ctx context.Context, w *api.Worker) (*api.Worker, } // send query to the database - result := e.client.Table(constants.TableWorker).Create(worker) + result := e.client. + WithContext(ctx). + Table(constants.TableWorker). + Create(worker) return worker.ToAPI(w.GetRunningBuilds()), result.Error } diff --git a/database/worker/delete.go b/database/worker/delete.go index 8df8ff5e0..252dd7831 100644 --- a/database/worker/delete.go +++ b/database/worker/delete.go @@ -25,6 +25,7 @@ func (e *engine) DeleteWorker(ctx context.Context, w *api.Worker) error { // send query to the database return e.client. + WithContext(ctx). Table(constants.TableWorker). Delete(worker). Error diff --git a/database/worker/get.go b/database/worker/get.go index cde920413..47be938a1 100644 --- a/database/worker/get.go +++ b/database/worker/get.go @@ -19,6 +19,7 @@ func (e *engine) GetWorker(ctx context.Context, id int64) (*api.Worker, error) { // send query to the database and store result in variable err := e.client. + WithContext(ctx). Table(constants.TableWorker). Where("id = ?", id). Take(w). diff --git a/database/worker/get_hostname.go b/database/worker/get_hostname.go index 1646e262e..d1150f80d 100644 --- a/database/worker/get_hostname.go +++ b/database/worker/get_hostname.go @@ -23,6 +23,7 @@ func (e *engine) GetWorkerForHostname(ctx context.Context, hostname string) (*ap // send query to the database and store result in variable err := e.client. + WithContext(ctx). Table(constants.TableWorker). Where("hostname = ?", hostname). Take(w). diff --git a/database/worker/index.go b/database/worker/index.go index 65fe7326f..6e1fc58e3 100644 --- a/database/worker/index.go +++ b/database/worker/index.go @@ -20,5 +20,7 @@ func (e *engine) CreateWorkerIndexes(ctx context.Context) error { e.logger.Tracef("creating indexes for workers table") // create the hostname and address columns index for the workers table - return e.client.Exec(CreateHostnameAddressIndex).Error + return e.client. + WithContext(ctx). + Exec(CreateHostnameAddressIndex).Error } diff --git a/database/worker/list.go b/database/worker/list.go index e2af759dd..34fc2ef79 100644 --- a/database/worker/list.go +++ b/database/worker/list.go @@ -21,7 +21,9 @@ func (e *engine) ListWorkers(ctx context.Context, active string, before, after i workers := []*api.Worker{} // build query with checked in constraints - query := e.client.Table(constants.TableWorker). + query := e.client. + WithContext(ctx). + Table(constants.TableWorker). Where("last_checked_in < ?", before). Where("last_checked_in > ?", after) diff --git a/database/worker/table.go b/database/worker/table.go index 403dd5ad3..921734f38 100644 --- a/database/worker/table.go +++ b/database/worker/table.go @@ -59,11 +59,15 @@ func (e *engine) CreateWorkerTable(ctx context.Context, driver string) error { switch driver { case constants.DriverPostgres: // create the workers table for Postgres - return e.client.Exec(CreatePostgresTable).Error + return e.client. + WithContext(ctx). + Exec(CreatePostgresTable).Error case constants.DriverSqlite: fallthrough default: // create the workers table for Sqlite - return e.client.Exec(CreateSqliteTable).Error + return e.client. + WithContext(ctx). + Exec(CreateSqliteTable).Error } } diff --git a/database/worker/update.go b/database/worker/update.go index 872d9c82d..49c78e420 100644 --- a/database/worker/update.go +++ b/database/worker/update.go @@ -32,7 +32,10 @@ func (e *engine) UpdateWorker(ctx context.Context, w *api.Worker) (*api.Worker, } // send query to the database - result := e.client.Table(constants.TableWorker).Save(worker) + result := e.client. + WithContext(ctx). + Table(constants.TableWorker). + Save(worker) return worker.ToAPI(w.GetRunningBuilds()), result.Error } From ae40a655e928e951fb04581f09ed36404a6d0ab1 Mon Sep 17 00:00:00 2001 From: davidvader Date: Tue, 13 Aug 2024 09:42:52 -0500 Subject: [PATCH 25/59] fix: lint --- database/executable/clean.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/database/executable/clean.go b/database/executable/clean.go index 10a15db41..ba4c06172 100644 --- a/database/executable/clean.go +++ b/database/executable/clean.go @@ -36,11 +36,13 @@ func (e *engine) CleanBuildExecutables(ctx context.Context) (int64, error) { res := e.client. WithContext(ctx). Exec(CleanExecutablesPostgres) + return res.RowsAffected, res.Error default: res := e.client. WithContext(ctx). Exec(CleanExecutablesSqlite) + return res.RowsAffected, res.Error } } From d921b3e4f58796a3cd98a8832242f3db66d471e4 Mon Sep 17 00:00:00 2001 From: davidvader Date: Tue, 20 Aug 2024 09:21:58 -0500 Subject: [PATCH 26/59] chore: merge with enhance/db-ctx --- docker-compose.yml | 1 + tracing/tracer.go | 7 +++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 4907ea58e..3062df209 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -47,6 +47,7 @@ services: VELA_ENABLE_TRACING: true VELA_TRACING_SERVICE_NAME: 'vela-server' OTEL_EXPORTER_OTLP_ENDPOINT: http://jaeger:4318 + VELA_TRACING_SAMPLER_RATIO: 1 env_file: - .env restart: always diff --git a/tracing/tracer.go b/tracing/tracer.go index 6a43f3c1b..0b2cd1d62 100644 --- a/tracing/tracer.go +++ b/tracing/tracer.go @@ -5,6 +5,7 @@ package tracing import ( "context" + "github.com/sirupsen/logrus" "github.com/urfave/cli/v2" "go.opentelemetry.io/otel" otlp "go.opentelemetry.io/otel/exporters/otlp/otlptrace" @@ -33,10 +34,12 @@ func initTracer(c *cli.Context) (*sdktrace.TracerProvider, error) { return nil, err } + logrus.Info("intializing tracing using sampler ratio: ", c.Float64("tracing.sampler.ratio")) + tp := sdktrace.NewTracerProvider( - sdktrace.WithSampler(sdktrace.AlwaysSample()), + // sdktrace.WithSampler(sdktrace.AlwaysSample()), sdktrace.WithSampler(sdktrace.ParentBased(sdktrace.TraceIDRatioBased(c.Float64("tracing.sampler.ratio")))), - // sdktrace.WithSampler(sdktrace.RatioSampler(c.Float64("tracing.sampler.ratio"))), + // sdktrace.WithSampler(sdktrace.RateLimiting(1000, 1000)), sdktrace.WithBatcher(exporter), sdktrace.WithResource(res), ) From e590f23f7991508ead44070b2b21133d87b8bfde Mon Sep 17 00:00:00 2001 From: davidvader Date: Wed, 21 Aug 2024 09:28:13 -0500 Subject: [PATCH 27/59] enhance: simple head samplers for ratio and rate limit --- docker-compose.yml | 3 ++- tracing/flags.go | 12 +++++++++--- tracing/sampler/sampler.go | 2 +- tracing/tracer.go | 11 +++++++---- 4 files changed, 19 insertions(+), 9 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 3062df209..dda303f4b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -46,8 +46,9 @@ services: VELA_SCHEDULE_ALLOWLIST: '*' VELA_ENABLE_TRACING: true VELA_TRACING_SERVICE_NAME: 'vela-server' + VELA_TRACING_SAMPLE_RATIO: 1 + VELA_TRACING_SAMPLE_RATELIMIT_PER_SECOND: 1 OTEL_EXPORTER_OTLP_ENDPOINT: http://jaeger:4318 - VELA_TRACING_SAMPLER_RATIO: 1 env_file: - .env restart: always diff --git a/tracing/flags.go b/tracing/flags.go index 5d8746ac5..e00010fda 100644 --- a/tracing/flags.go +++ b/tracing/flags.go @@ -22,9 +22,15 @@ var Flags = []cli.Flag{ Value: "vela-server", }, &cli.Float64Flag{ - EnvVars: []string{"VELA_TRACING_SAMPLER_RATIO", "TRACING_SAMPLER_RATIO"}, - Name: "tracing.sampler.ratio", - Usage: "set otel tracing sampler ratio. see: https://opentelemetry.io/docs/concepts/sampling/", + EnvVars: []string{"VELA_TRACING_SAMPLE_RATIO", "OTEL_TRACE_SAMPLE_RATIO"}, + Name: "tracing.sample.ratio", + Usage: "set otel tracing head-sampler acceptance ratio. see: https://opentelemetry.io/docs/concepts/sampling/", Value: 0.5, }, + &cli.Float64Flag{ + EnvVars: []string{"VELA_TRACING_SAMPLE_RATELIMIT_PER_SECOND", "OTEL_TRACING_SAMPLE_RATELIMIT_PER_SECOND"}, + Name: "tracing.sample.persecond", + Usage: "set otel tracing head-sampler rate-limiting to N per second. see: https://opentelemetry.io/docs/concepts/sampling/", + Value: 1, + }, } diff --git a/tracing/sampler/sampler.go b/tracing/sampler/sampler.go index e11dfdf97..e0db792f4 100644 --- a/tracing/sampler/sampler.go +++ b/tracing/sampler/sampler.go @@ -24,7 +24,7 @@ type rateLimitSampler struct { limiter *rate.Limiter } -func newRateLimitSampler(perSec float64) *rateLimitSampler { +func NewRateLimitSampler(perSec float64) *rateLimitSampler { return &rateLimitSampler{ maxPerSecond: perSec, limiter: rate.NewLimiter(rate.Every(time.Duration(1.0/perSec)*time.Second), 1), diff --git a/tracing/tracer.go b/tracing/tracer.go index 0b2cd1d62..1c24e5714 100644 --- a/tracing/tracer.go +++ b/tracing/tracer.go @@ -5,6 +5,7 @@ package tracing import ( "context" + "github.com/go-vela/server/tracing/sampler" "github.com/sirupsen/logrus" "github.com/urfave/cli/v2" "go.opentelemetry.io/otel" @@ -34,12 +35,14 @@ func initTracer(c *cli.Context) (*sdktrace.TracerProvider, error) { return nil, err } - logrus.Info("intializing tracing using sampler ratio: ", c.Float64("tracing.sampler.ratio")) + logrus.Infof("intializing tracing using samplers: ratio(%f), persecond(%f)", c.Float64("tracing.sampler.ratio"), c.Float64("tracing.sampler.persecond")) + + ratioSampler := sdktrace.ParentBased(sdktrace.TraceIDRatioBased(c.Float64("tracing.sampler.ratio"))) + rateLimitSampler := sdktrace.ParentBased(sampler.NewRateLimitSampler(c.Float64("tracing.sampler.persecond"))) tp := sdktrace.NewTracerProvider( - // sdktrace.WithSampler(sdktrace.AlwaysSample()), - sdktrace.WithSampler(sdktrace.ParentBased(sdktrace.TraceIDRatioBased(c.Float64("tracing.sampler.ratio")))), - // sdktrace.WithSampler(sdktrace.RateLimiting(1000, 1000)), + sdktrace.WithSampler(ratioSampler), + sdktrace.WithSampler(rateLimitSampler), sdktrace.WithBatcher(exporter), sdktrace.WithResource(res), ) From 3aa5b8d6d4e39310f70876df757906301090c345 Mon Sep 17 00:00:00 2001 From: davidvader Date: Wed, 21 Aug 2024 16:45:01 -0500 Subject: [PATCH 28/59] enhance: customizable config, code cleanup, middleware --- cmd/vela-server/server.go | 17 +--- database/context.go | 4 +- database/context_test.go | 2 +- database/database.go | 4 +- database/database_test.go | 2 +- database/opts.go | 5 +- go.mod | 4 +- router/middleware/tracing.go | 30 +++++++ router/middleware/tracing/context.go | 37 +++++++++ router/middleware/tracing/context_test.go | 95 +++++++++++++++++++++++ router/middleware/tracing/doc.go | 10 +++ router/middleware/tracing/tracing.go | 28 +++++++ router/middleware/tracing/tracing_test.go | 76 ++++++++++++++++++ scm/github/repo.go | 5 +- tracing/config.go | 45 +++++++---- tracing/flags.go | 5 ++ tracing/sampler.go | 61 +++++++++++++++ tracing/sampler/sampler.go | 57 -------------- tracing/tracer.go | 25 +++--- 19 files changed, 401 insertions(+), 111 deletions(-) create mode 100644 router/middleware/tracing.go create mode 100644 router/middleware/tracing/context.go create mode 100644 router/middleware/tracing/context_test.go create mode 100644 router/middleware/tracing/doc.go create mode 100644 router/middleware/tracing/tracing.go create mode 100644 router/middleware/tracing/tracing_test.go create mode 100644 tracing/sampler.go delete mode 100644 tracing/sampler/sampler.go diff --git a/cmd/vela-server/server.go b/cmd/vela-server/server.go index 0e9283f71..c64a15024 100644 --- a/cmd/vela-server/server.go +++ b/cmd/vela-server/server.go @@ -16,7 +16,6 @@ import ( "github.com/gin-gonic/gin" "github.com/sirupsen/logrus" "github.com/urfave/cli/v2" - "go.opentelemetry.io/contrib/instrumentation/github.com/gin-gonic/gin/otelgin" "golang.org/x/sync/errgroup" "gorm.io/gorm" "k8s.io/apimachinery/pkg/util/wait" @@ -80,20 +79,13 @@ func server(c *cli.Context) error { return err } - // create the tracing config - tc, err := tracing.New(c) + tc, err := tracing.FromCLIContext(c) if err != nil { return err } - // opt-in for telemetry - tracingMiddleware := func(ctx *gin.Context) {} + // shutdown opt-in tracing when server closes if tc.EnableTracing { - // initialize tracing middleware - tracingMiddleware = otelgin.Middleware(c.String("tracing.service.name"), - otelgin.WithTracerProvider(tc.TracerProvider)) - - // shutdown tracing when server closes defer func() { err := tc.TracerProvider.Shutdown(context.Background()) if err != nil { @@ -213,9 +205,8 @@ func server(c *cli.Context) error { middleware.DefaultRepoEventsMask(c.Int64("default-repo-events-mask")), middleware.DefaultRepoApproveBuild(c.String("default-repo-approve-build")), middleware.ScheduleFrequency(c.Duration("schedule-minimum-frequency")), - - // tracing middle, which is an empty handler when tracing is disabled - tracingMiddleware, + middleware.TracingClient(tc), + middleware.TracingInstrumentation(tc), ) addr, err := url.Parse(c.String("server-addr")) diff --git a/database/context.go b/database/context.go index 54d0b9f0f..fe2fe753a 100644 --- a/database/context.go +++ b/database/context.go @@ -40,7 +40,7 @@ func ToContext(c Setter, d Interface) { } // FromCLIContext creates and returns a database engine from the urfave/cli context. -func FromCLIContext(c *cli.Context, tc *tracing.Config) (Interface, error) { +func FromCLIContext(c *cli.Context, tc *tracing.Client) (Interface, error) { logrus.Debug("creating database engine from CLI configuration") return New( @@ -56,6 +56,6 @@ func FromCLIContext(c *cli.Context, tc *tracing.Config) (Interface, error) { WithLogSlowThreshold(c.Duration("database.log.slow_threshold")), WithLogShowSQL(c.Bool("database.log.show_sql")), WithSkipCreation(c.Bool("database.skip_creation")), - WithTracingConfig(tc), + WithTracing(tc), ) } diff --git a/database/context_test.go b/database/context_test.go index 80e26ea58..b6b12c98e 100644 --- a/database/context_test.go +++ b/database/context_test.go @@ -134,7 +134,7 @@ func TestDatabase_FromCLIContext(t *testing.T) { // run tests for _, test := range tests { t.Run(test.name, func(t *testing.T) { - _, err := FromCLIContext(test.context, &tracing.Config{EnableTracing: false}) + _, err := FromCLIContext(test.context, &tracing.Client{Config: tracing.Config{EnableTracing: false}}) if test.failure { if err == nil { diff --git a/database/database.go b/database/database.go index 757482962..441e904f5 100644 --- a/database/database.go +++ b/database/database.go @@ -73,7 +73,7 @@ type ( // sirupsen/logrus logger used in database functions logger *logrus.Entry // configurations related to telemetry/tracing - tracing *tracing.Config + tracing *tracing.Client settings.SettingsInterface build.BuildInterface @@ -243,7 +243,7 @@ func NewTest() (Interface, error) { WithDriver("sqlite3"), WithEncryptionKey("A1B2C3D4E5G6H7I8J9K0LMNOPQRSTUVW"), WithSkipCreation(false), - WithTracingConfig(&tracing.Config{EnableTracing: false}), + WithTracing(&tracing.Client{Config: tracing.Config{EnableTracing: false}}), WithLogLevel("warn"), WithLogShowSQL(false), WithLogSkipNotFound(true), diff --git a/database/database_test.go b/database/database_test.go index 6ea4fe92d..00866b7a1 100644 --- a/database/database_test.go +++ b/database/database_test.go @@ -112,7 +112,7 @@ func TestDatabase_New(t *testing.T) { WithLogSlowThreshold(test.config.LogSlowThreshold), WithEncryptionKey(test.config.EncryptionKey), WithSkipCreation(test.config.SkipCreation), - WithTracingConfig(&tracing.Config{EnableTracing: false}), + WithTracing(&tracing.Client{Config: tracing.Config{EnableTracing: false}}), ) if test.failure { diff --git a/database/opts.go b/database/opts.go index 5158b8382..14607a856 100644 --- a/database/opts.go +++ b/database/opts.go @@ -132,10 +132,9 @@ func WithSkipCreation(skipCreation bool) EngineOpt { } } -// WithTracingConfig sets the shared tracing config in the database engine. -func WithTracingConfig(tracing *tracing.Config) EngineOpt { +// WithTracing sets the shared tracing config in the database engine. +func WithTracing(tracing *tracing.Client) EngineOpt { return func(e *engine) error { - // set the tracing config e.tracing = tracing return nil diff --git a/go.mod b/go.mod index a4462c5bb..6757efd83 100644 --- a/go.mod +++ b/go.mod @@ -46,11 +46,11 @@ require ( go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.28.0 go.opentelemetry.io/otel/sdk v1.28.0 - go.opentelemetry.io/otel/trace v1.28.0 go.starlark.net v0.0.0-20240314022150-ee8ed142361c golang.org/x/crypto v0.25.0 golang.org/x/oauth2 v0.21.0 golang.org/x/sync v0.7.0 + golang.org/x/time v0.5.0 gorm.io/driver/postgres v1.5.9 gorm.io/driver/sqlite v1.5.6 gorm.io/gorm v1.25.11 @@ -137,12 +137,12 @@ require ( github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 // indirect github.com/yuin/gopher-lua v1.1.1 // indirect go.opentelemetry.io/otel/metric v1.28.0 // indirect + go.opentelemetry.io/otel/trace v1.28.0 // indirect go.opentelemetry.io/proto/otlp v1.3.1 // indirect golang.org/x/arch v0.8.0 // indirect golang.org/x/net v0.27.0 // indirect golang.org/x/sys v0.22.0 // indirect golang.org/x/text v0.16.0 // indirect - golang.org/x/time v0.5.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240730163845-b1a4ccb954bf // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240730163845-b1a4ccb954bf // indirect google.golang.org/grpc v1.65.0 // indirect diff --git a/router/middleware/tracing.go b/router/middleware/tracing.go new file mode 100644 index 000000000..bd8cf7b5d --- /dev/null +++ b/router/middleware/tracing.go @@ -0,0 +1,30 @@ +// SPDX-License-Identifier: Apache-2.0 + +package middleware + +import ( + "github.com/gin-gonic/gin" + tracingMiddleware "github.com/go-vela/server/router/middleware/tracing" + "github.com/go-vela/server/tracing" + "go.opentelemetry.io/contrib/instrumentation/github.com/gin-gonic/gin/otelgin" +) + +// TracingClient is a middleware function that attaches the tracing config +// to the context of every http.Request. +func TracingClient(tc *tracing.Client) gin.HandlerFunc { + return func(c *gin.Context) { + tracingMiddleware.ToContext(c, tc) + + c.Next() + } +} + +// TracingInstrumentation is a middleware function that attaches the tracing config +// to the context of every http.Request. +func TracingInstrumentation(tc *tracing.Client) gin.HandlerFunc { + if tc.EnableTracing { + return otelgin.Middleware(tc.ServiceName, otelgin.WithTracerProvider(tc.TracerProvider)) + } + + return func(c *gin.Context) {} +} diff --git a/router/middleware/tracing/context.go b/router/middleware/tracing/context.go new file mode 100644 index 000000000..ea457bbf1 --- /dev/null +++ b/router/middleware/tracing/context.go @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: Apache-2.0 + +package tracing + +import ( + "context" + + "github.com/go-vela/server/tracing" +) + +const key = "tracing" + +// Setter defines a context that enables setting values. +type Setter interface { + Set(string, interface{}) +} + +// FromContext returns the associated value with this context. +func FromContext(c context.Context) *tracing.Client { + value := c.Value(key) + if value == nil { + return nil + } + + tc, ok := value.(*tracing.Client) + if !ok { + return nil + } + + return tc +} + +// ToContext adds the value to this context if it supports +// the Setter interface. +func ToContext(c Setter, tc *tracing.Client) { + c.Set(key, tc) +} diff --git a/router/middleware/tracing/context_test.go b/router/middleware/tracing/context_test.go new file mode 100644 index 000000000..08eb330c0 --- /dev/null +++ b/router/middleware/tracing/context_test.go @@ -0,0 +1,95 @@ +// SPDX-License-Identifier: Apache-2.0 + +package tracing + +import ( + "testing" + + "github.com/gin-gonic/gin" + "github.com/go-vela/server/tracing" +) + +func TestTracing_FromContext(t *testing.T) { + // setup types + serviceName := "vela-test" + want := &tracing.Client{ + Config: tracing.Config{ + ServiceName: serviceName, + }, + } + + // setup context + gin.SetMode(gin.TestMode) + context, _ := gin.CreateTestContext(nil) + context.Set(key, want) + + // run test + got := FromContext(context) + + if got != want { + t.Errorf("FromContext is %v, want %v", got, want) + } +} + +func TestTracing_FromContext_Bad(t *testing.T) { + // setup context + gin.SetMode(gin.TestMode) + context, _ := gin.CreateTestContext(nil) + context.Set(key, nil) + + // run test + got := FromContext(context) + + if got != nil { + t.Errorf("FromContext is %v, want nil", got) + } +} + +func TestTracing_FromContext_WrongType(t *testing.T) { + // setup context + gin.SetMode(gin.TestMode) + context, _ := gin.CreateTestContext(nil) + context.Set(key, 1) + + // run test + got := FromContext(context) + + if got != nil { + t.Errorf("FromContext is %v, want nil", got) + } +} + +func TestTracing_FromContext_Empty(t *testing.T) { + // setup context + gin.SetMode(gin.TestMode) + context, _ := gin.CreateTestContext(nil) + + // run test + got := FromContext(context) + + if got != nil { + t.Errorf("FromContext is %v, want nil", got) + } +} + +func TestTracing_ToContext(t *testing.T) { + // setup types + serviceName := "vela-test" + want := &tracing.Client{ + Config: tracing.Config{ + ServiceName: serviceName, + }, + } + + // setup context + gin.SetMode(gin.TestMode) + context, _ := gin.CreateTestContext(nil) + ToContext(context, want) + + // run test + got := context.Value(key) + + if got != want { + t.Errorf("ToContext is %v, want %v", got, want) + } +} diff --git a/router/middleware/tracing/doc.go b/router/middleware/tracing/doc.go new file mode 100644 index 000000000..7c3816b93 --- /dev/null +++ b/router/middleware/tracing/doc.go @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: Apache-2.0 + +// Package tracing provides the ability for inserting +// or extracting the Vela OTEL tracing client +// from the middleware chain for the API. +// +// Usage: +// +// import "github.com/go-vela/server/router/middleware/tracing" +package tracing diff --git a/router/middleware/tracing/tracing.go b/router/middleware/tracing/tracing.go new file mode 100644 index 000000000..d12115cea --- /dev/null +++ b/router/middleware/tracing/tracing.go @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: Apache-2.0 + +package tracing + +import ( + "github.com/gin-gonic/gin" + "github.com/sirupsen/logrus" + + "github.com/go-vela/server/tracing" +) + +// Retrieve gets the value in the given context. +func Retrieve(c *gin.Context) *tracing.Client { + return FromContext(c) +} + +// Establish sets the value in the given context. +func Establish() gin.HandlerFunc { + return func(c *gin.Context) { + l := c.MustGet("logger").(*logrus.Entry) + tc := Retrieve(c) + + l.Debugf("reading tracing client from context") + + ToContext(c, tc) + c.Next() + } +} diff --git a/router/middleware/tracing/tracing_test.go b/router/middleware/tracing/tracing_test.go new file mode 100644 index 000000000..eb08a8c23 --- /dev/null +++ b/router/middleware/tracing/tracing_test.go @@ -0,0 +1,76 @@ +// SPDX-License-Identifier: Apache-2.0 + +package tracing + +import ( + "net/http" + "net/http/httptest" + "reflect" + "testing" + + "github.com/gin-gonic/gin" + "github.com/go-vela/server/tracing" + "github.com/sirupsen/logrus" +) + +func TestTracing_Retrieve(t *testing.T) { + // setup types + serviceName := "vela-test" + want := &tracing.Client{ + Config: tracing.Config{ + ServiceName: serviceName, + }, + } + + // setup context + gin.SetMode(gin.TestMode) + context, _ := gin.CreateTestContext(nil) + ToContext(context, want) + + // run test + got := Retrieve(context) + + if got != want { + t.Errorf("Retrieve is %v, want %v", got, want) + } +} + +func TestTracing_Establish(t *testing.T) { + // setup types + serviceName := "vela-test" + want := &tracing.Client{ + Config: tracing.Config{ + ServiceName: serviceName, + }, + } + + got := new(tracing.Client) + + // setup context + gin.SetMode(gin.TestMode) + + resp := httptest.NewRecorder() + context, engine := gin.CreateTestContext(resp) + context.Request, _ = http.NewRequest(http.MethodGet, "/hello", nil) + + // setup mock server + engine.Use(func(c *gin.Context) { c.Set("logger", logrus.NewEntry(logrus.StandardLogger())) }) + engine.Use(func(c *gin.Context) { ToContext(c, want) }) + engine.Use(Establish()) + engine.GET("/hello", func(c *gin.Context) { + got = Retrieve(c) + + c.Status(http.StatusOK) + }) + + // run test + engine.ServeHTTP(resp, context.Request) + + if resp.Code != http.StatusOK { + t.Errorf("Establish returned %v, want %v", resp.Code, http.StatusOK) + } + + if !reflect.DeepEqual(got, want) { + t.Errorf("Establish is %v, want %v", got, want) + } +} diff --git a/scm/github/repo.go b/scm/github/repo.go index d19d6afc9..ad9db4271 100644 --- a/scm/github/repo.go +++ b/scm/github/repo.go @@ -12,7 +12,6 @@ import ( "github.com/google/go-github/v63/github" "github.com/sirupsen/logrus" - "go.opentelemetry.io/otel/trace" api "github.com/go-vela/server/api/types" "github.com/go-vela/types/constants" @@ -506,9 +505,7 @@ func (c *client) GetOrgAndRepoName(ctx context.Context, u *api.User, o string, r // ListUserRepos returns a list of all repos the user has access to. func (c *client) ListUserRepos(ctx context.Context, u *api.User) ([]*api.Repo, error) { c.Logger.WithFields(logrus.Fields{ - "user": u.GetName(), - "span_id": trace.SpanFromContext(ctx).SpanContext().SpanID(), - "trace_id": trace.SpanFromContext(ctx).SpanContext().TraceID(), + "user": u.GetName(), }).Tracef("listing source repositories for %s", u.GetName()) // create GitHub OAuth client with user's token diff --git a/tracing/config.go b/tracing/config.go index e61561c5d..880d9d3c9 100644 --- a/tracing/config.go +++ b/tracing/config.go @@ -7,27 +7,46 @@ import ( sdktrace "go.opentelemetry.io/otel/sdk/trace" ) +// Client represents the tracing client and the configurations that were used to initialize it. +type Client struct { + Config + TracerProvider *sdktrace.TracerProvider +} + // Config represents the configurations for otel tracing. type Config struct { - EnableTracing bool - ServiceName string - TracerProvider *sdktrace.TracerProvider + EnableTracing bool + ServiceName string + Sampler } -// New takes cli context and returns a tracing config to supply to traceable services. -func New(c *cli.Context) (*Config, error) { - enable := c.Bool("tracing.enable") - serviceName := c.String("tracing.service.name") +// Sampler represents the configurations for the otel sampler. +// Used to determine if a trace should be sampled. +type Sampler struct { + Tags []string + Ratio float64 + PerSecond float64 +} + +// FromCLIContext takes cli context and returns a tracing config to supply to traceable services. +func FromCLIContext(c *cli.Context) (*Client, error) { + tCfg := Config{ + EnableTracing: c.Bool("tracing.enable"), + ServiceName: c.String("tracing.service.name"), + Sampler: Sampler{ + Tags: c.StringSlice("tracing.sample.tags"), + Ratio: c.Float64("tracing.sample.ratio"), + PerSecond: c.Float64("tracing.sample.persecond"), + }, + } - // could skip creating the tracer if tracing is disabled - tracerProvider, err := initTracer(c) + tracer, err := initTracer(c.Context, tCfg) if err != nil { return nil, err } - return &Config{ - EnableTracing: enable, - ServiceName: serviceName, - TracerProvider: tracerProvider, + return &Client{ + Config: tCfg, + TracerProvider: tracer, }, nil } diff --git a/tracing/flags.go b/tracing/flags.go index e00010fda..95750c913 100644 --- a/tracing/flags.go +++ b/tracing/flags.go @@ -21,6 +21,11 @@ var Flags = []cli.Flag{ Usage: "set otel tracing service name", Value: "vela-server", }, + &cli.StringSliceFlag{ + EnvVars: []string{"VELA_TRACING_SAMPLE_TAGS", "OTEL_TRACING_SAMPLE_TAGS"}, + Name: "tracing.sample.tags", + Usage: "set otel trace sample state tags. see: https://opentelemetry.io/docs/concepts/sampling/", + }, &cli.Float64Flag{ EnvVars: []string{"VELA_TRACING_SAMPLE_RATIO", "OTEL_TRACE_SAMPLE_RATIO"}, Name: "tracing.sample.ratio", diff --git a/tracing/sampler.go b/tracing/sampler.go new file mode 100644 index 000000000..8d80e6d52 --- /dev/null +++ b/tracing/sampler.go @@ -0,0 +1,61 @@ +// SPDX-License-Identifier: Apache-2.0 + +package tracing + +import ( + "fmt" + "time" + + "go.opentelemetry.io/otel/attribute" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + "golang.org/x/time/rate" +) + +const ( + SAMPLER_TYPE = "sampler.type" + SAMPLER_PARAM = "sampler.param" + SAMPLER_PATH = "sampler.path" + SAMPLER_PARENT = "sampler.parent" + SAMPLER_ALGORITHM = "sampler.algorithm" +) + +var _ sdktrace.Sampler = (*rateLimitSampler)(nil) + +type rateLimitSampler struct { + maxPerSecond float64 + limiter *rate.Limiter +} + +// NewRateLimitSampler returns a new rate limit sampler. +func NewRateLimitSampler(tc Config) *rateLimitSampler { + return &rateLimitSampler{ + maxPerSecond: tc.PerSecond, + limiter: rate.NewLimiter(rate.Every(time.Duration(1.0/tc.PerSecond)*time.Second), 1), + } +} + +// ShouldSample determines if a trace should be sampled. +func (s *rateLimitSampler) ShouldSample(parameters sdktrace.SamplingParameters) sdktrace.SamplingResult { + // apply sampler attributes + attributes := []attribute.KeyValue{ + attribute.String(SAMPLER_ALGORITHM, "rate-limiting"), + attribute.Float64(SAMPLER_PARAM, s.maxPerSecond), + } + + // default to drop + result := sdktrace.SamplingResult{ + Decision: sdktrace.Drop, + Attributes: attributes, + } + + if s.limiter.Allow() { + result.Decision = sdktrace.RecordAndSample + } + + return result +} + +// Description returns the description of the rate limit sampler. +func (s *rateLimitSampler) Description() string { + return fmt.Sprintf("rate-limit-sampler{%v}", s.maxPerSecond) +} diff --git a/tracing/sampler/sampler.go b/tracing/sampler/sampler.go deleted file mode 100644 index e0db792f4..000000000 --- a/tracing/sampler/sampler.go +++ /dev/null @@ -1,57 +0,0 @@ -package sampler - -import ( - "fmt" - "time" - - "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/sdk/trace" - "golang.org/x/time/rate" -) - -const ( - SAMPLER_TYPE = "sampler.type" - SAMPLER_PARAM = "sampler.param" - SAMPLER_PATH = "sampler.path" - SAMPLER_PARENT = "sampler.parent" - SAMPLER_ALGORITHM = "sampler.algorithm" -) - -var _ trace.Sampler = (*rateLimitSampler)(nil) - -type rateLimitSampler struct { - maxPerSecond float64 - limiter *rate.Limiter -} - -func NewRateLimitSampler(perSec float64) *rateLimitSampler { - return &rateLimitSampler{ - maxPerSecond: perSec, - limiter: rate.NewLimiter(rate.Every(time.Duration(1.0/perSec)*time.Second), 1), - } -} - -func (s *rateLimitSampler) ShouldSample(parameters trace.SamplingParameters) trace.SamplingResult { - if s.limiter.Allow() { - return trace.SamplingResult{ - Decision: trace.RecordAndSample, - Attributes: []attribute.KeyValue{ - attribute.String(SAMPLER_ALGORITHM, "rate-limiting"), - attribute.Float64(SAMPLER_PARAM, s.maxPerSecond), - }, - } - } else { - return trace.SamplingResult{ - Decision: trace.Drop, - Attributes: []attribute.KeyValue{ - attribute.String(SAMPLER_ALGORITHM, "rate-limiting"), - attribute.Float64(SAMPLER_PARAM, s.maxPerSecond), - }, - } - } - -} - -func (s *rateLimitSampler) Description() string { - return fmt.Sprintf("RateLimitSampler{%v}", s.maxPerSecond) -} diff --git a/tracing/tracer.go b/tracing/tracer.go index 1c24e5714..a30c1bdc7 100644 --- a/tracing/tracer.go +++ b/tracing/tracer.go @@ -5,40 +5,39 @@ package tracing import ( "context" - "github.com/go-vela/server/tracing/sampler" - "github.com/sirupsen/logrus" - "github.com/urfave/cli/v2" "go.opentelemetry.io/otel" - otlp "go.opentelemetry.io/otel/exporters/otlp/otlptrace" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace" "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" "go.opentelemetry.io/otel/propagation" "go.opentelemetry.io/otel/sdk/resource" sdktrace "go.opentelemetry.io/otel/sdk/trace" - semconv "go.opentelemetry.io/otel/semconv/v1.20.0" + semconv "go.opentelemetry.io/otel/semconv/v1.17.0" ) // initTracer returns the tracer provider supplied to the tracing config. -func initTracer(c *cli.Context) (*sdktrace.TracerProvider, error) { +func initTracer(ctx context.Context, tCfg Config) (*sdktrace.TracerProvider, error) { client := otlptracehttp.NewClient() - ctx := context.Background() - // TODO: inject actual context - exporter, err := otlp.New(ctx, client) + exporter, err := otlptrace.New(ctx, client) if err != nil { return nil, err } res, err := resource.New(ctx, resource.WithAttributes( - semconv.ServiceName(c.String("tracing.service.name")), + semconv.ServiceName(tCfg.ServiceName), )) if err != nil { return nil, err } - logrus.Infof("intializing tracing using samplers: ratio(%f), persecond(%f)", c.Float64("tracing.sampler.ratio"), c.Float64("tracing.sampler.persecond")) + ratioSampler := sdktrace.ParentBased( + sdktrace.TraceIDRatioBased(tCfg.Sampler.Ratio), + ) + rateLimitSampler := sdktrace.ParentBased( + NewRateLimitSampler(tCfg), + ) - ratioSampler := sdktrace.ParentBased(sdktrace.TraceIDRatioBased(c.Float64("tracing.sampler.ratio"))) - rateLimitSampler := sdktrace.ParentBased(sampler.NewRateLimitSampler(c.Float64("tracing.sampler.persecond"))) + // todo: apply the tags to the sampler tp := sdktrace.NewTracerProvider( sdktrace.WithSampler(ratioSampler), From 868fa97ea32b1ac2a523884027fd6df477603304 Mon Sep 17 00:00:00 2001 From: davidvader Date: Wed, 21 Aug 2024 16:48:51 -0500 Subject: [PATCH 29/59] fix: lint --- router/middleware/tracing.go | 5 +++-- router/middleware/tracing/context_test.go | 1 + router/middleware/tracing/tracing_test.go | 3 ++- tracing/sampler.go | 26 +++++++++++------------ 4 files changed, 19 insertions(+), 16 deletions(-) diff --git a/router/middleware/tracing.go b/router/middleware/tracing.go index bd8cf7b5d..1ba2db0ff 100644 --- a/router/middleware/tracing.go +++ b/router/middleware/tracing.go @@ -4,9 +4,10 @@ package middleware import ( "github.com/gin-gonic/gin" + "go.opentelemetry.io/contrib/instrumentation/github.com/gin-gonic/gin/otelgin" + tracingMiddleware "github.com/go-vela/server/router/middleware/tracing" "github.com/go-vela/server/tracing" - "go.opentelemetry.io/contrib/instrumentation/github.com/gin-gonic/gin/otelgin" ) // TracingClient is a middleware function that attaches the tracing config @@ -26,5 +27,5 @@ func TracingInstrumentation(tc *tracing.Client) gin.HandlerFunc { return otelgin.Middleware(tc.ServiceName, otelgin.WithTracerProvider(tc.TracerProvider)) } - return func(c *gin.Context) {} + return func(_ *gin.Context) {} } diff --git a/router/middleware/tracing/context_test.go b/router/middleware/tracing/context_test.go index 08eb330c0..fce8efeea 100644 --- a/router/middleware/tracing/context_test.go +++ b/router/middleware/tracing/context_test.go @@ -6,6 +6,7 @@ import ( "testing" "github.com/gin-gonic/gin" + "github.com/go-vela/server/tracing" ) diff --git a/router/middleware/tracing/tracing_test.go b/router/middleware/tracing/tracing_test.go index eb08a8c23..8c1d3cec1 100644 --- a/router/middleware/tracing/tracing_test.go +++ b/router/middleware/tracing/tracing_test.go @@ -9,8 +9,9 @@ import ( "testing" "github.com/gin-gonic/gin" - "github.com/go-vela/server/tracing" "github.com/sirupsen/logrus" + + "github.com/go-vela/server/tracing" ) func TestTracing_Retrieve(t *testing.T) { diff --git a/tracing/sampler.go b/tracing/sampler.go index 8d80e6d52..5631ad896 100644 --- a/tracing/sampler.go +++ b/tracing/sampler.go @@ -12,34 +12,34 @@ import ( ) const ( - SAMPLER_TYPE = "sampler.type" - SAMPLER_PARAM = "sampler.param" - SAMPLER_PATH = "sampler.path" - SAMPLER_PARENT = "sampler.parent" - SAMPLER_ALGORITHM = "sampler.algorithm" + SamplerType = "sampler.type" + SamplerParam = "sampler.param" + SamplerPath = "sampler.path" + SamplerParent = "sampler.parent" + SamplerAlgorithm = "sampler.algorithm" ) -var _ sdktrace.Sampler = (*rateLimitSampler)(nil) +var _ sdktrace.Sampler = (*RateLimitSampler)(nil) -type rateLimitSampler struct { +type RateLimitSampler struct { maxPerSecond float64 limiter *rate.Limiter } // NewRateLimitSampler returns a new rate limit sampler. -func NewRateLimitSampler(tc Config) *rateLimitSampler { - return &rateLimitSampler{ +func NewRateLimitSampler(tc Config) *RateLimitSampler { + return &RateLimitSampler{ maxPerSecond: tc.PerSecond, limiter: rate.NewLimiter(rate.Every(time.Duration(1.0/tc.PerSecond)*time.Second), 1), } } // ShouldSample determines if a trace should be sampled. -func (s *rateLimitSampler) ShouldSample(parameters sdktrace.SamplingParameters) sdktrace.SamplingResult { +func (s *RateLimitSampler) ShouldSample(_ sdktrace.SamplingParameters) sdktrace.SamplingResult { // apply sampler attributes attributes := []attribute.KeyValue{ - attribute.String(SAMPLER_ALGORITHM, "rate-limiting"), - attribute.Float64(SAMPLER_PARAM, s.maxPerSecond), + attribute.String(SamplerAlgorithm, "rate-limiting"), + attribute.Float64(SamplerParam, s.maxPerSecond), } // default to drop @@ -56,6 +56,6 @@ func (s *rateLimitSampler) ShouldSample(parameters sdktrace.SamplingParameters) } // Description returns the description of the rate limit sampler. -func (s *rateLimitSampler) Description() string { +func (s *RateLimitSampler) Description() string { return fmt.Sprintf("rate-limit-sampler{%v}", s.maxPerSecond) } From 02899ae13eabbfe0d7abc97c33f1ef57d44d002d Mon Sep 17 00:00:00 2001 From: davidvader Date: Mon, 26 Aug 2024 08:46:30 -0500 Subject: [PATCH 30/59] enhance: customizable resource and span attributes for restrictive collectors --- cmd/vela-server/server.go | 1 - tracing/config.go | 67 +++++++++++++++++++++++++++++++-------- tracing/flags.go | 65 +++++++++++++++++++++++++++++++------ tracing/sampler.go | 41 ++++++++++++------------ tracing/tracer.go | 53 +++++++++++++++++++++++-------- 5 files changed, 169 insertions(+), 58 deletions(-) diff --git a/cmd/vela-server/server.go b/cmd/vela-server/server.go index c64a15024..5e251c7c2 100644 --- a/cmd/vela-server/server.go +++ b/cmd/vela-server/server.go @@ -84,7 +84,6 @@ func server(c *cli.Context) error { return err } - // shutdown opt-in tracing when server closes if tc.EnableTracing { defer func() { err := tc.TracerProvider.Shutdown(context.Background()) diff --git a/tracing/config.go b/tracing/config.go index 880d9d3c9..abdf91437 100644 --- a/tracing/config.go +++ b/tracing/config.go @@ -3,6 +3,10 @@ package tracing import ( + "fmt" + "os" + "strings" + "github.com/urfave/cli/v2" sdktrace "go.opentelemetry.io/otel/sdk/trace" ) @@ -15,38 +19,75 @@ type Client struct { // Config represents the configurations for otel tracing. type Config struct { - EnableTracing bool - ServiceName string + EnableTracing bool + ServiceName string + ExporterURL string + CertPath string + ResourceAttributes map[string]string Sampler } // Sampler represents the configurations for the otel sampler. // Used to determine if a trace should be sampled. type Sampler struct { - Tags []string - Ratio float64 - PerSecond float64 + TraceStateAttributes map[string]string + SpanAttributes map[string]string + Ratio float64 + PerSecond float64 } // FromCLIContext takes cli context and returns a tracing config to supply to traceable services. func FromCLIContext(c *cli.Context) (*Client, error) { - tCfg := Config{ - EnableTracing: c.Bool("tracing.enable"), - ServiceName: c.String("tracing.service.name"), + cfg := Config{ + EnableTracing: c.Bool("tracing.enable"), + ServiceName: c.String("tracing.service.name"), + ExporterURL: c.String("tracing.exporter.endpoint"), + CertPath: c.String("tracing.exporter.cert_path"), + ResourceAttributes: map[string]string{}, Sampler: Sampler{ - Tags: c.StringSlice("tracing.sample.tags"), - Ratio: c.Float64("tracing.sample.ratio"), - PerSecond: c.Float64("tracing.sample.persecond"), + TraceStateAttributes: map[string]string{ + "sampler": c.String("tracing.sampler.tracestate"), + }, + SpanAttributes: map[string]string{ + "w3c.tracestate": fmt.Sprintf("sampler=%s", c.String("tracing.sampler.tracestate")), + "sampler.parent": c.String("tracing.sampler.parent"), + "sampler.type": c.String("tracing.sampler.type"), + }, + Ratio: c.Float64("tracing.sampler.ratio"), + PerSecond: c.Float64("tracing.sampler.persecond"), }, } - tracer, err := initTracer(c.Context, tCfg) + // add resource attributes + for _, attr := range c.StringSlice("tracing.resource.attributes") { + kv := strings.Split(attr, "=") + if len(kv) != 2 { + continue + } + + cfg.ResourceAttributes[kv[0]] = kv[1] + } + + // add resource attributes from environment + for _, attr := range c.StringSlice("tracing.resource.env_attributes") { + kv := strings.Split(attr, "=") + if len(kv) != 2 { + continue + } + + v, found := os.LookupEnv(kv[1]) + if found { + cfg.ResourceAttributes[kv[0]] = v + } + } + + tracer, err := initTracer(c.Context, cfg) if err != nil { return nil, err } return &Client{ - Config: tCfg, + Config: cfg, TracerProvider: tracer, }, nil } diff --git a/tracing/flags.go b/tracing/flags.go index 95750c913..1dd3b6911 100644 --- a/tracing/flags.go +++ b/tracing/flags.go @@ -10,31 +10,76 @@ var Flags = []cli.Flag{ // Tracing Flags &cli.BoolFlag{ - EnvVars: []string{"VELA_ENABLE_TRACING", "TRACING_ENABLE"}, + EnvVars: []string{"VELA_OTEL_TRACING_ENABLE"}, Name: "tracing.enable", Usage: "enable otel tracing", Value: false, }, + + // Exporter Flags + &cli.StringFlag{ - EnvVars: []string{"VELA_TRACING_SERVICE_NAME", "TRACING_SERVICE_NAME"}, + EnvVars: []string{"VELA_OTEL_TRACING_SERVICE_NAME"}, Name: "tracing.service.name", Usage: "set otel tracing service name", - Value: "vela-server", + Value: "go-otel-server", + }, + &cli.StringFlag{ + EnvVars: []string{"VELA_OTEL_TRACING_EXPORTER_SSL_CERT_PATH"}, + Name: "tracing.exporter.cert_path", + Usage: "set the path to certs used for communicating with the otel exporter. if not set, will use insecure communication", + }, + &cli.StringFlag{ + EnvVars: []string{"VELA_OTEL_TRACING_ENDPOINT"}, + Name: "tracing.exporter.endpoint", + Usage: "set the otel exporter endpoint", + Value: "127.0.0.1:4318", + }, + + // Resource Flags + + &cli.StringSliceFlag{ + EnvVars: []string{"VELA_OTEL_TRACING_RESOURCE_ATTRIBUTES"}, + Name: "tracing.resource.attributes", + Usage: "set otel resource attributes as a list of key=value pairs. each one will be attached to each span as an attribute", + Value: cli.NewStringSlice("process.runtime.name=go"), }, &cli.StringSliceFlag{ - EnvVars: []string{"VELA_TRACING_SAMPLE_TAGS", "OTEL_TRACING_SAMPLE_TAGS"}, - Name: "tracing.sample.tags", - Usage: "set otel trace sample state tags. see: https://opentelemetry.io/docs/concepts/sampling/", + EnvVars: []string{"VELA_OTEL_TRACING_RESOURCE_ENV_ATTRIBUTES"}, + Name: "tracing.resource.env_attributes", + Usage: "set otel resource attributes as a list of key=env_variable_key pairs. each one will be attached to each span as an attribute where the value is retrieved from the environment using the pair value", + Value: cli.NewStringSlice("deployment.environment=CLOUD_ENVIRONMENT"), + }, + + // Sampler Flags + + &cli.StringFlag{ + EnvVars: []string{"VELA_OTEL_TRACING_SAMPLER_TRACESTATE"}, + Name: "tracing.sampler.tracestate", + Usage: "set otel sampler trace state attached to each span as an attribute.", + Value: "sampler.tracestate", + }, + &cli.StringFlag{ + EnvVars: []string{"VELA_OTEL_TRACING_SAMPLER_PARENT"}, + Name: "tracing.sampler.parent", + Usage: "set otel sampler parent attribute attached to each span.", + Value: "sampler.parent", + }, + &cli.StringFlag{ + EnvVars: []string{"VELA_OTEL_TRACING_SAMPLER_TYPE"}, + Name: "tracing.sampler.type", + Usage: "set otel sampler type attribute attached to each span.", + Value: "sampler.type", }, &cli.Float64Flag{ - EnvVars: []string{"VELA_TRACING_SAMPLE_RATIO", "OTEL_TRACE_SAMPLE_RATIO"}, - Name: "tracing.sample.ratio", + EnvVars: []string{"VELA_OTEL_TRACING_SAMPLER_RATIO"}, + Name: "tracing.sampler.ratio", Usage: "set otel tracing head-sampler acceptance ratio. see: https://opentelemetry.io/docs/concepts/sampling/", Value: 0.5, }, &cli.Float64Flag{ - EnvVars: []string{"VELA_TRACING_SAMPLE_RATELIMIT_PER_SECOND", "OTEL_TRACING_SAMPLE_RATELIMIT_PER_SECOND"}, - Name: "tracing.sample.persecond", + EnvVars: []string{"VELA_OTEL_TRACING_SAMPLER_RATELIMIT_PER_SECOND"}, + Name: "tracing.sampler.persecond", Usage: "set otel tracing head-sampler rate-limiting to N per second. see: https://opentelemetry.io/docs/concepts/sampling/", Value: 1, }, diff --git a/tracing/sampler.go b/tracing/sampler.go index 5631ad896..c63266aae 100644 --- a/tracing/sampler.go +++ b/tracing/sampler.go @@ -8,44 +8,45 @@ import ( "go.opentelemetry.io/otel/attribute" sdktrace "go.opentelemetry.io/otel/sdk/trace" + "go.opentelemetry.io/otel/trace" "golang.org/x/time/rate" ) -const ( - SamplerType = "sampler.type" - SamplerParam = "sampler.param" - SamplerPath = "sampler.path" - SamplerParent = "sampler.parent" - SamplerAlgorithm = "sampler.algorithm" -) - -var _ sdktrace.Sampler = (*RateLimitSampler)(nil) - +// RateLimitSampler is a sampler that uses time-based rate limiting. type RateLimitSampler struct { - maxPerSecond float64 - limiter *rate.Limiter + Config + limiter *rate.Limiter } // NewRateLimitSampler returns a new rate limit sampler. func NewRateLimitSampler(tc Config) *RateLimitSampler { return &RateLimitSampler{ - maxPerSecond: tc.PerSecond, - limiter: rate.NewLimiter(rate.Every(time.Duration(1.0/tc.PerSecond)*time.Second), 1), + Config: tc, + limiter: rate.NewLimiter(rate.Every(time.Duration(1.0/tc.PerSecond)*time.Second), 1), } } // ShouldSample determines if a trace should be sampled. -func (s *RateLimitSampler) ShouldSample(_ sdktrace.SamplingParameters) sdktrace.SamplingResult { - // apply sampler attributes +func (s *RateLimitSampler) ShouldSample(p sdktrace.SamplingParameters) sdktrace.SamplingResult { + psc := trace.SpanContextFromContext(p.ParentContext) + ts := psc.TraceState() + for k, v := range s.Config.TraceStateAttributes { + ts, _ = ts.Insert(k, v) + } + attributes := []attribute.KeyValue{ - attribute.String(SamplerAlgorithm, "rate-limiting"), - attribute.Float64(SamplerParam, s.maxPerSecond), + attribute.String("sampler.algorithm", "rate-limiting"), + attribute.Float64("sampler.param", s.Config.PerSecond), + } + + for k, v := range s.Config.SpanAttributes { + attributes = append(attributes, attribute.String(k, v)) } - // default to drop result := sdktrace.SamplingResult{ Decision: sdktrace.Drop, Attributes: attributes, + Tracestate: ts, } if s.limiter.Allow() { @@ -57,5 +58,5 @@ func (s *RateLimitSampler) ShouldSample(_ sdktrace.SamplingParameters) sdktrace. // Description returns the description of the rate limit sampler. func (s *RateLimitSampler) Description() string { - return fmt.Sprintf("rate-limit-sampler{%v}", s.maxPerSecond) + return fmt.Sprintf("rate-limit-sampler{%v}", s.Config.PerSecond) } diff --git a/tracing/tracer.go b/tracing/tracer.go index a30c1bdc7..ca5d314a7 100644 --- a/tracing/tracer.go +++ b/tracing/tracer.go @@ -4,8 +4,13 @@ package tracing import ( "context" + "crypto/tls" + "crypto/x509" + "os" + "github.com/sirupsen/logrus" "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/exporters/otlp/otlptrace" "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" "go.opentelemetry.io/otel/propagation" @@ -15,32 +20,52 @@ import ( ) // initTracer returns the tracer provider supplied to the tracing config. -func initTracer(ctx context.Context, tCfg Config) (*sdktrace.TracerProvider, error) { - client := otlptracehttp.NewClient() +func initTracer(ctx context.Context, cfg Config) (*sdktrace.TracerProvider, error) { + withTLS := otlptracehttp.WithInsecure() + + if len(cfg.CertPath) > 0 { + pem, err := os.ReadFile(cfg.CertPath) + if err != nil { + return nil, err + } + + certs := x509.NewCertPool() + certs.AppendCertsFromPEM(pem) + + withTLS = otlptracehttp.WithTLSClientConfig(&tls.Config{RootCAs: certs}) + } else { + logrus.Warn("no otel cert path set, exporting traces insecurely") + } + + client := otlptracehttp.NewClient( + otlptracehttp.WithEndpoint(cfg.ExporterURL), + withTLS, + ) exporter, err := otlptrace.New(ctx, client) if err != nil { return nil, err } - res, err := resource.New(ctx, resource.WithAttributes( - semconv.ServiceName(tCfg.ServiceName), - )) - if err != nil { - return nil, err + attrs := []attribute.KeyValue{ + semconv.ServiceName(cfg.ServiceName), } - ratioSampler := sdktrace.ParentBased( - sdktrace.TraceIDRatioBased(tCfg.Sampler.Ratio), - ) - rateLimitSampler := sdktrace.ParentBased( - NewRateLimitSampler(tCfg), + for k, v := range cfg.ResourceAttributes { + attrs = append(attrs, attribute.String(k, v)) + } + + res, err := resource.Merge( + resource.Default(), + resource.NewSchemaless(attrs...), ) + if err != nil { + return nil, err + } - // todo: apply the tags to the sampler + rateLimitSampler := NewRateLimitSampler(cfg) tp := sdktrace.NewTracerProvider( - sdktrace.WithSampler(ratioSampler), sdktrace.WithSampler(rateLimitSampler), sdktrace.WithBatcher(exporter), sdktrace.WithResource(res), From 14619d2ce1975c191094259435f036fb3e8998ef Mon Sep 17 00:00:00 2001 From: davidvader Date: Mon, 26 Aug 2024 08:51:42 -0500 Subject: [PATCH 31/59] fix: min tls version --- go.mod | 2 +- tracing/sampler.go | 1 + tracing/tracer.go | 6 +++++- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 6757efd83..192d335d3 100644 --- a/go.mod +++ b/go.mod @@ -46,6 +46,7 @@ require ( go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.28.0 go.opentelemetry.io/otel/sdk v1.28.0 + go.opentelemetry.io/otel/trace v1.28.0 go.starlark.net v0.0.0-20240314022150-ee8ed142361c golang.org/x/crypto v0.25.0 golang.org/x/oauth2 v0.21.0 @@ -137,7 +138,6 @@ require ( github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 // indirect github.com/yuin/gopher-lua v1.1.1 // indirect go.opentelemetry.io/otel/metric v1.28.0 // indirect - go.opentelemetry.io/otel/trace v1.28.0 // indirect go.opentelemetry.io/proto/otlp v1.3.1 // indirect golang.org/x/arch v0.8.0 // indirect golang.org/x/net v0.27.0 // indirect diff --git a/tracing/sampler.go b/tracing/sampler.go index c63266aae..198a74700 100644 --- a/tracing/sampler.go +++ b/tracing/sampler.go @@ -30,6 +30,7 @@ func NewRateLimitSampler(tc Config) *RateLimitSampler { func (s *RateLimitSampler) ShouldSample(p sdktrace.SamplingParameters) sdktrace.SamplingResult { psc := trace.SpanContextFromContext(p.ParentContext) ts := psc.TraceState() + for k, v := range s.Config.TraceStateAttributes { ts, _ = ts.Insert(k, v) } diff --git a/tracing/tracer.go b/tracing/tracer.go index ca5d314a7..a1e21755b 100644 --- a/tracing/tracer.go +++ b/tracing/tracer.go @@ -32,7 +32,11 @@ func initTracer(ctx context.Context, cfg Config) (*sdktrace.TracerProvider, erro certs := x509.NewCertPool() certs.AppendCertsFromPEM(pem) - withTLS = otlptracehttp.WithTLSClientConfig(&tls.Config{RootCAs: certs}) + withTLS = otlptracehttp.WithTLSClientConfig( + &tls.Config{ + RootCAs: certs, + MinVersion: tls.VersionTLS12, + }) } else { logrus.Warn("no otel cert path set, exporting traces insecurely") } From 3d273ce365df44c9e745cd7b2b571d61dd096ca5 Mon Sep 17 00:00:00 2001 From: davidvader Date: Mon, 26 Aug 2024 08:56:39 -0500 Subject: [PATCH 32/59] trigger pipeline From 0397f5a8f7459c7313b606c0c325851a88704942 Mon Sep 17 00:00:00 2001 From: davidvader Date: Mon, 26 Aug 2024 08:58:30 -0500 Subject: [PATCH 33/59] fix: compose options --- docker-compose.yml | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index dda303f4b..b8d629cb0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -44,11 +44,14 @@ services: VELA_ENABLE_SECURE_COOKIE: 'false' VELA_REPO_ALLOWLIST: '*' VELA_SCHEDULE_ALLOWLIST: '*' - VELA_ENABLE_TRACING: true - VELA_TRACING_SERVICE_NAME: 'vela-server' - VELA_TRACING_SAMPLE_RATIO: 1 - VELA_TRACING_SAMPLE_RATELIMIT_PER_SECOND: 1 - OTEL_EXPORTER_OTLP_ENDPOINT: http://jaeger:4318 + VELA_OTEL_EXPORTER_OTLP_ENDPOINT: http://jaeger:4318 + VELA_OTEL_TRACING_ENDPOINT: jaeger:4318 + VELA_OTEL_TRACING_ENABLE: true + VELA_OTEL_TRACING_SERVICE_NAME: vela-server + VELA_OTEL_TRACING_RESOURCE_ATTRIBUTES: "process.runtime.name=go" + VELA_OTEL_TRACING_RESOURCE_ENV_ATTRIBUTES: "deployment.environment=CLOUD_ENVIRONMENT" + VELA_OTEL_TRACING_SAMPLER_RATIO: 1 + VELA_OTEL_TRACING_SAMPLER_RATELIMIT_PER_SECOND: 0.2 env_file: - .env restart: always From fcc8b238531d44ce5e87792d534ea4b72f46e9b2 Mon Sep 17 00:00:00 2001 From: davidvader Date: Mon, 26 Aug 2024 09:06:15 -0500 Subject: [PATCH 34/59] fix: add disabled tracing config to db test --- database/database.go | 2 +- database/integration_test.go | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/database/database.go b/database/database.go index 441e904f5..6f21ec40a 100644 --- a/database/database.go +++ b/database/database.go @@ -243,10 +243,10 @@ func NewTest() (Interface, error) { WithDriver("sqlite3"), WithEncryptionKey("A1B2C3D4E5G6H7I8J9K0LMNOPQRSTUVW"), WithSkipCreation(false), - WithTracing(&tracing.Client{Config: tracing.Config{EnableTracing: false}}), WithLogLevel("warn"), WithLogShowSQL(false), WithLogSkipNotFound(true), WithLogSlowThreshold(200*time.Millisecond), + WithTracing(&tracing.Client{Config: tracing.Config{EnableTracing: false}}), ) } diff --git a/database/integration_test.go b/database/integration_test.go index ad3748c4f..75a7624d3 100644 --- a/database/integration_test.go +++ b/database/integration_test.go @@ -33,6 +33,7 @@ import ( "github.com/go-vela/server/database/testutils" "github.com/go-vela/server/database/user" "github.com/go-vela/server/database/worker" + "github.com/go-vela/server/tracing" "github.com/go-vela/types/constants" "github.com/go-vela/types/library" "github.com/go-vela/types/raw" @@ -114,6 +115,7 @@ func TestDatabase_Integration(t *testing.T) { WithDriver(test.config.Driver), WithEncryptionKey(test.config.EncryptionKey), WithSkipCreation(test.config.SkipCreation), + WithTracing(&tracing.Client{Config: tracing.Config{EnableTracing: false}}), ) if err != nil { t.Errorf("unable to create new database engine for %s: %v", test.name, err) From 33bf1359390364444aebf75f46bc0dc85a575f35 Mon Sep 17 00:00:00 2001 From: davidvader Date: Mon, 26 Aug 2024 11:57:54 -0500 Subject: [PATCH 35/59] fix: remove unused var --- docker-compose.yml | 1 - tracing/flags.go | 8 +------- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index b8d629cb0..f2ef7afbe 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -50,7 +50,6 @@ services: VELA_OTEL_TRACING_SERVICE_NAME: vela-server VELA_OTEL_TRACING_RESOURCE_ATTRIBUTES: "process.runtime.name=go" VELA_OTEL_TRACING_RESOURCE_ENV_ATTRIBUTES: "deployment.environment=CLOUD_ENVIRONMENT" - VELA_OTEL_TRACING_SAMPLER_RATIO: 1 VELA_OTEL_TRACING_SAMPLER_RATELIMIT_PER_SECOND: 0.2 env_file: - .env diff --git a/tracing/flags.go b/tracing/flags.go index 1dd3b6911..390d1a61e 100644 --- a/tracing/flags.go +++ b/tracing/flags.go @@ -71,16 +71,10 @@ var Flags = []cli.Flag{ Usage: "set otel sampler type attribute attached to each span.", Value: "sampler.type", }, - &cli.Float64Flag{ - EnvVars: []string{"VELA_OTEL_TRACING_SAMPLER_RATIO"}, - Name: "tracing.sampler.ratio", - Usage: "set otel tracing head-sampler acceptance ratio. see: https://opentelemetry.io/docs/concepts/sampling/", - Value: 0.5, - }, &cli.Float64Flag{ EnvVars: []string{"VELA_OTEL_TRACING_SAMPLER_RATELIMIT_PER_SECOND"}, Name: "tracing.sampler.persecond", Usage: "set otel tracing head-sampler rate-limiting to N per second. see: https://opentelemetry.io/docs/concepts/sampling/", - Value: 1, + Value: 0.2, }, } From 069c4d7a6c1aaf4bbd6cd802b527af4b1774c9c2 Mon Sep 17 00:00:00 2001 From: davidvader Date: Wed, 4 Sep 2024 08:58:15 -0500 Subject: [PATCH 36/59] chore: go mod tidy --- docker-compose.yml | 2 +- go.sum | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index f2ef7afbe..250dc2498 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -50,7 +50,7 @@ services: VELA_OTEL_TRACING_SERVICE_NAME: vela-server VELA_OTEL_TRACING_RESOURCE_ATTRIBUTES: "process.runtime.name=go" VELA_OTEL_TRACING_RESOURCE_ENV_ATTRIBUTES: "deployment.environment=CLOUD_ENVIRONMENT" - VELA_OTEL_TRACING_SAMPLER_RATELIMIT_PER_SECOND: 0.2 + VELA_OTEL_TRACING_SAMPLER_RATELIMIT_PER_SECOND: 5 env_file: - .env restart: always diff --git a/go.sum b/go.sum index f8ff064a0..6baf24652 100644 --- a/go.sum +++ b/go.sum @@ -35,8 +35,6 @@ github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= -github.com/buildkite/yaml v0.0.0-20181016232759-0caa5f0796e3 h1:q+sMKdA6L8LyGVudTkpGoC73h6ak2iWSPFiFo/pFOU8= -github.com/buildkite/yaml v0.0.0-20181016232759-0caa5f0796e3/go.mod h1:5hCug3EZaHXU3FdCA3gJm0YTNi+V+ooA2qNTiVpky4A= github.com/bytedance/sonic v1.11.9 h1:LFHENlIY/SLzDWverzdOvgMztTxcfcF+cqNsz9pK5zg= github.com/bytedance/sonic v1.11.9/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4= github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3zCYjJFNM= From 85b67b1fe6c232cd2b21e249d11358d054026e0b Mon Sep 17 00:00:00 2001 From: davidvader Date: Wed, 4 Sep 2024 08:59:30 -0500 Subject: [PATCH 37/59] chore: revert compose var --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 250dc2498..f2ef7afbe 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -50,7 +50,7 @@ services: VELA_OTEL_TRACING_SERVICE_NAME: vela-server VELA_OTEL_TRACING_RESOURCE_ATTRIBUTES: "process.runtime.name=go" VELA_OTEL_TRACING_RESOURCE_ENV_ATTRIBUTES: "deployment.environment=CLOUD_ENVIRONMENT" - VELA_OTEL_TRACING_SAMPLER_RATELIMIT_PER_SECOND: 5 + VELA_OTEL_TRACING_SAMPLER_RATELIMIT_PER_SECOND: 0.2 env_file: - .env restart: always From 0aa5de5abb71171ef426b67e8e9907d36dc6202b Mon Sep 17 00:00:00 2001 From: davidvader Date: Thu, 5 Sep 2024 14:23:45 -0500 Subject: [PATCH 38/59] chore: remove bogus file --- something.json | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 something.json diff --git a/something.json b/something.json deleted file mode 100644 index e69de29bb..000000000 From ee8a03ab7eeeee4c3211ee9f3a3d0b72a68a7c2a Mon Sep 17 00:00:00 2001 From: davidvader Date: Mon, 9 Sep 2024 10:02:17 -0500 Subject: [PATCH 39/59] feat: tracing middleware tests --- router/middleware/tracing_test.go | 117 ++++++++++++++++++++++++++++++ 1 file changed, 117 insertions(+) create mode 100644 router/middleware/tracing_test.go diff --git a/router/middleware/tracing_test.go b/router/middleware/tracing_test.go new file mode 100644 index 000000000..88af9258e --- /dev/null +++ b/router/middleware/tracing_test.go @@ -0,0 +1,117 @@ +// SPDX-License-Identifier: Apache-2.0 + +package middleware + +import ( + "errors" + "net/http" + "net/http/httptest" + "reflect" + "testing" + + "github.com/gin-gonic/gin" + "github.com/go-vela/server/tracing" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + "go.opentelemetry.io/otel/trace" +) + +func TestMiddleware_TracingClient(t *testing.T) { + // setup types + var got *tracing.Client + want := &tracing.Client{ + Config: tracing.Config{ + EnableTracing: true, + }, + } + + // setup context + gin.SetMode(gin.TestMode) + + resp := httptest.NewRecorder() + context, engine := gin.CreateTestContext(resp) + context.Request, _ = http.NewRequest(http.MethodGet, "/health", nil) + + // setup mock server + engine.Use(TracingClient(want)) + engine.GET("/health", func(c *gin.Context) { + got = c.Value("tracing").(*tracing.Client) + + c.Status(http.StatusOK) + }) + + // run test + engine.ServeHTTP(context.Writer, context.Request) + + if !reflect.DeepEqual(got, want) { + t.Errorf("TracingClient is %v, want %v", got, want) + } +} + +func TestMiddleware_TracingInstrumentation(t *testing.T) { + // setup types + tt := []struct { + tc *tracing.Client + assert func(trace.SpanContext) error + }{ + { + tc: &tracing.Client{ + Config: tracing.Config{ + EnableTracing: false, + ServiceName: "vela-test", + }, + TracerProvider: sdktrace.NewTracerProvider(), + }, + assert: func(got trace.SpanContext) error { + if !reflect.DeepEqual(got, trace.SpanContext{}) { + return errors.New("span context is not empty") + } + return nil + }, + }, + { + tc: &tracing.Client{ + Config: tracing.Config{ + EnableTracing: true, + ServiceName: "vela-test", + }, + TracerProvider: sdktrace.NewTracerProvider(), + }, + assert: func(got trace.SpanContext) error { + if reflect.DeepEqual(got, trace.SpanContext{}) { + return errors.New("span context is empty") + } + return nil + }, + }, + } + + // setup context + gin.SetMode(gin.TestMode) + + for _, test := range tt { + got := trace.SpanContext{} + resp := httptest.NewRecorder() + context, engine := gin.CreateTestContext(resp) + context.Request, _ = http.NewRequest(http.MethodGet, "/health", nil) + + // setup mock server + engine.Use(TracingInstrumentation(test.tc)) + engine.GET("/health", func(c *gin.Context) { + got = trace.SpanContextFromContext(c.Request.Context()) + + c.Status(http.StatusOK) + }) + + // run test + engine.ServeHTTP(context.Writer, context.Request) + + if resp.Code != http.StatusOK { + t.Errorf("TracingInstrumentation returned %v, want %v", resp.Code, http.StatusOK) + } + + err := test.assert(got) + if err != nil { + t.Errorf("TracingInstrumentation test assertion failed: %s", err) + } + } +} From cd973d5e03d5821c9a0a782859b8e50af44eb9ce Mon Sep 17 00:00:00 2001 From: davidvader Date: Mon, 9 Sep 2024 10:04:27 -0500 Subject: [PATCH 40/59] chore: lint --- router/middleware/tracing_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/router/middleware/tracing_test.go b/router/middleware/tracing_test.go index 88af9258e..76313cc41 100644 --- a/router/middleware/tracing_test.go +++ b/router/middleware/tracing_test.go @@ -10,9 +10,10 @@ import ( "testing" "github.com/gin-gonic/gin" - "github.com/go-vela/server/tracing" sdktrace "go.opentelemetry.io/otel/sdk/trace" "go.opentelemetry.io/otel/trace" + + "github.com/go-vela/server/tracing" ) func TestMiddleware_TracingClient(t *testing.T) { From 7b27836044eca6e563a85b3490394d0d7fd7b205 Mon Sep 17 00:00:00 2001 From: davidvader Date: Wed, 11 Sep 2024 10:50:04 -0500 Subject: [PATCH 41/59] enhance: configurable min-tls --- tracing/config.go | 2 ++ tracing/flags.go | 6 ++++++ tracing/tracer.go | 32 +++++++++++++++++++++++++++----- 3 files changed, 35 insertions(+), 5 deletions(-) diff --git a/tracing/config.go b/tracing/config.go index abdf91437..aa48f3490 100644 --- a/tracing/config.go +++ b/tracing/config.go @@ -23,6 +23,7 @@ type Config struct { ServiceName string ExporterURL string CertPath string + TLSMinVersion string ResourceAttributes map[string]string Sampler } @@ -43,6 +44,7 @@ func FromCLIContext(c *cli.Context) (*Client, error) { ServiceName: c.String("tracing.service.name"), ExporterURL: c.String("tracing.exporter.endpoint"), CertPath: c.String("tracing.exporter.cert_path"), + TLSMinVersion: c.String("tracing.tls-min-version"), ResourceAttributes: map[string]string{}, Sampler: Sampler{ TraceStateAttributes: map[string]string{ diff --git a/tracing/flags.go b/tracing/flags.go index 390d1a61e..e1d9ccbb5 100644 --- a/tracing/flags.go +++ b/tracing/flags.go @@ -29,6 +29,12 @@ var Flags = []cli.Flag{ Name: "tracing.exporter.cert_path", Usage: "set the path to certs used for communicating with the otel exporter. if not set, will use insecure communication", }, + &cli.StringFlag{ + EnvVars: []string{"VELA_OTEL_TRACING_TLS_MIN_VERSION"}, + Name: "tracing.tls-min-version", + Usage: "optional TLS minimum version requirement to set when communicating with the otel exporter", + Value: "1.2", + }, &cli.StringFlag{ EnvVars: []string{"VELA_OTEL_TRACING_ENDPOINT"}, Name: "tracing.exporter.endpoint", diff --git a/tracing/tracer.go b/tracing/tracer.go index a1e21755b..a5c141641 100644 --- a/tracing/tracer.go +++ b/tracing/tracer.go @@ -6,6 +6,7 @@ import ( "context" "crypto/tls" "crypto/x509" + "fmt" "os" "github.com/sirupsen/logrus" @@ -32,11 +33,32 @@ func initTracer(ctx context.Context, cfg Config) (*sdktrace.TracerProvider, erro certs := x509.NewCertPool() certs.AppendCertsFromPEM(pem) - withTLS = otlptracehttp.WithTLSClientConfig( - &tls.Config{ - RootCAs: certs, - MinVersion: tls.VersionTLS12, - }) + tlsCfg := &tls.Config{ + RootCAs: certs, + MinVersion: tls.VersionTLS12, + } + + // if a TLS minimum version is supplied, set that in the config + if len(cfg.TLSMinVersion) > 0 { + var tlsVersion uint16 + + switch cfg.TLSMinVersion { + case "1.0": + tlsVersion = tls.VersionTLS10 + case "1.1": + tlsVersion = tls.VersionTLS11 + case "1.2": + tlsVersion = tls.VersionTLS12 + case "1.3": + tlsVersion = tls.VersionTLS13 + default: + return nil, fmt.Errorf("invalid TLS minimum version supplied: %s", cfg.TLSMinVersion) + } + + tlsCfg.MinVersion = tlsVersion + } + + withTLS = otlptracehttp.WithTLSClientConfig(tlsCfg) } else { logrus.Warn("no otel cert path set, exporting traces insecurely") } From 0525a143a7b2ead5f0cf10c6a885d105e6429cfe Mon Sep 17 00:00:00 2001 From: davidvader Date: Wed, 11 Sep 2024 10:51:01 -0500 Subject: [PATCH 42/59] fix: vela-server default service name --- tracing/flags.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tracing/flags.go b/tracing/flags.go index e1d9ccbb5..cb025fd6a 100644 --- a/tracing/flags.go +++ b/tracing/flags.go @@ -22,7 +22,7 @@ var Flags = []cli.Flag{ EnvVars: []string{"VELA_OTEL_TRACING_SERVICE_NAME"}, Name: "tracing.service.name", Usage: "set otel tracing service name", - Value: "go-otel-server", + Value: "vela-server", }, &cli.StringFlag{ EnvVars: []string{"VELA_OTEL_TRACING_EXPORTER_SSL_CERT_PATH"}, From 00e22c4276860fd76f406f5a38b9cabc7700d633 Mon Sep 17 00:00:00 2001 From: davidvader Date: Wed, 11 Sep 2024 10:53:17 -0500 Subject: [PATCH 43/59] chore: remove default env resource attrs --- tracing/flags.go | 1 - 1 file changed, 1 deletion(-) diff --git a/tracing/flags.go b/tracing/flags.go index cb025fd6a..bf3fb92de 100644 --- a/tracing/flags.go +++ b/tracing/flags.go @@ -54,7 +54,6 @@ var Flags = []cli.Flag{ EnvVars: []string{"VELA_OTEL_TRACING_RESOURCE_ENV_ATTRIBUTES"}, Name: "tracing.resource.env_attributes", Usage: "set otel resource attributes as a list of key=env_variable_key pairs. each one will be attached to each span as an attribute where the value is retrieved from the environment using the pair value", - Value: cli.NewStringSlice("deployment.environment=CLOUD_ENVIRONMENT"), }, // Sampler Flags From 7e5ac7eaeb099c6e9d2c79746f90fad2aa5ebf4a Mon Sep 17 00:00:00 2001 From: davidvader Date: Wed, 11 Sep 2024 12:13:42 -0500 Subject: [PATCH 44/59] enhance: refactored attribute mapping --- docker-compose.yml | 7 +--- tracing/config.go | 100 +++++++++++++++++++++++++-------------------- tracing/flags.go | 41 ++++++++----------- 3 files changed, 73 insertions(+), 75 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index f2ef7afbe..b06e45d8f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -44,12 +44,9 @@ services: VELA_ENABLE_SECURE_COOKIE: 'false' VELA_REPO_ALLOWLIST: '*' VELA_SCHEDULE_ALLOWLIST: '*' - VELA_OTEL_EXPORTER_OTLP_ENDPOINT: http://jaeger:4318 - VELA_OTEL_TRACING_ENDPOINT: jaeger:4318 VELA_OTEL_TRACING_ENABLE: true - VELA_OTEL_TRACING_SERVICE_NAME: vela-server - VELA_OTEL_TRACING_RESOURCE_ATTRIBUTES: "process.runtime.name=go" - VELA_OTEL_TRACING_RESOURCE_ENV_ATTRIBUTES: "deployment.environment=CLOUD_ENVIRONMENT" + OTEL_EXPORTER_OTLP_ENDPOINT: http://jaeger:4318 + VELA_OTEL_TRACING_ENDPOINT: jaeger:4318 VELA_OTEL_TRACING_SAMPLER_RATELIMIT_PER_SECOND: 0.2 env_file: - .env diff --git a/tracing/config.go b/tracing/config.go index aa48f3490..acdca2173 100644 --- a/tracing/config.go +++ b/tracing/config.go @@ -3,7 +3,7 @@ package tracing import ( - "fmt" + "maps" "os" "strings" @@ -19,70 +19,58 @@ type Client struct { // Config represents the configurations for otel tracing. type Config struct { - EnableTracing bool - ServiceName string - ExporterURL string - CertPath string - TLSMinVersion string - ResourceAttributes map[string]string + EnableTracing bool + ServiceName string + ExporterURL string + CertPath string + TLSMinVersion string + ResourceAttributes map[string]string + TraceStateAttributes map[string]string + SpanAttributes map[string]string Sampler } // Sampler represents the configurations for the otel sampler. // Used to determine if a trace should be sampled. type Sampler struct { - TraceStateAttributes map[string]string - SpanAttributes map[string]string - Ratio float64 - PerSecond float64 + PerSecond float64 } // FromCLIContext takes cli context and returns a tracing config to supply to traceable services. func FromCLIContext(c *cli.Context) (*Client, error) { cfg := Config{ - EnableTracing: c.Bool("tracing.enable"), - ServiceName: c.String("tracing.service.name"), - ExporterURL: c.String("tracing.exporter.endpoint"), - CertPath: c.String("tracing.exporter.cert_path"), - TLSMinVersion: c.String("tracing.tls-min-version"), - ResourceAttributes: map[string]string{}, + EnableTracing: c.Bool("tracing.enable"), + ServiceName: c.String("tracing.service.name"), + ExporterURL: c.String("tracing.exporter.endpoint"), + CertPath: c.String("tracing.exporter.cert_path"), + TLSMinVersion: c.String("tracing.tls-min-version"), + ResourceAttributes: map[string]string{}, + TraceStateAttributes: map[string]string{}, + SpanAttributes: map[string]string{}, Sampler: Sampler{ - TraceStateAttributes: map[string]string{ - "sampler": c.String("tracing.sampler.tracestate"), - }, - SpanAttributes: map[string]string{ - "w3c.tracestate": fmt.Sprintf("sampler=%s", c.String("tracing.sampler.tracestate")), - "sampler.parent": c.String("tracing.sampler.parent"), - "sampler.type": c.String("tracing.sampler.type"), - }, - Ratio: c.Float64("tracing.sampler.ratio"), PerSecond: c.Float64("tracing.sampler.persecond"), }, } - // add resource attributes - for _, attr := range c.StringSlice("tracing.resource.attributes") { - kv := strings.Split(attr, "=") - if len(kv) != 2 { - continue - } - - cfg.ResourceAttributes[kv[0]] = kv[1] + // identity func used to map a string back to itself + identityFn := func(s string) string { + return s } - // add resource attributes from environment - for _, attr := range c.StringSlice("tracing.resource.env_attributes") { - kv := strings.Split(attr, "=") - if len(kv) != 2 { - continue - } + // static span attributes + cfg.SpanAttributes = keyValueSliceToMap(c.StringSlice("tracing.span.attributes"), identityFn) - v, found := os.LookupEnv(kv[1]) - if found { - cfg.ResourceAttributes[kv[0]] = v - } - } + // static tracestate attributes + cfg.TraceStateAttributes = keyValueSliceToMap(c.StringSlice("tracing.tracestate.attributes"), identityFn) + + // static resource attributes + cfg.ResourceAttributes = keyValueSliceToMap(c.StringSlice("tracing.resource.attributes"), identityFn) + + // merge static resource attributes with those fetched from the environment + m := keyValueSliceToMap(c.StringSlice("tracing.resource.env_attributes"), os.Getenv) + maps.Copy(cfg.ResourceAttributes, m) + // initialize the tracer provider and assign it to the client tracer, err := initTracer(c.Context, cfg) if err != nil { return nil, err @@ -93,3 +81,25 @@ func FromCLIContext(c *cli.Context) (*Client, error) { TracerProvider: tracer, }, nil } + +// keyValueSliceToMap converts a slice of key=value strings to a map of key to value using the supplied map function. +func keyValueSliceToMap(kv []string, fn func(string) string) map[string]string { + m := map[string]string{} + for _, attr := range kv { + parts := strings.SplitN(attr, "=", 2) + + if len(parts) != 2 || len(parts[1]) == 0 { + continue + } + + v := fn(parts[1]) + + if len(v) == 0 { + continue + } + + m[parts[0]] = v + } + + return m +} diff --git a/tracing/flags.go b/tracing/flags.go index bf3fb92de..95f39643a 100644 --- a/tracing/flags.go +++ b/tracing/flags.go @@ -24,6 +24,11 @@ var Flags = []cli.Flag{ Usage: "set otel tracing service name", Value: "vela-server", }, + &cli.StringFlag{ + EnvVars: []string{"VELA_OTEL_TRACING_ENDPOINT"}, + Name: "tracing.exporter.endpoint", + Usage: "set the otel exporter endpoint", + }, &cli.StringFlag{ EnvVars: []string{"VELA_OTEL_TRACING_EXPORTER_SSL_CERT_PATH"}, Name: "tracing.exporter.cert_path", @@ -35,19 +40,13 @@ var Flags = []cli.Flag{ Usage: "optional TLS minimum version requirement to set when communicating with the otel exporter", Value: "1.2", }, - &cli.StringFlag{ - EnvVars: []string{"VELA_OTEL_TRACING_ENDPOINT"}, - Name: "tracing.exporter.endpoint", - Usage: "set the otel exporter endpoint", - Value: "127.0.0.1:4318", - }, // Resource Flags &cli.StringSliceFlag{ EnvVars: []string{"VELA_OTEL_TRACING_RESOURCE_ATTRIBUTES"}, Name: "tracing.resource.attributes", - Usage: "set otel resource attributes as a list of key=value pairs. each one will be attached to each span as an attribute", + Usage: "set otel resource attributes as a list of key=value pairs. each one will be attached to each span as a resource attribute", Value: cli.NewStringSlice("process.runtime.name=go"), }, &cli.StringSliceFlag{ @@ -55,27 +54,19 @@ var Flags = []cli.Flag{ Name: "tracing.resource.env_attributes", Usage: "set otel resource attributes as a list of key=env_variable_key pairs. each one will be attached to each span as an attribute where the value is retrieved from the environment using the pair value", }, + &cli.StringSliceFlag{ + EnvVars: []string{"VELA_OTEL_TRACING_SPAN_ATTRIBUTES"}, + Name: "tracing.span.attributes", + Usage: "set otel span attributes as a list of key=value pairs. each one will be attached to each span as a sampler attribute", + }, + &cli.StringSliceFlag{ + EnvVars: []string{"VELA_OTEL_TRACING_TRACESTATE_ATTRIBUTES"}, + Name: "tracing.tracestate.attributes", + Usage: "set otel tracestate attributes as a list of key=value pairs. each one will be inserted into the tracestate for each sampled span", + }, // Sampler Flags - &cli.StringFlag{ - EnvVars: []string{"VELA_OTEL_TRACING_SAMPLER_TRACESTATE"}, - Name: "tracing.sampler.tracestate", - Usage: "set otel sampler trace state attached to each span as an attribute.", - Value: "sampler.tracestate", - }, - &cli.StringFlag{ - EnvVars: []string{"VELA_OTEL_TRACING_SAMPLER_PARENT"}, - Name: "tracing.sampler.parent", - Usage: "set otel sampler parent attribute attached to each span.", - Value: "sampler.parent", - }, - &cli.StringFlag{ - EnvVars: []string{"VELA_OTEL_TRACING_SAMPLER_TYPE"}, - Name: "tracing.sampler.type", - Usage: "set otel sampler type attribute attached to each span.", - Value: "sampler.type", - }, &cli.Float64Flag{ EnvVars: []string{"VELA_OTEL_TRACING_SAMPLER_RATELIMIT_PER_SECOND"}, Name: "tracing.sampler.persecond", From 23393846a16b6f54b1c211a87f54723c0697d452 Mon Sep 17 00:00:00 2001 From: davidvader Date: Wed, 11 Sep 2024 12:15:44 -0500 Subject: [PATCH 45/59] chore: cleanup --- tracing/config.go | 2 +- tracing/flags.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tracing/config.go b/tracing/config.go index acdca2173..e0e92f8b2 100644 --- a/tracing/config.go +++ b/tracing/config.go @@ -43,7 +43,7 @@ func FromCLIContext(c *cli.Context) (*Client, error) { ServiceName: c.String("tracing.service.name"), ExporterURL: c.String("tracing.exporter.endpoint"), CertPath: c.String("tracing.exporter.cert_path"), - TLSMinVersion: c.String("tracing.tls-min-version"), + TLSMinVersion: c.String("tracing.exporter.tls-min-version"), ResourceAttributes: map[string]string{}, TraceStateAttributes: map[string]string{}, SpanAttributes: map[string]string{}, diff --git a/tracing/flags.go b/tracing/flags.go index 95f39643a..ba2e62110 100644 --- a/tracing/flags.go +++ b/tracing/flags.go @@ -36,7 +36,7 @@ var Flags = []cli.Flag{ }, &cli.StringFlag{ EnvVars: []string{"VELA_OTEL_TRACING_TLS_MIN_VERSION"}, - Name: "tracing.tls-min-version", + Name: "tracing.exporter.tls-min-version", Usage: "optional TLS minimum version requirement to set when communicating with the otel exporter", Value: "1.2", }, From 8379079914d98c02793b3f550a1bf3fc0d38f137 Mon Sep 17 00:00:00 2001 From: davidvader Date: Wed, 11 Sep 2024 12:17:56 -0500 Subject: [PATCH 46/59] chore: lint --- tracing/config.go | 1 + 1 file changed, 1 insertion(+) diff --git a/tracing/config.go b/tracing/config.go index e0e92f8b2..72b8fa7ff 100644 --- a/tracing/config.go +++ b/tracing/config.go @@ -85,6 +85,7 @@ func FromCLIContext(c *cli.Context) (*Client, error) { // keyValueSliceToMap converts a slice of key=value strings to a map of key to value using the supplied map function. func keyValueSliceToMap(kv []string, fn func(string) string) map[string]string { m := map[string]string{} + for _, attr := range kv { parts := strings.SplitN(attr, "=", 2) From e5f3a29323e84ffddb7c23a7b3533b737071717a Mon Sep 17 00:00:00 2001 From: davidvader Date: Wed, 11 Sep 2024 12:20:37 -0500 Subject: [PATCH 47/59] chore: cleanup --- tracing/config.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/tracing/config.go b/tracing/config.go index 72b8fa7ff..e0ca90169 100644 --- a/tracing/config.go +++ b/tracing/config.go @@ -57,16 +57,14 @@ func FromCLIContext(c *cli.Context) (*Client, error) { return s } - // static span attributes + // span attributes cfg.SpanAttributes = keyValueSliceToMap(c.StringSlice("tracing.span.attributes"), identityFn) - // static tracestate attributes + // tracestate attributes cfg.TraceStateAttributes = keyValueSliceToMap(c.StringSlice("tracing.tracestate.attributes"), identityFn) - // static resource attributes + // merge static resource attributes with those fetched from the environment using os.Getenv cfg.ResourceAttributes = keyValueSliceToMap(c.StringSlice("tracing.resource.attributes"), identityFn) - - // merge static resource attributes with those fetched from the environment m := keyValueSliceToMap(c.StringSlice("tracing.resource.env_attributes"), os.Getenv) maps.Copy(cfg.ResourceAttributes, m) From f03fa06ef4583c16f91b0d4ba69e16b82e51dbe3 Mon Sep 17 00:00:00 2001 From: davidvader Date: Wed, 11 Sep 2024 12:21:59 -0500 Subject: [PATCH 48/59] chore: cleanup --- tracing/config.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tracing/config.go b/tracing/config.go index e0ca90169..db1240139 100644 --- a/tracing/config.go +++ b/tracing/config.go @@ -91,13 +91,14 @@ func keyValueSliceToMap(kv []string, fn func(string) string) map[string]string { continue } + k := parts[0] v := fn(parts[1]) if len(v) == 0 { continue } - m[parts[0]] = v + m[k] = v } return m From 18ea2c406621803e74da373cb24db6eab30327b6 Mon Sep 17 00:00:00 2001 From: davidvader Date: Wed, 11 Sep 2024 12:22:33 -0500 Subject: [PATCH 49/59] chore: cleanup --- tracing/config.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tracing/config.go b/tracing/config.go index db1240139..e51fda72c 100644 --- a/tracing/config.go +++ b/tracing/config.go @@ -53,9 +53,7 @@ func FromCLIContext(c *cli.Context) (*Client, error) { } // identity func used to map a string back to itself - identityFn := func(s string) string { - return s - } + identityFn := func(s string) string { return s } // span attributes cfg.SpanAttributes = keyValueSliceToMap(c.StringSlice("tracing.span.attributes"), identityFn) From bb6b59af3f49b5b1eb93dd978713f51c03bb3cf4 Mon Sep 17 00:00:00 2001 From: davidvader Date: Wed, 11 Sep 2024 13:42:52 -0500 Subject: [PATCH 50/59] fix: better url parsing --- docker-compose.yml | 5 ++--- tracing/flags.go | 4 ++-- tracing/tracer.go | 22 +++++++++++++++++++++- 3 files changed, 25 insertions(+), 6 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index b06e45d8f..a96b4b87f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -45,9 +45,8 @@ services: VELA_REPO_ALLOWLIST: '*' VELA_SCHEDULE_ALLOWLIST: '*' VELA_OTEL_TRACING_ENABLE: true - OTEL_EXPORTER_OTLP_ENDPOINT: http://jaeger:4318 - VELA_OTEL_TRACING_ENDPOINT: jaeger:4318 - VELA_OTEL_TRACING_SAMPLER_RATELIMIT_PER_SECOND: 0.2 + VELA_OTEL_EXPORTER_OTLP_ENDPOINT: http://jaeger:4318 + VELA_OTEL_TRACING_SAMPLER_RATELIMIT_PER_SECOND: 100 env_file: - .env restart: always diff --git a/tracing/flags.go b/tracing/flags.go index ba2e62110..45261b81f 100644 --- a/tracing/flags.go +++ b/tracing/flags.go @@ -25,9 +25,9 @@ var Flags = []cli.Flag{ Value: "vela-server", }, &cli.StringFlag{ - EnvVars: []string{"VELA_OTEL_TRACING_ENDPOINT"}, + EnvVars: []string{"VELA_OTEL_EXPORTER_OTLP_ENDPOINT"}, Name: "tracing.exporter.endpoint", - Usage: "set the otel exporter endpoint", + Usage: "set the otel exporter endpoint (ex. scheme://host:port)", }, &cli.StringFlag{ EnvVars: []string{"VELA_OTEL_TRACING_EXPORTER_SSL_CERT_PATH"}, diff --git a/tracing/tracer.go b/tracing/tracer.go index a5c141641..4cc05f389 100644 --- a/tracing/tracer.go +++ b/tracing/tracer.go @@ -7,8 +7,10 @@ import ( "crypto/tls" "crypto/x509" "fmt" + "net/url" "os" + "github.com/pkg/errors" "github.com/sirupsen/logrus" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" @@ -63,8 +65,26 @@ func initTracer(ctx context.Context, cfg Config) (*sdktrace.TracerProvider, erro logrus.Warn("no otel cert path set, exporting traces insecurely") } + if len(cfg.ExporterURL) == 0 { + return nil, errors.New("no otel exporter URL set (ex. scheme://host:port)") + } + + exporterURL, err := url.Parse(cfg.ExporterURL) + if err != nil { + return nil, errors.Wrap(err, fmt.Sprintf("unable to parse otel exporter URL (ex. scheme://host:port): %s", cfg.ExporterURL)) + } + + if len(exporterURL.Hostname()) == 0 { + return nil, fmt.Errorf("unable to parse host from otel exporter URL (ex. scheme://host:port): %s", cfg.ExporterURL) + } + + exporterEndpoint := exporterURL.Hostname() + if len(exporterURL.Port()) > 0 { + exporterEndpoint = fmt.Sprintf("%s:%s", exporterEndpoint, exporterURL.Port()) + } + client := otlptracehttp.NewClient( - otlptracehttp.WithEndpoint(cfg.ExporterURL), + otlptracehttp.WithEndpoint(exporterEndpoint), withTLS, ) From 2c4fe9ec73aa02f6caa63256c0c07f2a58bc0e39 Mon Sep 17 00:00:00 2001 From: davidvader Date: Wed, 11 Sep 2024 14:05:01 -0500 Subject: [PATCH 51/59] enhance: docs links and flag clarification --- tracing/flags.go | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/tracing/flags.go b/tracing/flags.go index 45261b81f..9f7ce1cc9 100644 --- a/tracing/flags.go +++ b/tracing/flags.go @@ -12,27 +12,27 @@ var Flags = []cli.Flag{ &cli.BoolFlag{ EnvVars: []string{"VELA_OTEL_TRACING_ENABLE"}, Name: "tracing.enable", - Usage: "enable otel tracing", + Usage: "enable otel tracing. see: https://opentelemetry.io/docs/concepts/signals/traces/", Value: false, }, - - // Exporter Flags - &cli.StringFlag{ EnvVars: []string{"VELA_OTEL_TRACING_SERVICE_NAME"}, Name: "tracing.service.name", - Usage: "set otel tracing service name", + Usage: "set otel tracing service name. see: https://opentelemetry.io/docs/languages/sdk-configuration/general/", Value: "vela-server", }, + + // Exporter Flags + &cli.StringFlag{ EnvVars: []string{"VELA_OTEL_EXPORTER_OTLP_ENDPOINT"}, Name: "tracing.exporter.endpoint", - Usage: "set the otel exporter endpoint (ex. scheme://host:port)", + Usage: "set the otel exporter endpoint (ex. scheme://host:port). see: https://opentelemetry.io/docs/languages/sdk-configuration/otlp-exporter/", }, &cli.StringFlag{ EnvVars: []string{"VELA_OTEL_TRACING_EXPORTER_SSL_CERT_PATH"}, Name: "tracing.exporter.cert_path", - Usage: "set the path to certs used for communicating with the otel exporter. if not set, will use insecure communication", + Usage: "set the path to certs used for communicating with the otel exporter. if not set, will use insecure communication. see: https://opentelemetry.io/docs/specs/otel/protocol/exporter/", }, &cli.StringFlag{ EnvVars: []string{"VELA_OTEL_TRACING_TLS_MIN_VERSION"}, @@ -41,28 +41,28 @@ var Flags = []cli.Flag{ Value: "1.2", }, - // Resource Flags + // Attribute Flags &cli.StringSliceFlag{ EnvVars: []string{"VELA_OTEL_TRACING_RESOURCE_ATTRIBUTES"}, Name: "tracing.resource.attributes", - Usage: "set otel resource attributes as a list of key=value pairs. each one will be attached to each span as a resource attribute", + Usage: "set otel resource attributes as a list of key=value pairs. each one will be attached to each span as a 'process' attribute. see: https://opentelemetry.io/docs/languages/go/instrumentation/#span-attributes", Value: cli.NewStringSlice("process.runtime.name=go"), }, &cli.StringSliceFlag{ EnvVars: []string{"VELA_OTEL_TRACING_RESOURCE_ENV_ATTRIBUTES"}, Name: "tracing.resource.env_attributes", - Usage: "set otel resource attributes as a list of key=env_variable_key pairs. each one will be attached to each span as an attribute where the value is retrieved from the environment using the pair value", + Usage: "set otel resource attributes as a list of key=env_variable_key pairs. each one will be attached to each span as a 'process' attribute where the value is retrieved from the environment using the pair value. see: https://opentelemetry.io/docs/languages/go/instrumentation/#span-attributes", }, &cli.StringSliceFlag{ EnvVars: []string{"VELA_OTEL_TRACING_SPAN_ATTRIBUTES"}, Name: "tracing.span.attributes", - Usage: "set otel span attributes as a list of key=value pairs. each one will be attached to each span as a sampler attribute", + Usage: "set otel span attributes as a list of key=value pairs. each one will be attached to each span as a 'tag' attribute. see: https://opentelemetry.io/docs/languages/go/instrumentation/#span-attributes", }, &cli.StringSliceFlag{ EnvVars: []string{"VELA_OTEL_TRACING_TRACESTATE_ATTRIBUTES"}, Name: "tracing.tracestate.attributes", - Usage: "set otel tracestate attributes as a list of key=value pairs. each one will be inserted into the tracestate for each sampled span", + Usage: "set otel tracestate attributes as a list of key=value pairs. each one will be inserted into the tracestate for each sampled span. see: https://www.w3.org/TR/trace-context", }, // Sampler Flags From 6db95498a8ce9ecee766c54a9476037763fce548 Mon Sep 17 00:00:00 2001 From: davidvader Date: Wed, 11 Sep 2024 14:28:42 -0500 Subject: [PATCH 52/59] enhance: more flag clarification --- tracing/flags.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tracing/flags.go b/tracing/flags.go index 9f7ce1cc9..5e08868b6 100644 --- a/tracing/flags.go +++ b/tracing/flags.go @@ -46,13 +46,13 @@ var Flags = []cli.Flag{ &cli.StringSliceFlag{ EnvVars: []string{"VELA_OTEL_TRACING_RESOURCE_ATTRIBUTES"}, Name: "tracing.resource.attributes", - Usage: "set otel resource attributes as a list of key=value pairs. each one will be attached to each span as a 'process' attribute. see: https://opentelemetry.io/docs/languages/go/instrumentation/#span-attributes", + Usage: "set otel resource (span) attributes as a list of key=value pairs. each one will be attached to each span as a 'process' attribute. see: https://opentelemetry.io/docs/languages/go/instrumentation/#span-attributes", Value: cli.NewStringSlice("process.runtime.name=go"), }, &cli.StringSliceFlag{ EnvVars: []string{"VELA_OTEL_TRACING_RESOURCE_ENV_ATTRIBUTES"}, Name: "tracing.resource.env_attributes", - Usage: "set otel resource attributes as a list of key=env_variable_key pairs. each one will be attached to each span as a 'process' attribute where the value is retrieved from the environment using the pair value. see: https://opentelemetry.io/docs/languages/go/instrumentation/#span-attributes", + Usage: "set otel resource (span) attributes as a list of key=env_variable_key pairs. each one will be attached to each span as a 'process' attribute where the value is retrieved from the environment using the pair value. see: https://opentelemetry.io/docs/languages/go/instrumentation/#span-attributes", }, &cli.StringSliceFlag{ EnvVars: []string{"VELA_OTEL_TRACING_SPAN_ATTRIBUTES"}, @@ -62,7 +62,7 @@ var Flags = []cli.Flag{ &cli.StringSliceFlag{ EnvVars: []string{"VELA_OTEL_TRACING_TRACESTATE_ATTRIBUTES"}, Name: "tracing.tracestate.attributes", - Usage: "set otel tracestate attributes as a list of key=value pairs. each one will be inserted into the tracestate for each sampled span. see: https://www.w3.org/TR/trace-context", + Usage: "set otel tracestate (span) attributes as a list of key=value pairs. each one will be inserted into the tracestate for each sampled span. see: https://www.w3.org/TR/trace-context", }, // Sampler Flags @@ -71,6 +71,6 @@ var Flags = []cli.Flag{ EnvVars: []string{"VELA_OTEL_TRACING_SAMPLER_RATELIMIT_PER_SECOND"}, Name: "tracing.sampler.persecond", Usage: "set otel tracing head-sampler rate-limiting to N per second. see: https://opentelemetry.io/docs/concepts/sampling/", - Value: 0.2, + Value: 100, }, } From d12f845f05d8eedc8604498dd82365c929799a10 Mon Sep 17 00:00:00 2001 From: davidvader Date: Thu, 12 Sep 2024 10:50:14 -0500 Subject: [PATCH 53/59] fix: conditionally apply scm tracing --- cmd/vela-server/scm.go | 4 +++- cmd/vela-server/server.go | 2 +- scm/github/github.go | 17 +++++++++++------ scm/github/opts.go | 11 +++++++++++ scm/setup.go | 4 ++++ tracing/config.go | 21 +++++++++++++-------- 6 files changed, 43 insertions(+), 16 deletions(-) diff --git a/cmd/vela-server/scm.go b/cmd/vela-server/scm.go index c8c441442..7124761f0 100644 --- a/cmd/vela-server/scm.go +++ b/cmd/vela-server/scm.go @@ -7,10 +7,11 @@ import ( "github.com/urfave/cli/v2" "github.com/go-vela/server/scm" + "github.com/go-vela/server/tracing" ) // helper function to setup the scm from the CLI arguments. -func setupSCM(c *cli.Context) (scm.Service, error) { +func setupSCM(c *cli.Context, tc *tracing.Client) (scm.Service, error) { logrus.Debug("creating scm client from CLI configuration") // scm configuration @@ -24,6 +25,7 @@ func setupSCM(c *cli.Context) (scm.Service, error) { StatusContext: c.String("scm.context"), WebUIAddress: c.String("webui-addr"), Scopes: c.StringSlice("scm.scopes"), + Tracing: tc, } // setup the scm diff --git a/cmd/vela-server/server.go b/cmd/vela-server/server.go index 5e251c7c2..7733b9c15 100644 --- a/cmd/vela-server/server.go +++ b/cmd/vela-server/server.go @@ -108,7 +108,7 @@ func server(c *cli.Context) error { return err } - scm, err := setupSCM(c) + scm, err := setupSCM(c, tc) if err != nil { return err } diff --git a/scm/github/github.go b/scm/github/github.go index ac84e3a73..851556664 100644 --- a/scm/github/github.go +++ b/scm/github/github.go @@ -13,6 +13,8 @@ import ( "go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace" "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" "golang.org/x/oauth2" + + "github.com/go-vela/server/tracing" ) const ( @@ -53,6 +55,7 @@ type client struct { config *config OAuth *oauth2.Config AuthReq *github.AuthorizationRequest + Tracing *tracing.Client // https://pkg.go.dev/github.com/sirupsen/logrus#Entry Logger *logrus.Entry } @@ -158,12 +161,14 @@ func (c *client) newClientToken(ctx context.Context, token string) *github.Clien // } // } - tc.Transport = otelhttp.NewTransport( - tc.Transport, - otelhttp.WithClientTrace(func(ctx context.Context) *httptrace.ClientTrace { - return otelhttptrace.NewClientTrace(ctx, otelhttptrace.WithoutSubSpans()) - }), - ) + if c.Tracing.Config.EnableTracing { + tc.Transport = otelhttp.NewTransport( + tc.Transport, + otelhttp.WithClientTrace(func(ctx context.Context) *httptrace.ClientTrace { + return otelhttptrace.NewClientTrace(ctx, otelhttptrace.WithoutSubSpans()) + }), + ) + } // create the GitHub client from the OAuth client github := github.NewClient(tc) diff --git a/scm/github/opts.go b/scm/github/opts.go index df6d46506..bb7385827 100644 --- a/scm/github/opts.go +++ b/scm/github/opts.go @@ -5,6 +5,8 @@ package github import ( "fmt" "strings" + + "github.com/go-vela/server/tracing" ) // ClientOpt represents a configuration option to initialize the scm client for GitHub. @@ -149,3 +151,12 @@ func WithScopes(scopes []string) ClientOpt { return nil } } + +// WithTracing sets the shared tracing config in the scm client for GitHub. +func WithTracing(tracing *tracing.Client) ClientOpt { + return func(e *client) error { + e.Tracing = tracing + + return nil + } +} diff --git a/scm/setup.go b/scm/setup.go index 3b4082f7f..c32a0cf66 100644 --- a/scm/setup.go +++ b/scm/setup.go @@ -9,6 +9,7 @@ import ( "github.com/sirupsen/logrus" "github.com/go-vela/server/scm/github" + "github.com/go-vela/server/tracing" "github.com/go-vela/types/constants" ) @@ -36,6 +37,8 @@ type Setup struct { WebUIAddress string // specifies the OAuth scopes to use for the scm client Scopes []string + // specifies OTel tracing configurations + Tracing *tracing.Client } // Github creates and returns a Vela service capable of @@ -55,6 +58,7 @@ func (s *Setup) Github() (Service, error) { github.WithStatusContext(s.StatusContext), github.WithWebUIAddress(s.WebUIAddress), github.WithScopes(s.Scopes), + github.WithTracing(s.Tracing), ) } diff --git a/tracing/config.go b/tracing/config.go index e51fda72c..7cbb7b431 100644 --- a/tracing/config.go +++ b/tracing/config.go @@ -66,16 +66,21 @@ func FromCLIContext(c *cli.Context) (*Client, error) { m := keyValueSliceToMap(c.StringSlice("tracing.resource.env_attributes"), os.Getenv) maps.Copy(cfg.ResourceAttributes, m) - // initialize the tracer provider and assign it to the client - tracer, err := initTracer(c.Context, cfg) - if err != nil { - return nil, err + client := &Client{ + Config: cfg, } - return &Client{ - Config: cfg, - TracerProvider: tracer, - }, nil + if cfg.EnableTracing { + // initialize the tracer provider and assign it to the client + tracer, err := initTracer(c.Context, cfg) + if err != nil { + return nil, err + } + + client.TracerProvider = tracer + } + + return client, nil } // keyValueSliceToMap converts a slice of key=value strings to a map of key to value using the supplied map function. From eb0d87020ee2fc6cac61581f11a70f5a9ddfd886 Mon Sep 17 00:00:00 2001 From: davidvader Date: Thu, 12 Sep 2024 10:54:32 -0500 Subject: [PATCH 54/59] chore: tests --- scm/github/opts_test.go | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/scm/github/opts_test.go b/scm/github/opts_test.go index 40ed4da87..8a6a2617e 100644 --- a/scm/github/opts_test.go +++ b/scm/github/opts_test.go @@ -5,6 +5,8 @@ package github import ( "reflect" "testing" + + "github.com/go-vela/server/tracing" ) func TestGithub_ClientOpt_WithAddress(t *testing.T) { @@ -348,3 +350,41 @@ func TestGithub_ClientOpt_WithScopes(t *testing.T) { } } } + +func TestGithub_ClientOpt_WithTracing(t *testing.T) { + // setup tests + tests := []struct { + failure bool + tracing *tracing.Client + want *tracing.Client + }{ + { + failure: false, + tracing: &tracing.Client{}, + want: &tracing.Client{}, + }, + } + + // run tests + for _, test := range tests { + _service, err := New( + WithTracing(test.tracing), + ) + + if test.failure { + if err == nil { + t.Errorf("WithTracing should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("WithTracing returned err: %v", err) + } + + if !reflect.DeepEqual(_service.Tracing, test.want) { + t.Errorf("WithTracing is %v, want %v", _service.Tracing, test.want) + } + } +} From 7c98676f792d79f2acb8cdc5d04fb66e658991ff Mon Sep 17 00:00:00 2001 From: davidvader Date: Thu, 12 Sep 2024 12:13:18 -0500 Subject: [PATCH 55/59] fix: do not cancel goroutine context used after http response closes --- api/webhook/post.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/webhook/post.go b/api/webhook/post.go index c854c44d9..cb45e9c18 100644 --- a/api/webhook/post.go +++ b/api/webhook/post.go @@ -591,7 +591,7 @@ func PostWebhook(c *gin.Context) { // publish the build to the queue go build.Enqueue( - ctx, + context.WithoutCancel(ctx), queue.FromGinContext(c), database.FromContext(c), item, From f4a949d05308e9c1887b97e284a753662b812e92 Mon Sep 17 00:00:00 2001 From: davidvader Date: Thu, 12 Sep 2024 12:16:11 -0500 Subject: [PATCH 56/59] fix: do not cancel goroutine context used after http response closes --- api/build/approve.go | 3 ++- api/build/create.go | 3 ++- api/build/restart.go | 3 ++- cmd/vela-server/schedule.go | 2 +- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/api/build/approve.go b/api/build/approve.go index 86c491c04..61b1b6a4e 100644 --- a/api/build/approve.go +++ b/api/build/approve.go @@ -3,6 +3,7 @@ package build import ( + "context" "fmt" "net/http" "strings" @@ -110,7 +111,7 @@ func ApproveBuild(c *gin.Context) { // publish the build to the queue go Enqueue( - ctx, + context.WithoutCancel(ctx), queue.FromGinContext(c), database.FromContext(c), models.ToItem(b), diff --git a/api/build/create.go b/api/build/create.go index 8fef779c7..dd9da486b 100644 --- a/api/build/create.go +++ b/api/build/create.go @@ -3,6 +3,7 @@ package build import ( + "context" "fmt" "net/http" @@ -149,7 +150,7 @@ func CreateBuild(c *gin.Context) { // publish the build to the queue go Enqueue( - ctx, + context.WithoutCancel(ctx), queue.FromGinContext(c), database.FromContext(c), item, diff --git a/api/build/restart.go b/api/build/restart.go index aed5fccd4..24b30409d 100644 --- a/api/build/restart.go +++ b/api/build/restart.go @@ -3,6 +3,7 @@ package build import ( + "context" "fmt" "net/http" "strings" @@ -157,7 +158,7 @@ func RestartBuild(c *gin.Context) { // publish the build to the queue go Enqueue( - ctx, + context.WithoutCancel(ctx), queue.FromGinContext(c), database.FromContext(c), item, diff --git a/cmd/vela-server/schedule.go b/cmd/vela-server/schedule.go index 422dcd710..21729bed9 100644 --- a/cmd/vela-server/schedule.go +++ b/cmd/vela-server/schedule.go @@ -226,7 +226,7 @@ func processSchedule(ctx context.Context, s *api.Schedule, settings *settings.Pl // publish the build to the queue go build.Enqueue( - ctx, + context.WithoutCancel(ctx), queue, database, item, From 2139c9c8b8243ca62372f8485d585db3351225b4 Mon Sep 17 00:00:00 2001 From: davidvader Date: Thu, 12 Sep 2024 12:20:04 -0500 Subject: [PATCH 57/59] fix: avoid double responses --- api/webhook/post.go | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/api/webhook/post.go b/api/webhook/post.go index cb45e9c18..d4a847178 100644 --- a/api/webhook/post.go +++ b/api/webhook/post.go @@ -489,8 +489,6 @@ func PostWebhook(c *gin.Context) { } } - c.JSON(http.StatusCreated, b) - // regardless of whether the build is published to queue, we want to attempt to auto-cancel if no errors defer func() { if err == nil && build.ShouldAutoCancel(p.Metadata.AutoCancel, b, repo.GetBranch()) { @@ -531,6 +529,10 @@ func PostWebhook(c *gin.Context) { } }() + // track if we have already responded to the http request + // helps prevent multiple responses to the same request in the event of errors + responded := false + // if the webhook was from a Pull event from a forked repository, verify it is allowed to run if webhook.PullRequest.IsFromFork { l.Tracef("inside %s workflow for fork PR build %s/%d", repo.GetApproveBuild(), repo.GetFullName(), b.GetNumber()) @@ -540,8 +542,12 @@ func PostWebhook(c *gin.Context) { err = gatekeepBuild(c, b, repo) if err != nil { util.HandleError(c, http.StatusInternalServerError, err) + } else { + c.JSON(http.StatusCreated, b) } + responded = true + return case constants.ApproveForkNoWrite: // determine if build sender has write access to parent repo. If not, this call will result in an error @@ -550,8 +556,12 @@ func PostWebhook(c *gin.Context) { err = gatekeepBuild(c, b, repo) if err != nil { util.HandleError(c, http.StatusInternalServerError, err) + } else { + c.JSON(http.StatusCreated, b) } + responded = true + return } @@ -564,14 +574,20 @@ func PostWebhook(c *gin.Context) { contributor, err := scm.FromContext(c).RepoContributor(ctx, repo.GetOwner(), b.GetSender(), repo.GetOrg(), repo.GetName()) if err != nil { util.HandleError(c, http.StatusInternalServerError, err) + + responded = true } if !contributor { err = gatekeepBuild(c, b, repo) if err != nil { util.HandleError(c, http.StatusInternalServerError, err) + } else if !responded { + c.JSON(http.StatusCreated, b) } + responded = true + return } @@ -597,6 +613,11 @@ func PostWebhook(c *gin.Context) { item, b.GetHost(), ) + + // respond only when necessary + if !responded { + c.JSON(http.StatusCreated, b) + } } // handleRepositoryEvent is a helper function that processes repository events from the SCM and updates From da2bbeed184092c7066361d671d72589f603c5bd Mon Sep 17 00:00:00 2001 From: davidvader Date: Thu, 12 Sep 2024 12:24:22 -0500 Subject: [PATCH 58/59] chore: lint... --- api/webhook/post.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/api/webhook/post.go b/api/webhook/post.go index d4a847178..0f770021d 100644 --- a/api/webhook/post.go +++ b/api/webhook/post.go @@ -546,8 +546,6 @@ func PostWebhook(c *gin.Context) { c.JSON(http.StatusCreated, b) } - responded = true - return case constants.ApproveForkNoWrite: // determine if build sender has write access to parent repo. If not, this call will result in an error @@ -560,8 +558,6 @@ func PostWebhook(c *gin.Context) { c.JSON(http.StatusCreated, b) } - responded = true - return } @@ -586,8 +582,6 @@ func PostWebhook(c *gin.Context) { c.JSON(http.StatusCreated, b) } - responded = true - return } From 1e20dd906aa1eb0e83f9440d23231e89539e97fa Mon Sep 17 00:00:00 2001 From: davidvader Date: Thu, 12 Sep 2024 12:49:18 -0500 Subject: [PATCH 59/59] fix: scm test client --- scm/github/github.go | 1 + 1 file changed, 1 insertion(+) diff --git a/scm/github/github.go b/scm/github/github.go index 851556664..5fa96dc16 100644 --- a/scm/github/github.go +++ b/scm/github/github.go @@ -140,6 +140,7 @@ func NewTest(urls ...string) (*client, error) { WithServerWebhookAddress(""), WithStatusContext("continuous-integration/vela"), WithWebUIAddress(address), + WithTracing(&tracing.Client{Config: tracing.Config{EnableTracing: false}}), ) }