diff --git a/go.mod b/go.mod index 81bc8ac..afaca76 100644 --- a/go.mod +++ b/go.mod @@ -18,6 +18,7 @@ require ( k8s.io/apimachinery v0.22.5 k8s.io/client-go v0.22.5 k8s.io/klog v1.0.0 + k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0 ) require ( @@ -31,6 +32,7 @@ require ( github.com/docker/docker-credential-helpers v0.7.0 // indirect github.com/docker/go-connections v0.4.0 // indirect github.com/docker/go-units v0.5.0 // indirect + github.com/evanphx/json-patch v4.11.0+incompatible // indirect github.com/go-logr/logr v1.2.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/gofrs/flock v0.8.1 // indirect @@ -82,8 +84,8 @@ require ( google.golang.org/genproto v0.0.0-20220706185917-7780775163c4 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - k8s.io/klog/v2 v2.60.1 // indirect - k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 // indirect + k8s.io/klog/v2 v2.80.1 // indirect + k8s.io/kube-openapi v0.0.0-20211109043538-20434351676c // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect sigs.k8s.io/yaml v1.2.0 // indirect ) diff --git a/go.sum b/go.sum index 38e7a33..8b239b4 100644 --- a/go.sum +++ b/go.sum @@ -126,6 +126,7 @@ github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoD github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= @@ -298,13 +299,16 @@ github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2 github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.14.0 h1:2mOpI4JVVPBN+WQRa0WKH2eXR+Ey+uK4n7Zj0aYpIQA= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= @@ -641,6 +645,7 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= @@ -764,6 +769,7 @@ gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -797,13 +803,13 @@ k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= -k8s.io/klog/v2 v2.60.1 h1:VW25q3bZx9uE3vvdL6M8ezOX79vA2Aq1nEWLqNQclHc= -k8s.io/klog/v2 v2.60.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/klog/v2 v2.80.1 h1:atnLQ121W371wYYFawwYx1aEY2eUfs4l3J72wtgAwV4= +k8s.io/klog/v2 v2.80.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/kube-openapi v0.0.0-20211109043538-20434351676c h1:jvamsI1tn9V0S8jicyX82qaFC0H/NKxv2e5mbqsgR80= k8s.io/kube-openapi v0.0.0-20211109043538-20434351676c/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw= k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 h1:HNSDgDCrr/6Ly3WEGKZftiE7IY19Vz2GdbOCyI4qqhc= -k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0 h1:jgGTlFYnhF1PM1Ax/lAlxUPE+KfCIXHaathvJg1C3ak= +k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/main.go b/main.go index 0ca4189..6b99870 100644 --- a/main.go +++ b/main.go @@ -1,4 +1,4 @@ -// Copyright 2023 tsuru authors. All rights reserved. +// Copyright 2024 tsuru authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -40,7 +40,9 @@ var cfg struct { BuildKitAutoDiscoveryKubernetesPodSelector string BuildKitAutoDiscoveryKubernetesNamespace string BuildKitAutoDiscoveryKubernetesLeasePrefix string + BuildKitAutoDiscoveryStatefulset string KubernetesConfig string + BuildKitAutoDiscoveryScaleGracefulPeriod time.Duration BuildKitAutoDiscoveryTimeout time.Duration BuildKitAutoDiscoveryKubernetesPort int Port int @@ -71,6 +73,8 @@ func main() { flag.IntVar(&cfg.BuildKitAutoDiscoveryKubernetesPort, "buildkit-autodiscovery-kubernetes-port", 80, "TCP port number which BuldKit's service is listening") flag.BoolVar(&cfg.BuildKitAutoDiscoveryKubernetesSetTsuruAppLabels, "buildkit-autodiscovery-kubernetes-set-tsuru-app-labels", false, "Whether should set the Tsuru app labels in the selected BuildKit pod") flag.BoolVar(&cfg.BuildKitAutoDiscoveryKubernetesUseSameNamespaceAsTsuruApp, "buildkit-autodiscovery-kubernetes-use-same-namespace-as-tsuru-app", false, "Whether should look for BuildKit in the Tsuru app's namespace") + flag.StringVar(&cfg.BuildKitAutoDiscoveryStatefulset, "buildkit-autodiscovery-scale-statefulset", "", "Name of statefulset of buildkit that scale from zero") + flag.DurationVar(&cfg.BuildKitAutoDiscoveryScaleGracefulPeriod, "buildkit-autodiscovery-scale-graceful-period", (2 * time.Hour), "how long time after a build to retain buildkit running") flag.Parse() @@ -170,6 +174,8 @@ func newBuildKit() (*buildkit.BuildKit, error) { SetTsuruAppLabel: cfg.BuildKitAutoDiscoveryKubernetesSetTsuruAppLabels, UseSameNamespaceAsApp: cfg.BuildKitAutoDiscoveryKubernetesUseSameNamespaceAsTsuruApp, LeasePrefix: cfg.BuildKitAutoDiscoveryKubernetesLeasePrefix, + Statefulset: cfg.BuildKitAutoDiscoveryStatefulset, + ScaleGracefulPeriod: cfg.BuildKitAutoDiscoveryScaleGracefulPeriod, } return b.WithKubernetesDiscovery(cs, dcs, kdopts), nil diff --git a/pkg/build/build.go b/pkg/build/build.go index 96c0f2f..468280d 100644 --- a/pkg/build/build.go +++ b/pkg/build/build.go @@ -1,4 +1,4 @@ -// Copyright 2023 tsuru authors. All rights reserved. +// Copyright 2024 tsuru authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/pkg/build/buildkit/build.go b/pkg/build/buildkit/build.go index 26e8cd1..2bf8d4d 100644 --- a/pkg/build/buildkit/build.go +++ b/pkg/build/buildkit/build.go @@ -1,4 +1,4 @@ -// Copyright 2023 tsuru authors. All rights reserved. +// Copyright 2024 tsuru authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -38,6 +38,7 @@ import ( "k8s.io/client-go/kubernetes" "github.com/tsuru/deploy-agent/pkg/build" + "github.com/tsuru/deploy-agent/pkg/build/buildkit/scaler" pb "github.com/tsuru/deploy-agent/pkg/build/grpc_build_v1" "github.com/tsuru/deploy-agent/pkg/util" ) @@ -66,9 +67,11 @@ type KubernertesDiscoveryOptions struct { PodSelector string Namespace string LeasePrefix string + Statefulset string Port int UseSameNamespaceAsApp bool SetTsuruAppLabel bool + ScaleGracefulPeriod time.Duration Timeout time.Duration } @@ -76,6 +79,11 @@ func (b *BuildKit) WithKubernetesDiscovery(cs *kubernetes.Clientset, dcs dynamic b.k8s = cs b.dk8s = dcs b.kdopts = &opts + + if opts.Statefulset != "" { + scaler.StartWorker(cs, opts.PodSelector, opts.Statefulset, opts.ScaleGracefulPeriod) + } + return b } @@ -100,7 +108,7 @@ func (b *BuildKit) Build(ctx context.Context, r *pb.BuildRequest, w io.Writer) ( return nil, errors.New("writer must implement console.File") } - c, clientCleanUp, err := b.client(ctx, r) + c, clientCleanUp, err := b.client(ctx, r, w) if err != nil { return nil, err } @@ -539,7 +547,7 @@ func callBuildKitToExtractTsuruConfigs(ctx context.Context, c *client.Client, lo return tc, nil } -func (b *BuildKit) client(ctx context.Context, req *pb.BuildRequest) (*client.Client, func(), error) { +func (b *BuildKit) client(ctx context.Context, req *pb.BuildRequest, w io.Writer) (*client.Client, func(), error) { isBuildForApp := strings.HasPrefix(pb.BuildKind_name[int32(req.Kind)], "BUILD_KIND_APP_") if isBuildForApp && b.opts.DiscoverBuildKitClientForApp { @@ -547,7 +555,7 @@ func (b *BuildKit) client(ctx context.Context, req *pb.BuildRequest) (*client.Cl cs: b.k8s, dcs: b.dk8s, } - return d.Discover(ctx, *b.kdopts, req) + return d.Discover(ctx, *b.kdopts, req, w) } return b.cli, noopFunc, nil diff --git a/pkg/build/buildkit/build_test.go b/pkg/build/buildkit/build_test.go index 1791c5e..780249e 100644 --- a/pkg/build/buildkit/build_test.go +++ b/pkg/build/buildkit/build_test.go @@ -1,4 +1,4 @@ -// Copyright 2023 tsuru authors. All rights reserved. +// Copyright 2024 tsuru authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/pkg/build/buildkit/k8s_autodiscovery.go b/pkg/build/buildkit/k8s_autodiscovery.go index b48cbe7..71b6199 100644 --- a/pkg/build/buildkit/k8s_autodiscovery.go +++ b/pkg/build/buildkit/k8s_autodiscovery.go @@ -1,4 +1,4 @@ -// Copyright 2023 tsuru authors. All rights reserved. +// Copyright 2024 tsuru authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -8,13 +8,16 @@ import ( "context" "encoding/json" "fmt" + "io" "os" "strconv" "strings" "time" "github.com/moby/buildkit/client" + "github.com/tsuru/deploy-agent/pkg/build/buildkit/scaler" pb "github.com/tsuru/deploy-agent/pkg/build/grpc_build_v1" + "github.com/tsuru/deploy-agent/pkg/build/metadata" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -28,15 +31,6 @@ import ( "k8s.io/klog" ) -const ( - DeployAgentLastBuildStartingLabelKey = "deploy-agent.tsuru.io/last-build-starting-time" - DeployAgentLastBuildEndingTimeLabelKey = "deploy-agent.tsuru.io/last-build-ending-time" - - TsuruAppNamespace = "tsuru" - TsuruAppNameLabelKey = "tsuru.io/app-name" - TsuruIsBuildLabelKey = "tsuru.io/is-build" -) - var ( noopFunc = func() {} @@ -52,15 +46,15 @@ type k8sDiscoverer struct { dcs dynamic.Interface } -func (d *k8sDiscoverer) Discover(ctx context.Context, opts KubernertesDiscoveryOptions, req *pb.BuildRequest) (*client.Client, func(), error) { +func (d *k8sDiscoverer) Discover(ctx context.Context, opts KubernertesDiscoveryOptions, req *pb.BuildRequest, w io.Writer) (*client.Client, func(), error) { if req.App == nil { return nil, noopFunc, fmt.Errorf("there's only support for discovering BuildKit pods from Tsuru apps") } - return d.discoverBuildKitClientFromApp(ctx, opts, req.App.Name) + return d.discoverBuildKitClientFromApp(ctx, opts, req.App.Name, w) } -func (d *k8sDiscoverer) discoverBuildKitClientFromApp(ctx context.Context, opts KubernertesDiscoveryOptions, app string) (*client.Client, func(), error) { +func (d *k8sDiscoverer) discoverBuildKitClientFromApp(ctx context.Context, opts KubernertesDiscoveryOptions, app string, w io.Writer) (*client.Client, func(), error) { leaderCtx, leaderCancel := context.WithCancel(ctx) cfns := []func(){ func() { @@ -69,7 +63,7 @@ func (d *k8sDiscoverer) discoverBuildKitClientFromApp(ctx context.Context, opts }, } - pod, err := d.discoverBuildKitPod(leaderCtx, opts, app) + pod, err := d.discoverBuildKitPod(leaderCtx, opts, app, w) if err != nil { return nil, cleanUps(cfns...), err } @@ -84,7 +78,7 @@ func (d *k8sDiscoverer) discoverBuildKitClientFromApp(ctx context.Context, opts cfns = append(cfns, func() { klog.V(4).Infoln("Removing Tsuru app labels in the pod", pod.Name) - nerr := unsetTsuruAppLabelOnBuildKitPod(context.Background(), d.cs, pod.Name, pod.Namespace) + nerr := unsetTsuruAppLabelOnBuildKitPod(ctx, d.cs, pod.Name, pod.Namespace) if nerr != nil { klog.Errorf("failed to unset Tsuru app labels: %s", nerr) } @@ -108,7 +102,7 @@ func (d *k8sDiscoverer) discoverBuildKitClientFromApp(ctx context.Context, opts return c, cleanUps(cfns...), nil } -func (d *k8sDiscoverer) discoverBuildKitPod(ctx context.Context, opts KubernertesDiscoveryOptions, app string) (*corev1.Pod, error) { +func (d *k8sDiscoverer) discoverBuildKitPod(ctx context.Context, opts KubernertesDiscoveryOptions, app string, w io.Writer) (*corev1.Pod, error) { deadlineCtx, deadlineCancel := context.WithCancel(ctx) defer deadlineCancel() @@ -127,7 +121,7 @@ func (d *k8sDiscoverer) discoverBuildKitPod(ctx context.Context, opts Kubernerte defer watchCancel() // watch cancellation must happen before than closing the pods channel go func() { - nerr := watchBuildKitPods(watchCtx, d.cs, opts.PodSelector, ns, pods) + nerr := watchBuildKitPods(watchCtx, d.cs, opts, ns, pods, w) if nerr != nil { errCh <- nerr } @@ -188,7 +182,7 @@ func (d *k8sDiscoverer) buildkitPodNamespace(ctx context.Context, opts Kubernert klog.V(4).Infof("Discovering the namespace where app %s is running on...", app) - tsuruApp, err := d.dcs.Resource(tsuruAppGVR).Namespace(TsuruAppNamespace).Get(ctx, app, metav1.GetOptions{}) + tsuruApp, err := d.dcs.Resource(tsuruAppGVR).Namespace(metadata.TsuruAppNamespace).Get(ctx, app, metav1.GetOptions{}) if err != nil { return "", err } @@ -208,9 +202,16 @@ func (d *k8sDiscoverer) buildkitPodNamespace(ctx context.Context, opts Kubernert return ns, nil } -func watchBuildKitPods(ctx context.Context, cs *kubernetes.Clientset, labelSelector, ns string, pods chan<- *corev1.Pod) error { +func watchBuildKitPods(ctx context.Context, cs *kubernetes.Clientset, opts KubernertesDiscoveryOptions, ns string, pods chan<- *corev1.Pod, writer io.Writer) error { + if opts.Statefulset != "" { + scaleErr := scaler.MayUpscale(ctx, cs, ns, opts.Statefulset, writer) + if scaleErr != nil { + return scaleErr + } + } + w, err := cs.CoreV1().Pods(ns).Watch(ctx, metav1.ListOptions{ - LabelSelector: labelSelector, + LabelSelector: opts.PodSelector, Watch: true, }) if err != nil { @@ -290,22 +291,22 @@ func setTsuruAppLabelOnBuildKitPod(ctx context.Context, cs *kubernetes.Clientset patch, err := json.Marshal([]any{ map[string]any{ "op": "replace", - "path": fmt.Sprintf("/metadata/labels/%s", normalizeAppLabelForJSONPatch(TsuruAppNameLabelKey)), + "path": fmt.Sprintf("/metadata/labels/%s", normalizeAppLabelForJSONPatch(metadata.TsuruAppNameLabelKey)), "value": app, }, map[string]any{ "op": "replace", - "path": fmt.Sprintf("/metadata/labels/%s", normalizeAppLabelForJSONPatch(TsuruIsBuildLabelKey)), + "path": fmt.Sprintf("/metadata/labels/%s", normalizeAppLabelForJSONPatch(metadata.TsuruIsBuildLabelKey)), "value": strconv.FormatBool(true), }, map[string]any{ "op": "replace", - "path": fmt.Sprintf("/metadata/annotations/%s", normalizeAppLabelForJSONPatch(DeployAgentLastBuildEndingTimeLabelKey)), + "path": fmt.Sprintf("/metadata/annotations/%s", normalizeAppLabelForJSONPatch(metadata.DeployAgentLastBuildEndingTimeLabelKey)), "value": "", // set annotation value to empty rather than removing it, since it might not exist at first run }, map[string]any{ "op": "replace", - "path": fmt.Sprintf("/metadata/annotations/%s", normalizeAppLabelForJSONPatch(DeployAgentLastBuildStartingLabelKey)), + "path": fmt.Sprintf("/metadata/annotations/%s", normalizeAppLabelForJSONPatch(metadata.DeployAgentLastBuildStartingLabelKey)), "value": strconv.FormatInt(time.Now().Unix(), 10), }, }) @@ -321,15 +322,15 @@ func unsetTsuruAppLabelOnBuildKitPod(ctx context.Context, cs *kubernetes.Clients patch, err := json.Marshal([]any{ map[string]any{ "op": "remove", - "path": fmt.Sprintf("/metadata/labels/%s", normalizeAppLabelForJSONPatch(TsuruAppNameLabelKey)), + "path": fmt.Sprintf("/metadata/labels/%s", normalizeAppLabelForJSONPatch(metadata.TsuruAppNameLabelKey)), }, map[string]any{ "op": "remove", - "path": fmt.Sprintf("/metadata/labels/%s", normalizeAppLabelForJSONPatch(TsuruIsBuildLabelKey)), + "path": fmt.Sprintf("/metadata/labels/%s", normalizeAppLabelForJSONPatch(metadata.TsuruIsBuildLabelKey)), }, map[string]any{ "op": "replace", - "path": fmt.Sprintf("/metadata/annotations/%s", normalizeAppLabelForJSONPatch(DeployAgentLastBuildEndingTimeLabelKey)), + "path": fmt.Sprintf("/metadata/annotations/%s", normalizeAppLabelForJSONPatch(metadata.DeployAgentLastBuildEndingTimeLabelKey)), "value": strconv.FormatInt(time.Now().Unix(), 10), }, }) diff --git a/pkg/build/buildkit/scaler/downscaler.go b/pkg/build/buildkit/scaler/downscaler.go new file mode 100644 index 0000000..3684b8b --- /dev/null +++ b/pkg/build/buildkit/scaler/downscaler.go @@ -0,0 +1,106 @@ +// Copyright 2024 tsuru authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package scaler + +import ( + "context" + "fmt" + "strconv" + "time" + + "github.com/tsuru/deploy-agent/pkg/build/metadata" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/klog" +) + +func StartWorker(clientset *kubernetes.Clientset, podSelector, statefulSet string, graceful time.Duration) { + ctx := context.Background() + + go func() { + for { + err := RunDownscaler(ctx, clientset, podSelector, statefulSet, graceful) + if err != nil { + klog.Errorf("failed to run downscaler tick: %s", err.Error()) + } + time.Sleep(time.Minute * 5) + } + }() +} + +func RunDownscaler(ctx context.Context, clientset kubernetes.Interface, podSelector, statefulSet string, graceful time.Duration) (err error) { + defer func() { + recoverErr := recover() + if recoverErr != nil { + err = fmt.Errorf("panic: %s", recoverErr) + } + }() + + buildKitPods, err := clientset.CoreV1().Pods("").List(ctx, v1.ListOptions{ + LabelSelector: podSelector, + }) + + if err != nil { + return err + } + + maxEndtimeByNS := map[string]int64{} + + for _, pod := range buildKitPods.Items { + if pod.Annotations[metadata.DeployAgentLastBuildEndingTimeLabelKey] == "" { + maxEndtimeByNS[pod.Namespace] = -1 // mark that namespace has least one pod of buildkit running + continue + } + + maxUsage, err := strconv.ParseInt(pod.Annotations[metadata.DeployAgentLastBuildEndingTimeLabelKey], 10, 64) + if err != nil { + klog.Errorf("failed to parseint: %s", err.Error()) + continue + } + + if maxEndtimeByNS[pod.Namespace] == -1 { + continue + } + + if maxEndtimeByNS[pod.Namespace] < maxUsage { + maxEndtimeByNS[pod.Namespace] = maxUsage + } + } + + now := time.Now().Unix() + gracefulPeriod := int64(graceful.Seconds()) + zero := int32(0) + + for ns, maxEndtime := range maxEndtimeByNS { + if maxEndtime == -1 { + continue + } + + if now-maxEndtime < gracefulPeriod { + continue + } + + statefulset, err := clientset.AppsV1().StatefulSets(ns).Get(ctx, statefulSet, v1.GetOptions{}) + + if err != nil { + klog.Errorf("failed to get statefullsets from ns: %s, err: %s", ns, err.Error()) + continue + } + + if statefulset.Spec.Replicas != nil { + statefulset.Annotations[metadata.DeployAgentLastReplicasAnnotationKey] = fmt.Sprintf("%d", *statefulset.Spec.Replicas) + } + + statefulset.Spec.Replicas = &zero + + _, err = clientset.AppsV1().StatefulSets(ns).Update(ctx, statefulset, v1.UpdateOptions{}) + if err != nil { + klog.Errorf("failed to update statefullsets from ns: %s, err: %s", ns, err.Error()) + continue + } + } + + return nil +} diff --git a/pkg/build/buildkit/scaler/downscaler_test.go b/pkg/build/buildkit/scaler/downscaler_test.go new file mode 100644 index 0000000..807f5ab --- /dev/null +++ b/pkg/build/buildkit/scaler/downscaler_test.go @@ -0,0 +1,229 @@ +// Copyright 2024 tsuru authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package scaler + +import ( + "context" + "strconv" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/tsuru/deploy-agent/pkg/build/metadata" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" + "k8s.io/utils/ptr" +) + +var testGraceful = time.Hour * 2 + +func TestRunDownscaler(t *testing.T) { + ctx := context.Background() + + lastBuild := time.Now().Add(-3 * time.Hour).Unix() + + cli := fake.NewSimpleClientset( + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "buildkit-0", + Namespace: "default", + Labels: map[string]string{ + "app": "buildkit", + }, + + Annotations: map[string]string{ + metadata.DeployAgentLastBuildEndingTimeLabelKey: strconv.Itoa(int(lastBuild)), + }, + }, + Spec: corev1.PodSpec{}, + }, + &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "buildkit", + Namespace: "default", + + Annotations: map[string]string{ + metadata.DeployAgentLastReplicasAnnotationKey: "3", + }, + }, + Spec: appsv1.StatefulSetSpec{ + Replicas: ptr.To(int32(1)), + }, + }, + ) + + err := RunDownscaler(ctx, cli, "app=buildkit", "buildkit", testGraceful) + assert.NoError(t, err) + + rs, err := cli.AppsV1().StatefulSets("").List(ctx, metav1.ListOptions{}) + assert.NoError(t, err) + + require.Len(t, rs.Items, 1) + assert.Equal(t, int32(0), *rs.Items[0].Spec.Replicas) +} + +func TestRunDownscalerWithEarlyBuild(t *testing.T) { + ctx := context.Background() + + lastBuild := time.Now().Add(-30 * time.Minute).Unix() + + cli := fake.NewSimpleClientset( + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "buildkit-0", + Namespace: "default", + Labels: map[string]string{ + "app": "buildkit", + }, + + Annotations: map[string]string{ + metadata.DeployAgentLastBuildEndingTimeLabelKey: strconv.Itoa(int(lastBuild)), + }, + }, + Spec: corev1.PodSpec{}, + }, + &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "buildkit", + Namespace: "default", + + Annotations: map[string]string{ + metadata.DeployAgentLastReplicasAnnotationKey: "3", + }, + }, + Spec: appsv1.StatefulSetSpec{ + Replicas: ptr.To(int32(1)), + }, + }, + ) + + err := RunDownscaler(ctx, cli, "app=buildkit", "buildkit", testGraceful) + assert.NoError(t, err) + + rs, err := cli.AppsV1().StatefulSets("").List(ctx, metav1.ListOptions{}) + assert.NoError(t, err) + + require.Len(t, rs.Items, 1) + assert.Equal(t, int32(1), *rs.Items[0].Spec.Replicas) +} + +func TestRunDownscalerWithOnePodBuilding(t *testing.T) { + ctx := context.Background() + + lastBuild := time.Now().Add(-3 * time.Hour).Unix() + + cli := fake.NewSimpleClientset( + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "buildkit-0", + Namespace: "default", + Labels: map[string]string{ + "app": "buildkit", + }, + + Annotations: map[string]string{ + metadata.DeployAgentLastBuildEndingTimeLabelKey: strconv.Itoa(int(lastBuild)), + }, + }, + Spec: corev1.PodSpec{}, + }, + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "buildkit-1", + Namespace: "default", + Labels: map[string]string{ + "app": "buildkit", + }, + + Annotations: map[string]string{}, // this pod is building for some app + }, + Spec: corev1.PodSpec{}, + }, + &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "buildkit", + Namespace: "default", + + Annotations: map[string]string{ + metadata.DeployAgentLastReplicasAnnotationKey: "3", + }, + }, + Spec: appsv1.StatefulSetSpec{ + Replicas: ptr.To(int32(2)), + }, + }, + ) + + err := RunDownscaler(ctx, cli, "app=buildkit", "buildkit", testGraceful) + assert.NoError(t, err) + + rs, err := cli.AppsV1().StatefulSets("").List(ctx, metav1.ListOptions{}) + assert.NoError(t, err) + + require.Len(t, rs.Items, 1) + assert.Equal(t, int32(2), *rs.Items[0].Spec.Replicas) +} + +func TestRunDownscalerWithManyPods(t *testing.T) { + ctx := context.Background() + + lastBuild := time.Now().Add(-3 * time.Hour).Unix() + + cli := fake.NewSimpleClientset( + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "buildkit-0", + Namespace: "default", + Labels: map[string]string{ + "app": "buildkit", + }, + + Annotations: map[string]string{ + metadata.DeployAgentLastBuildEndingTimeLabelKey: strconv.Itoa(int(lastBuild)), + }, + }, + Spec: corev1.PodSpec{}, + }, + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "buildkit-1", + Namespace: "default", + Labels: map[string]string{ + "app": "buildkit", + }, + + Annotations: map[string]string{ + metadata.DeployAgentLastBuildEndingTimeLabelKey: strconv.Itoa(int(lastBuild)), + }, + }, + Spec: corev1.PodSpec{}, + }, + &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "buildkit", + Namespace: "default", + + Annotations: map[string]string{ + metadata.DeployAgentLastReplicasAnnotationKey: "3", + }, + }, + Spec: appsv1.StatefulSetSpec{ + Replicas: ptr.To(int32(2)), + }, + }, + ) + + err := RunDownscaler(ctx, cli, "app=buildkit", "buildkit", testGraceful) + assert.NoError(t, err) + + rs, err := cli.AppsV1().StatefulSets("").List(ctx, metav1.ListOptions{}) + assert.NoError(t, err) + + require.Len(t, rs.Items, 1) + assert.Equal(t, int32(0), *rs.Items[0].Spec.Replicas) +} diff --git a/pkg/build/buildkit/scaler/upscaler.go b/pkg/build/buildkit/scaler/upscaler.go new file mode 100644 index 0000000..8c540d2 --- /dev/null +++ b/pkg/build/buildkit/scaler/upscaler.go @@ -0,0 +1,49 @@ +// Copyright 2024 tsuru authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package scaler + +import ( + "context" + "fmt" + "io" + "strconv" + + "github.com/tsuru/deploy-agent/pkg/build/metadata" + "k8s.io/client-go/kubernetes" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func MayUpscale(ctx context.Context, cs kubernetes.Interface, ns, statefulset string, w io.Writer) error { + stfullset, err := cs.AppsV1().StatefulSets(ns).Get(ctx, statefulset, metav1.GetOptions{}) + if err != nil { + return err + } + + if stfullset.Spec.Replicas != nil && *stfullset.Spec.Replicas > 0 { + return nil + } + + wantedReplicas := int32(1) + + if lastReplicas := stfullset.Annotations[metadata.DeployAgentLastReplicasAnnotationKey]; lastReplicas != "" { + var replicas int64 + replicas, err = strconv.ParseInt(lastReplicas, 10, 32) + if err != nil { + return err + } + wantedReplicas = int32(replicas) + } + + fmt.Fprintln(w, "There is no buildkits available, scaling to one replica") + stfullset.Spec.Replicas = &wantedReplicas + + _, err = cs.AppsV1().StatefulSets(ns).Update(ctx, stfullset, metav1.UpdateOptions{}) + if err != nil { + return err + } + + return nil +} diff --git a/pkg/build/buildkit/scaler/upscaler_test.go b/pkg/build/buildkit/scaler/upscaler_test.go new file mode 100644 index 0000000..e3de231 --- /dev/null +++ b/pkg/build/buildkit/scaler/upscaler_test.go @@ -0,0 +1,96 @@ +// Copyright 2024 tsuru authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. +package scaler + +import ( + "bytes" + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/tsuru/deploy-agent/pkg/build/metadata" + appsv1 "k8s.io/api/apps/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" + "k8s.io/utils/ptr" +) + +func TestMayScaleStatefulsetSkipScale(t *testing.T) { + ctx := context.Background() + + cli := fake.NewSimpleClientset(&appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "buildkit", + Namespace: "default", + }, + Spec: appsv1.StatefulSetSpec{ + Replicas: ptr.To(int32(10)), + }, + }) + + buf := bytes.Buffer{} + + err := MayUpscale(ctx, cli, "default", "buildkit", &buf) + + assert.Equal(t, "", buf.String()) + assert.NoError(t, err) +} +func TestMayScaleStatefulsetScale(t *testing.T) { + ctx := context.Background() + + cli := fake.NewSimpleClientset(&appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "buildkit", + Namespace: "default", + }, + Spec: appsv1.StatefulSetSpec{ + Replicas: ptr.To(int32(0)), + }, + }) + + buf := bytes.Buffer{} + + err := MayUpscale(ctx, cli, "default", "buildkit", &buf) + + assert.Equal(t, "There is no buildkits available, scaling to one replica\n", buf.String()) + assert.NoError(t, err) + + rs, err := cli.AppsV1().StatefulSets("").List(ctx, metav1.ListOptions{}) + assert.NoError(t, err) + + require.Len(t, rs.Items, 1) + assert.Equal(t, int32(1), *rs.Items[0].Spec.Replicas) +} + +func TestMayScaleStatefulsetScaleFromPreviousReplicas(t *testing.T) { + ctx := context.Background() + + cli := fake.NewSimpleClientset(&appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "buildkit", + Namespace: "default", + + Annotations: map[string]string{ + metadata.DeployAgentLastReplicasAnnotationKey: "3", + }, + }, + Spec: appsv1.StatefulSetSpec{ + Replicas: ptr.To(int32(0)), + }, + }) + + buf := bytes.Buffer{} + + err := MayUpscale(ctx, cli, "default", "buildkit", &buf) + + assert.Equal(t, "There is no buildkits available, scaling to one replica\n", buf.String()) + assert.NoError(t, err) + + rs, err := cli.AppsV1().StatefulSets("").List(ctx, metav1.ListOptions{}) + assert.NoError(t, err) + + require.Len(t, rs.Items, 1) + assert.Equal(t, int32(3), *rs.Items[0].Spec.Replicas) +} diff --git a/pkg/build/fake/build.go b/pkg/build/fake/build.go index 7f83818..635453b 100644 --- a/pkg/build/fake/build.go +++ b/pkg/build/fake/build.go @@ -1,4 +1,4 @@ -// Copyright 2023 tsuru authors. All rights reserved. +// Copyright 2024 tsuru authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/pkg/build/grpc_build_v1/build_service.pb.go b/pkg/build/grpc_build_v1/build_service.pb.go index c00c8e1..5d706b2 100644 --- a/pkg/build/grpc_build_v1/build_service.pb.go +++ b/pkg/build/grpc_build_v1/build_service.pb.go @@ -1,4 +1,4 @@ -// Copyright 2023 tsuru authors. All rights reserved. +// Copyright 2024 tsuru authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/pkg/build/grpc_build_v1/build_service.proto b/pkg/build/grpc_build_v1/build_service.proto index a695b26..42aac4b 100644 --- a/pkg/build/grpc_build_v1/build_service.proto +++ b/pkg/build/grpc_build_v1/build_service.proto @@ -1,4 +1,4 @@ -// Copyright 2023 tsuru authors. All rights reserved. +// Copyright 2024 tsuru authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/pkg/build/helpers.go b/pkg/build/helpers.go index 3527781..b8fe661 100644 --- a/pkg/build/helpers.go +++ b/pkg/build/helpers.go @@ -1,4 +1,4 @@ -// Copyright 2023 tsuru authors. All rights reserved. +// Copyright 2024 tsuru authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/pkg/build/helpers_test.go b/pkg/build/helpers_test.go index 661272c..421c13b 100644 --- a/pkg/build/helpers_test.go +++ b/pkg/build/helpers_test.go @@ -1,4 +1,4 @@ -// Copyright 2023 tsuru authors. All rights reserved. +// Copyright 2024 tsuru authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/pkg/build/metadata/metadata.go b/pkg/build/metadata/metadata.go new file mode 100644 index 0000000..61e08e6 --- /dev/null +++ b/pkg/build/metadata/metadata.go @@ -0,0 +1,15 @@ +// Copyright 2024 tsuru authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package metadata + +const ( + DeployAgentLastReplicasAnnotationKey = "deploy-agent.tsuru.io/last-replicas" + DeployAgentLastBuildStartingLabelKey = "deploy-agent.tsuru.io/last-build-starting-time" + DeployAgentLastBuildEndingTimeLabelKey = "deploy-agent.tsuru.io/last-build-ending-time" + + TsuruAppNamespace = "tsuru" + TsuruAppNameLabelKey = "tsuru.io/app-name" + TsuruIsBuildLabelKey = "tsuru.io/is-build" +) diff --git a/pkg/build/server.go b/pkg/build/server.go index 95f9759..a8dd4b7 100644 --- a/pkg/build/server.go +++ b/pkg/build/server.go @@ -1,4 +1,4 @@ -// Copyright 2023 tsuru authors. All rights reserved. +// Copyright 2024 tsuru authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/pkg/build/server_test.go b/pkg/build/server_test.go index e88259c..8da5bf7 100644 --- a/pkg/build/server_test.go +++ b/pkg/build/server_test.go @@ -1,4 +1,4 @@ -// Copyright 2023 tsuru authors. All rights reserved. +// Copyright 2024 tsuru authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/pkg/build/types.go b/pkg/build/types.go index ec2cf8a..0e438a5 100644 --- a/pkg/build/types.go +++ b/pkg/build/types.go @@ -1,4 +1,4 @@ -// Copyright 2023 tsuru authors. All rights reserved. +// Copyright 2024 tsuru authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/pkg/health/server.go b/pkg/health/server.go index 4fa1619..2995c29 100644 --- a/pkg/health/server.go +++ b/pkg/health/server.go @@ -1,4 +1,4 @@ -// Copyright 2023 tsuru authors. All rights reserved. +// Copyright 2024 tsuru authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/pkg/util/gzip.go b/pkg/util/gzip.go index f15b819..9ea18a3 100644 --- a/pkg/util/gzip.go +++ b/pkg/util/gzip.go @@ -1,4 +1,4 @@ -// Copyright 2023 tsuru authors. All rights reserved. +// Copyright 2024 tsuru authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file.