Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

scale buildkit to zero #40

Merged
merged 7 commits into from
May 7, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ require (
k8s.io/apimachinery v0.22.5
k8s.io/client-go v0.22.5
k8s.io/klog v1.0.0
k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0
)

require (
Expand All @@ -31,6 +32,7 @@ require (
github.com/docker/docker-credential-helpers v0.7.0 // indirect
github.com/docker/go-connections v0.4.0 // indirect
github.com/docker/go-units v0.5.0 // indirect
github.com/evanphx/json-patch v4.11.0+incompatible // indirect
github.com/go-logr/logr v1.2.3 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/gofrs/flock v0.8.1 // indirect
Expand Down Expand Up @@ -82,8 +84,8 @@ require (
google.golang.org/genproto v0.0.0-20220706185917-7780775163c4 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
k8s.io/klog/v2 v2.60.1 // indirect
k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 // indirect
k8s.io/klog/v2 v2.80.1 // indirect
k8s.io/kube-openapi v0.0.0-20211109043538-20434351676c // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
sigs.k8s.io/yaml v1.2.0 // indirect
)
Expand Down
14 changes: 10 additions & 4 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,7 @@ github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoD
github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
Expand Down Expand Up @@ -298,13 +299,16 @@ github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2
github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78=
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
github.com/onsi/ginkgo v1.14.0 h1:2mOpI4JVVPBN+WQRa0WKH2eXR+Ey+uK4n7Zj0aYpIQA=
github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE=
github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
Expand Down Expand Up @@ -641,6 +645,7 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
Expand Down Expand Up @@ -764,6 +769,7 @@ gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
Expand Down Expand Up @@ -797,13 +803,13 @@ k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8=
k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I=
k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE=
k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec=
k8s.io/klog/v2 v2.60.1 h1:VW25q3bZx9uE3vvdL6M8ezOX79vA2Aq1nEWLqNQclHc=
k8s.io/klog/v2 v2.60.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
k8s.io/klog/v2 v2.80.1 h1:atnLQ121W371wYYFawwYx1aEY2eUfs4l3J72wtgAwV4=
k8s.io/klog/v2 v2.80.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
k8s.io/kube-openapi v0.0.0-20211109043538-20434351676c h1:jvamsI1tn9V0S8jicyX82qaFC0H/NKxv2e5mbqsgR80=
k8s.io/kube-openapi v0.0.0-20211109043538-20434351676c/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw=
k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 h1:HNSDgDCrr/6Ly3WEGKZftiE7IY19Vz2GdbOCyI4qqhc=
k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0 h1:jgGTlFYnhF1PM1Ax/lAlxUPE+KfCIXHaathvJg1C3ak=
k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
Expand Down
8 changes: 7 additions & 1 deletion main.go
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright 2023 tsuru authors. All rights reserved.
// Copyright 2024 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

Expand Down Expand Up @@ -40,7 +40,9 @@ var cfg struct {
BuildKitAutoDiscoveryKubernetesPodSelector string
BuildKitAutoDiscoveryKubernetesNamespace string
BuildKitAutoDiscoveryKubernetesLeasePrefix string
BuildKitAutoDiscoveryStatefulset string
KubernetesConfig string
BuildKitAutoDiscoveryScaleGracefulPeriod time.Duration
BuildKitAutoDiscoveryTimeout time.Duration
BuildKitAutoDiscoveryKubernetesPort int
Port int
Expand Down Expand Up @@ -71,6 +73,8 @@ func main() {
flag.IntVar(&cfg.BuildKitAutoDiscoveryKubernetesPort, "buildkit-autodiscovery-kubernetes-port", 80, "TCP port number which BuldKit's service is listening")
flag.BoolVar(&cfg.BuildKitAutoDiscoveryKubernetesSetTsuruAppLabels, "buildkit-autodiscovery-kubernetes-set-tsuru-app-labels", false, "Whether should set the Tsuru app labels in the selected BuildKit pod")
flag.BoolVar(&cfg.BuildKitAutoDiscoveryKubernetesUseSameNamespaceAsTsuruApp, "buildkit-autodiscovery-kubernetes-use-same-namespace-as-tsuru-app", false, "Whether should look for BuildKit in the Tsuru app's namespace")
flag.StringVar(&cfg.BuildKitAutoDiscoveryStatefulset, "buildkit-autodiscovery-scale-statefulset", "", "Name of statefulset of buildkit that scale from zero")
flag.DurationVar(&cfg.BuildKitAutoDiscoveryScaleGracefulPeriod, "buildkit-autodiscovery-scale-graceful-period", (2 * time.Hour), "how long time after a build to retain buildkit running")

flag.Parse()

Expand Down Expand Up @@ -170,6 +174,8 @@ func newBuildKit() (*buildkit.BuildKit, error) {
SetTsuruAppLabel: cfg.BuildKitAutoDiscoveryKubernetesSetTsuruAppLabels,
UseSameNamespaceAsApp: cfg.BuildKitAutoDiscoveryKubernetesUseSameNamespaceAsTsuruApp,
LeasePrefix: cfg.BuildKitAutoDiscoveryKubernetesLeasePrefix,
Statefulset: cfg.BuildKitAutoDiscoveryStatefulset,
ScaleGracefulPeriod: cfg.BuildKitAutoDiscoveryScaleGracefulPeriod,
}

return b.WithKubernetesDiscovery(cs, dcs, kdopts), nil
Expand Down
2 changes: 1 addition & 1 deletion pkg/build/build.go
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright 2023 tsuru authors. All rights reserved.
// Copyright 2024 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

Expand Down
16 changes: 12 additions & 4 deletions pkg/build/buildkit/build.go
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright 2023 tsuru authors. All rights reserved.
// Copyright 2024 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

Expand Down Expand Up @@ -38,6 +38,7 @@ import (
"k8s.io/client-go/kubernetes"

"github.com/tsuru/deploy-agent/pkg/build"
"github.com/tsuru/deploy-agent/pkg/build/buildkit/scaler"
pb "github.com/tsuru/deploy-agent/pkg/build/grpc_build_v1"
"github.com/tsuru/deploy-agent/pkg/util"
)
Expand Down Expand Up @@ -66,16 +67,23 @@ type KubernertesDiscoveryOptions struct {
PodSelector string
Namespace string
LeasePrefix string
Statefulset string
Port int
UseSameNamespaceAsApp bool
SetTsuruAppLabel bool
ScaleGracefulPeriod time.Duration
Timeout time.Duration
}

func (b *BuildKit) WithKubernetesDiscovery(cs *kubernetes.Clientset, dcs dynamic.Interface, opts KubernertesDiscoveryOptions) *BuildKit {
b.k8s = cs
b.dk8s = dcs
b.kdopts = &opts

if opts.Statefulset != "" {
scaler.StartWorker(cs, opts.PodSelector, opts.Statefulset, opts.ScaleGracefulPeriod)
}

return b
}

Expand All @@ -100,7 +108,7 @@ func (b *BuildKit) Build(ctx context.Context, r *pb.BuildRequest, w io.Writer) (
return nil, errors.New("writer must implement console.File")
}

c, clientCleanUp, err := b.client(ctx, r)
c, clientCleanUp, err := b.client(ctx, r, w)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -539,15 +547,15 @@ func callBuildKitToExtractTsuruConfigs(ctx context.Context, c *client.Client, lo
return tc, nil
}

func (b *BuildKit) client(ctx context.Context, req *pb.BuildRequest) (*client.Client, func(), error) {
func (b *BuildKit) client(ctx context.Context, req *pb.BuildRequest, w io.Writer) (*client.Client, func(), error) {
isBuildForApp := strings.HasPrefix(pb.BuildKind_name[int32(req.Kind)], "BUILD_KIND_APP_")

if isBuildForApp && b.opts.DiscoverBuildKitClientForApp {
d := &k8sDiscoverer{
cs: b.k8s,
dcs: b.dk8s,
}
return d.Discover(ctx, *b.kdopts, req)
return d.Discover(ctx, *b.kdopts, req, w)
}

return b.cli, noopFunc, nil
Expand Down
2 changes: 1 addition & 1 deletion pkg/build/buildkit/build_test.go
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright 2023 tsuru authors. All rights reserved.
// Copyright 2024 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

Expand Down
55 changes: 28 additions & 27 deletions pkg/build/buildkit/k8s_autodiscovery.go
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright 2023 tsuru authors. All rights reserved.
// Copyright 2024 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

Expand All @@ -8,13 +8,16 @@ import (
"context"
"encoding/json"
"fmt"
"io"
"os"
"strconv"
"strings"
"time"

"github.com/moby/buildkit/client"
"github.com/tsuru/deploy-agent/pkg/build/buildkit/scaler"
pb "github.com/tsuru/deploy-agent/pkg/build/grpc_build_v1"
"github.com/tsuru/deploy-agent/pkg/build/metadata"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
Expand All @@ -28,15 +31,6 @@ import (
"k8s.io/klog"
)

const (
DeployAgentLastBuildStartingLabelKey = "deploy-agent.tsuru.io/last-build-starting-time"
DeployAgentLastBuildEndingTimeLabelKey = "deploy-agent.tsuru.io/last-build-ending-time"

TsuruAppNamespace = "tsuru"
TsuruAppNameLabelKey = "tsuru.io/app-name"
TsuruIsBuildLabelKey = "tsuru.io/is-build"
)

var (
noopFunc = func() {}

Expand All @@ -52,15 +46,15 @@ type k8sDiscoverer struct {
dcs dynamic.Interface
}

func (d *k8sDiscoverer) Discover(ctx context.Context, opts KubernertesDiscoveryOptions, req *pb.BuildRequest) (*client.Client, func(), error) {
func (d *k8sDiscoverer) Discover(ctx context.Context, opts KubernertesDiscoveryOptions, req *pb.BuildRequest, w io.Writer) (*client.Client, func(), error) {
if req.App == nil {
return nil, noopFunc, fmt.Errorf("there's only support for discovering BuildKit pods from Tsuru apps")
}

return d.discoverBuildKitClientFromApp(ctx, opts, req.App.Name)
return d.discoverBuildKitClientFromApp(ctx, opts, req.App.Name, w)
}

func (d *k8sDiscoverer) discoverBuildKitClientFromApp(ctx context.Context, opts KubernertesDiscoveryOptions, app string) (*client.Client, func(), error) {
func (d *k8sDiscoverer) discoverBuildKitClientFromApp(ctx context.Context, opts KubernertesDiscoveryOptions, app string, w io.Writer) (*client.Client, func(), error) {
leaderCtx, leaderCancel := context.WithCancel(ctx)
cfns := []func(){
func() {
Expand All @@ -69,7 +63,7 @@ func (d *k8sDiscoverer) discoverBuildKitClientFromApp(ctx context.Context, opts
},
}

pod, err := d.discoverBuildKitPod(leaderCtx, opts, app)
pod, err := d.discoverBuildKitPod(leaderCtx, opts, app, w)
if err != nil {
return nil, cleanUps(cfns...), err
}
Expand All @@ -84,7 +78,7 @@ func (d *k8sDiscoverer) discoverBuildKitClientFromApp(ctx context.Context, opts

cfns = append(cfns, func() {
klog.V(4).Infoln("Removing Tsuru app labels in the pod", pod.Name)
nerr := unsetTsuruAppLabelOnBuildKitPod(context.Background(), d.cs, pod.Name, pod.Namespace)
nerr := unsetTsuruAppLabelOnBuildKitPod(ctx, d.cs, pod.Name, pod.Namespace)
if nerr != nil {
klog.Errorf("failed to unset Tsuru app labels: %s", nerr)
}
Expand All @@ -108,7 +102,7 @@ func (d *k8sDiscoverer) discoverBuildKitClientFromApp(ctx context.Context, opts
return c, cleanUps(cfns...), nil
}

func (d *k8sDiscoverer) discoverBuildKitPod(ctx context.Context, opts KubernertesDiscoveryOptions, app string) (*corev1.Pod, error) {
func (d *k8sDiscoverer) discoverBuildKitPod(ctx context.Context, opts KubernertesDiscoveryOptions, app string, w io.Writer) (*corev1.Pod, error) {
deadlineCtx, deadlineCancel := context.WithCancel(ctx)
defer deadlineCancel()

Expand All @@ -127,7 +121,7 @@ func (d *k8sDiscoverer) discoverBuildKitPod(ctx context.Context, opts Kubernerte
defer watchCancel() // watch cancellation must happen before than closing the pods channel

go func() {
nerr := watchBuildKitPods(watchCtx, d.cs, opts.PodSelector, ns, pods)
nerr := watchBuildKitPods(watchCtx, d.cs, opts, ns, pods, w)
if nerr != nil {
errCh <- nerr
}
Expand Down Expand Up @@ -188,7 +182,7 @@ func (d *k8sDiscoverer) buildkitPodNamespace(ctx context.Context, opts Kubernert

klog.V(4).Infof("Discovering the namespace where app %s is running on...", app)

tsuruApp, err := d.dcs.Resource(tsuruAppGVR).Namespace(TsuruAppNamespace).Get(ctx, app, metav1.GetOptions{})
tsuruApp, err := d.dcs.Resource(tsuruAppGVR).Namespace(metadata.TsuruAppNamespace).Get(ctx, app, metav1.GetOptions{})
if err != nil {
return "", err
}
Expand All @@ -208,9 +202,16 @@ func (d *k8sDiscoverer) buildkitPodNamespace(ctx context.Context, opts Kubernert
return ns, nil
}

func watchBuildKitPods(ctx context.Context, cs *kubernetes.Clientset, labelSelector, ns string, pods chan<- *corev1.Pod) error {
func watchBuildKitPods(ctx context.Context, cs *kubernetes.Clientset, opts KubernertesDiscoveryOptions, ns string, pods chan<- *corev1.Pod, writer io.Writer) error {
if opts.Statefulset != "" {
scaleErr := scaler.MayUpscale(ctx, cs, ns, opts.Statefulset, writer)
if scaleErr != nil {
return scaleErr
}
}

w, err := cs.CoreV1().Pods(ns).Watch(ctx, metav1.ListOptions{
LabelSelector: labelSelector,
LabelSelector: opts.PodSelector,
Watch: true,
})
if err != nil {
Expand Down Expand Up @@ -290,22 +291,22 @@ func setTsuruAppLabelOnBuildKitPod(ctx context.Context, cs *kubernetes.Clientset
patch, err := json.Marshal([]any{
map[string]any{
"op": "replace",
"path": fmt.Sprintf("/metadata/labels/%s", normalizeAppLabelForJSONPatch(TsuruAppNameLabelKey)),
"path": fmt.Sprintf("/metadata/labels/%s", normalizeAppLabelForJSONPatch(metadata.TsuruAppNameLabelKey)),
"value": app,
},
map[string]any{
"op": "replace",
"path": fmt.Sprintf("/metadata/labels/%s", normalizeAppLabelForJSONPatch(TsuruIsBuildLabelKey)),
"path": fmt.Sprintf("/metadata/labels/%s", normalizeAppLabelForJSONPatch(metadata.TsuruIsBuildLabelKey)),
"value": strconv.FormatBool(true),
},
map[string]any{
"op": "replace",
"path": fmt.Sprintf("/metadata/annotations/%s", normalizeAppLabelForJSONPatch(DeployAgentLastBuildEndingTimeLabelKey)),
"path": fmt.Sprintf("/metadata/annotations/%s", normalizeAppLabelForJSONPatch(metadata.DeployAgentLastBuildEndingTimeLabelKey)),
"value": "", // set annotation value to empty rather than removing it, since it might not exist at first run
},
map[string]any{
"op": "replace",
"path": fmt.Sprintf("/metadata/annotations/%s", normalizeAppLabelForJSONPatch(DeployAgentLastBuildStartingLabelKey)),
"path": fmt.Sprintf("/metadata/annotations/%s", normalizeAppLabelForJSONPatch(metadata.DeployAgentLastBuildStartingLabelKey)),
"value": strconv.FormatInt(time.Now().Unix(), 10),
},
})
Expand All @@ -321,15 +322,15 @@ func unsetTsuruAppLabelOnBuildKitPod(ctx context.Context, cs *kubernetes.Clients
patch, err := json.Marshal([]any{
map[string]any{
"op": "remove",
"path": fmt.Sprintf("/metadata/labels/%s", normalizeAppLabelForJSONPatch(TsuruAppNameLabelKey)),
"path": fmt.Sprintf("/metadata/labels/%s", normalizeAppLabelForJSONPatch(metadata.TsuruAppNameLabelKey)),
},
map[string]any{
"op": "remove",
"path": fmt.Sprintf("/metadata/labels/%s", normalizeAppLabelForJSONPatch(TsuruIsBuildLabelKey)),
"path": fmt.Sprintf("/metadata/labels/%s", normalizeAppLabelForJSONPatch(metadata.TsuruIsBuildLabelKey)),
},
map[string]any{
"op": "replace",
"path": fmt.Sprintf("/metadata/annotations/%s", normalizeAppLabelForJSONPatch(DeployAgentLastBuildEndingTimeLabelKey)),
"path": fmt.Sprintf("/metadata/annotations/%s", normalizeAppLabelForJSONPatch(metadata.DeployAgentLastBuildEndingTimeLabelKey)),
"value": strconv.FormatInt(time.Now().Unix(), 10),
},
})
Expand Down
Loading
Loading