Skip to content

Commit

Permalink
feat: add option to create remote repository before first push
Browse files Browse the repository at this point in the history
fix lint
  • Loading branch information
morpheu committed Sep 13, 2024
1 parent 6831043 commit 7e8bf5a
Show file tree
Hide file tree
Showing 11 changed files with 481 additions and 7 deletions.
7 changes: 5 additions & 2 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,8 @@ require (
github.com/docker/docker v23.0.0-rc.1+incompatible
github.com/google/go-containerregistry v0.12.0
github.com/moby/buildkit v0.11.3
github.com/stretchr/testify v1.8.0
github.com/oracle/oci-go-sdk/v65 v65.73.0
github.com/stretchr/testify v1.8.4
golang.org/x/sync v0.1.0
google.golang.org/grpc v1.50.1
google.golang.org/protobuf v1.28.1
Expand Down Expand Up @@ -61,7 +62,9 @@ require (
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/sirupsen/logrus v1.9.0 // indirect
github.com/sony/gobreaker v0.5.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/stretchr/objx v0.5.0 // indirect
github.com/tonistiigi/fsutil v0.0.0-20230105215944-fb433841cbfa // indirect
github.com/tonistiigi/units v0.0.0-20180711220420-6950e57a87ea // indirect
github.com/tonistiigi/vt100 v0.0.0-20210615222946-8066bb97264f // indirect
Expand All @@ -75,7 +78,7 @@ require (
golang.org/x/mod v0.6.0 // indirect
golang.org/x/net v0.7.0 // indirect
golang.org/x/oauth2 v0.1.0 // indirect
golang.org/x/sys v0.5.0 // indirect
golang.org/x/sys v0.8.0 // indirect
golang.org/x/term v0.5.0 // indirect
golang.org/x/text v0.7.0 // indirect
golang.org/x/time v0.1.0 // indirect
Expand Down
13 changes: 10 additions & 3 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -322,6 +322,8 @@ github.com/opencontainers/selinux v1.10.0/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuh
github.com/opencontainers/selinux v1.11.0 h1:+5Zbo97w3Lbmb3PeqQtpmTkMwsW5nRI3YaLpt7tQ7oU=
github.com/opencontainers/selinux v1.11.0/go.mod h1:E5dMC3VPuVvVHDYmi78qvhJp8+M586T4DlDRYpFkyec=
github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
github.com/oracle/oci-go-sdk/v65 v65.73.0 h1:C7uel6CoKk4A1KPkdhFBAyvVyFRTHAmX8m0o64RmfPg=
github.com/oracle/oci-go-sdk/v65 v65.73.0/go.mod h1:IBEV9l1qBzUpo7zgGaRUhbB05BVfcDGYRFBCPlTcPp0=
github.com/package-url/packageurl-go v0.1.1-0.20220428063043-89078438f170 h1:DiLBVp4DAcZlBVBEtJpNWZpZVq0AEeCY7Hqk8URVs4o=
github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8=
github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
Expand All @@ -344,6 +346,8 @@ github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6Mwd
github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0=
github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/sony/gobreaker v0.5.0 h1:dRCvqm0P490vZPmy7ppEk2qCnCieBooFJ+YoXGYB+yg=
github.com/sony/gobreaker v0.5.0/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY=
github.com/spdx/tools-golang v0.3.1-0.20230104082527-d6f58551be3f h1:9B623Cfs+mclYK6dsae7gLSwuIBHvlgmEup87qpqsAQ=
github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
Expand All @@ -353,15 +357,18 @@ github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
github.com/tonistiigi/fsutil v0.0.0-20230105215944-fb433841cbfa h1:XOFp/3aBXlqmOFAg3r6e0qQjPnK5I970LilqX+Is1W8=
github.com/tonistiigi/fsutil v0.0.0-20230105215944-fb433841cbfa/go.mod h1:AvLEd1LEIl64G2Jpgwo7aVV5lGH0ePcKl0ygGIHNYl8=
Expand Down Expand Up @@ -569,8 +576,8 @@ golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU=
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
Expand Down
17 changes: 16 additions & 1 deletion main.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ import (
"github.com/tsuru/deploy-agent/pkg/build/buildkit"
buildpb "github.com/tsuru/deploy-agent/pkg/build/grpc_build_v1"
"github.com/tsuru/deploy-agent/pkg/health"
"github.com/tsuru/deploy-agent/pkg/repository"
)

const (
Expand All @@ -42,12 +43,13 @@ var cfg struct {
BuildKitAutoDiscoveryKubernetesLeasePrefix string
BuildKitAutoDiscoveryStatefulset string
KubernetesConfig string
BuildKitAutoDiscoveryScaleGracefulPeriod time.Duration
RemoteRepositoryPath string
BuildKitAutoDiscoveryTimeout time.Duration
BuildKitAutoDiscoveryKubernetesPort int
Port int
ServerMaxRecvMsgSize int
ServerMaxSendMsgSize int
BuildKitAutoDiscoveryScaleGracefulPeriod time.Duration
BuildKitAutoDiscovery bool
BuildKitAutoDiscoveryKubernetesSetTsuruAppLabels bool
BuildKitAutoDiscoveryKubernetesUseSameNamespaceAsTsuruApp bool
Expand All @@ -65,6 +67,8 @@ func main() {
flag.StringVar(&cfg.BuildkitAddress, "buildkit-addr", getEnvOrDefault("BUILDKIT_HOST", ""), "Buildkit server address")
flag.StringVar(&cfg.BuildkitTmpDir, "buildkit-tmp-dir", os.TempDir(), "Directory path to store temp files during container image builds")

flag.StringVar(&cfg.RemoteRepositoryPath, "remote-repository-path", getEnvOrDefault("REMOTE_REPOSITORY_PATH", ""), "Remote image repository providers config path")

flag.BoolVar(&cfg.BuildKitAutoDiscovery, "buildkit-autodiscovery", false, "Whether should dynamically discover the BuildKit service based on Tsuru app (if any)")
flag.DurationVar(&cfg.BuildKitAutoDiscoveryTimeout, "buildkit-autodiscovery-timeout", (5 * time.Minute), "Max duration to discover an available BuildKit")
flag.StringVar(&cfg.BuildKitAutoDiscoveryKubernetesPodSelector, "buildkit-autodiscovery-kubernetes-pod-selector", "", "Label selector of BuildKit's pods on Kubernetes")
Expand Down Expand Up @@ -148,6 +152,17 @@ func newBuildKit() (*buildkit.BuildKit, error) {
c = bc
}

if cfg.RemoteRepositoryPath != "" {
repositoryData, err := os.ReadFile(cfg.RemoteRepositoryPath)
if err != nil {
return nil, err
}
opts.RemoteRepository, err = repository.NewRemoteRepository(repositoryData)
if err != nil {
return nil, fmt.Errorf("failed to handle remote repository cfg: %w", err)
}
}

b := buildkit.NewBuildKit(c, opts)

if cfg.BuildKitAutoDiscovery {
Expand Down
53 changes: 52 additions & 1 deletion pkg/build/buildkit/build.go
Original file line number Diff line number Diff line change
Expand Up @@ -40,12 +40,14 @@ import (
"github.com/tsuru/deploy-agent/pkg/build"
"github.com/tsuru/deploy-agent/pkg/build/buildkit/scaler"
pb "github.com/tsuru/deploy-agent/pkg/build/grpc_build_v1"
repo "github.com/tsuru/deploy-agent/pkg/repository"
"github.com/tsuru/deploy-agent/pkg/util"
)

var _ build.Builder = (*BuildKit)(nil)

type BuildKitOptions struct {
RemoteRepository map[string]repo.Repository
TempDir string
DiscoverBuildKitClientForApp bool
}
Expand Down Expand Up @@ -163,6 +165,13 @@ func (b *BuildKit) buildFromAppSourceFiles(ctx context.Context, c *client.Client
}
defer cleanFunc()

if b.opts.RemoteRepository != nil {
err = b.createRemoteRepository(ctx, r)
if err != nil {
return nil, err
}
}

if err = callBuildKitBuild(ctx, c, tmpDir, r, w); err != nil {
return nil, err
}
Expand Down Expand Up @@ -219,6 +228,13 @@ func (b *BuildKit) buildFromContainerImage(ctx context.Context, c *client.Client
}
defer cleanFunc()

if b.opts.RemoteRepository != nil {
err = b.createRemoteRepository(ctx, r)
if err != nil {
return nil, err
}
}

if err = callBuildKitBuild(ctx, c, tmpDir, r, w); err != nil {
return nil, err
}
Expand Down Expand Up @@ -252,6 +268,29 @@ func (b *BuildKit) extractTsuruConfigsFromContainerImage(ctx context.Context, c
return callBuildKitToExtractTsuruConfigs(ctx, c, tmpDir, workingDir)
}

func (b *BuildKit) createRemoteRepository(ctx context.Context, r *pb.BuildRequest) error {
for _, v := range r.DestinationImages {
if provider, ok := b.opts.RemoteRepository[build.GetRegistry(v)]; ok {
err := provider.Auth(ctx)
if err != nil {
return err
}
exists, err := provider.Exists(ctx, v)
if err != nil {
return err
}
if exists {
continue
}
err = provider.Create(ctx, v)
if err != nil {
return err
}
}
}
return nil
}

func extractContainerImageConfigFromImageManifest(ctx context.Context, imageStr string, insecureRegistry bool) (*pb.ContainerImageConfig, error) {
if err := ctx.Err(); err != nil {
return nil, err
Expand Down Expand Up @@ -388,6 +427,13 @@ func (b *BuildKit) buildFromContainerFile(ctx context.Context, c *client.Client,
}
defer cleanFunc()

if b.opts.RemoteRepository != nil {
err = b.createRemoteRepository(ctx, r)
if err != nil {
return nil, err
}
}

if err = callBuildKitBuild(ctx, c, tmpDir, r, w); err != nil {
return nil, err
}
Expand Down Expand Up @@ -418,7 +464,12 @@ func (b *BuildKit) buildPlatform(ctx context.Context, c *client.Client, r *pb.Bu
return err
}
defer cleanFunc()

if b.opts.RemoteRepository != nil {
err = b.createRemoteRepository(ctx, r)
if err != nil {
return err
}
}
return callBuildKitBuild(ctx, c, tmpDir, r, w)
}

Expand Down
24 changes: 24 additions & 0 deletions pkg/build/buildkit/build_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@ import (

. "github.com/tsuru/deploy-agent/pkg/build/buildkit"
pb "github.com/tsuru/deploy-agent/pkg/build/grpc_build_v1"
"github.com/tsuru/deploy-agent/pkg/repository"
"github.com/tsuru/deploy-agent/pkg/repository/fake"
"github.com/tsuru/deploy-agent/pkg/util"
)

Expand Down Expand Up @@ -376,6 +378,28 @@ func TestBuildKit_Build_FromContainerImages(t *testing.T) {
}, appFiles)
})

t.Run("creating remote repository", func(t *testing.T) {
req := &pb.BuildRequest{
Kind: pb.BuildKind_BUILD_KIND_APP_DEPLOY_WITH_CONTAINER_IMAGE,
App: &pb.TsuruApp{
Name: "my-app",
},
SourceImage: "nginx:1.22-alpine",
DestinationImages: []string{baseRegistry(t, "app-my-app", "v1")},
PushOptions: &pb.PushOptions{InsecureRegistry: registryHTTP},
}
opts := &BuildKitOptions{TempDir: t.TempDir(), RemoteRepository: map[string]repository.Repository{registryAddress: &fake.FakeRepository{AuthSuccess: true}}}
assert.Equal(t, opts.RemoteRepository[registryAddress].(*fake.FakeRepository).RepoExists, map[string]bool(nil))
_, err := NewBuildKit(bc, *opts).
Build(context.TODO(), req, os.Stdout)
require.NoError(t, err)
assert.Equal(t, opts.RemoteRepository[registryAddress].(*fake.FakeRepository).RepoExists, map[string]bool{baseRegistry(t, "app-my-app", "v1"): true})
_, err = NewBuildKit(bc, *opts).
Build(context.TODO(), req, os.Stdout)
require.NoError(t, err)
assert.Equal(t, opts.RemoteRepository[registryAddress].(*fake.FakeRepository).RepoExists, map[string]bool{baseRegistry(t, "app-my-app", "v1"): true})
})

t.Run("container image without Tsuru app files (tsuru.yaml, Procfile) + job image push", func(t *testing.T) {
req := &pb.BuildRequest{
Kind: pb.BuildKind_BUILD_KIND_JOB_CREATE_WITH_CONTAINER_IMAGE,
Expand Down
14 changes: 14 additions & 0 deletions pkg/build/helpers.go
Original file line number Diff line number Diff line change
Expand Up @@ -293,3 +293,17 @@ func SortExposedPorts(ports map[string]struct{}) []string {

return ps
}

func GetRegistry(image string) string {
defaultRegistry := "docker.io"
if !strings.Contains(image, "/") {
return defaultRegistry
}

registry := strings.Split(image, "/")[0]
if strings.Contains(registry, ".") || strings.Contains(registry, ":") {
return registry
}

return defaultRegistry
}
46 changes: 46 additions & 0 deletions pkg/repository/fake/repository.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
// Copyright 2024 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package fake

import (
"context"
"errors"
)

type FakeRepository struct {
CreatedRepos map[string]bool
RepoExists map[string]bool
AuthSuccess bool
}

func (f *FakeRepository) Auth(ctx context.Context) error {
if f.AuthSuccess {
return nil
}
return errors.New("auth repository failed")
}

func (f *FakeRepository) Create(ctx context.Context, name string) error {
if _, exists := f.RepoExists[name]; exists {
return errors.New("repository already exists")
}
if f.CreatedRepos == nil {
f.CreatedRepos = make(map[string]bool)
}
if f.RepoExists == nil {
f.RepoExists = make(map[string]bool)
}
f.CreatedRepos[name] = true
f.RepoExists[name] = true
return nil
}

func (f *FakeRepository) Exists(ctx context.Context, name string) (bool, error) {
if f.RepoExists == nil {
f.RepoExists = make(map[string]bool)
}
exists := f.RepoExists[name]
return exists, nil
}
Loading

0 comments on commit 7e8bf5a

Please sign in to comment.