From 4acf15cba9eafbe6c5773a48fb1ed8cceccfd6d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Imre=20Kel=C3=A9nyi?= Date: Thu, 4 Jul 2024 13:20:30 +0200 Subject: [PATCH] [ACI-2287] New step input: custom_tar_args (#34) --- e2e/bitrise.yml | 30 +++++++++++++++ go.mod | 2 +- go.sum | 2 + step.yml | 13 +++++++ step/step.go | 4 +- .../v2/cache/compression/compression.go | 7 ++-- .../go-steputils/v2/cache/network/download.go | 13 +++++-- .../v2/cache/network/interface.go | 17 +++++++++ .../go-steputils/v2/cache/network/upload.go | 17 ++++++--- .../go-steputils/v2/cache/restore.go | 34 +++++++++++++++-- .../bitrise-io/go-steputils/v2/cache/save.go | 38 +++++++++++++------ vendor/modules.txt | 2 +- 12 files changed, 148 insertions(+), 31 deletions(-) create mode 100644 vendor/github.com/bitrise-io/go-steputils/v2/cache/network/interface.go diff --git a/e2e/bitrise.yml b/e2e/bitrise.yml index 7e8d5a0..f0f4de3 100644 --- a/e2e/bitrise.yml +++ b/e2e/bitrise.yml @@ -157,6 +157,36 @@ workflows: - verbose: "true" - is_key_unique: "true" + test_custom_tar_args: + envs: + - TEST_APP_URL: https://github.com/bitrise-io/Bitrise-React-Native-Sample + - BRANCH: master + before_run: + - _generate_api_token + - _setup + steps: + - change-workdir: + title: Switch working dir to _tmp + inputs: + - path: ./_tmp + - script: + title: Install dependencies + inputs: + - content: |- + set -ex + npm ci + - path::./: + title: Execute step + run_if: "true" + is_skippable: false + inputs: + - key: | + {{ .OS }}-{{ .Arch }}-node-modules-{{ checksum "package-lock.json" }} + - paths: |- + node_modules + - verbose: "true" + - custom_tar_args: --format posix + _setup: steps: - script: diff --git a/go.mod b/go.mod index 18ed80f..251d3ea 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/bitrise-steplib/bitrise-step-save-cache go 1.20 require ( - github.com/bitrise-io/go-steputils/v2 v2.0.0-alpha.31 + github.com/bitrise-io/go-steputils/v2 v2.0.0-alpha.33 github.com/bitrise-io/go-utils/v2 v2.0.0-alpha.22 ) diff --git a/go.sum b/go.sum index 3765bd6..7d06789 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,7 @@ github.com/bitrise-io/go-steputils/v2 v2.0.0-alpha.31 h1:dO+9LYBwizOhS8ruTWk3PkWHfZoyhK0Ci40kv5LSZIw= github.com/bitrise-io/go-steputils/v2 v2.0.0-alpha.31/go.mod h1:Wu1/fiQqxr7A7a9aphh/Jxpof6WV+LEO6vGfBQ9T99A= +github.com/bitrise-io/go-steputils/v2 v2.0.0-alpha.33 h1:BxTsbaY9hwC/3a2y4V8vHlKAu80DAEuv/rvEEAvaq8o= +github.com/bitrise-io/go-steputils/v2 v2.0.0-alpha.33/go.mod h1:Wu1/fiQqxr7A7a9aphh/Jxpof6WV+LEO6vGfBQ9T99A= github.com/bitrise-io/go-utils v1.0.13 h1:1QENhTS/JlKH9F7+/nB+TtbTcor6jGrE6cQ4CJWfp5U= github.com/bitrise-io/go-utils v1.0.13/go.mod h1:ZY1DI+fEpZuFpO9szgDeICM4QbqoWVt0RSY3tRI1heY= github.com/bitrise-io/go-utils/v2 v2.0.0-alpha.22 h1:/SD9xE4LlX/Ju9YZ+n/yW/uDs7hXMdFlXg4Nxlb7678= diff --git a/step.yml b/step.yml index d726fda..a89c50c 100644 --- a/step.yml +++ b/step.yml @@ -110,6 +110,19 @@ inputs: summary: Zstd compression level to control speed / archive size. Set to 1 for fastest option. Valid values are between 1 and 19. Defaults to 3. is_required: false +- custom_tar_args: + opts: + title: Custom tar arguments + summary: Additional arguments to pass to the tar command when creating the cache archive. + description: |- + Additional arguments to pass to the tar command when creating the cache archive. + + The arguments are passed directly to the `tar` command. Use this input to customize the behavior of the tar command when creating the cache archive + (these are appended to the default arguments used by the step). + + Example: `--format posix` + is_required: false + - is_key_unique: "false" opts: title: Unique cache key diff --git a/step/step.go b/step/step.go index d88d5c4..20bdd5e 100644 --- a/step/step.go +++ b/step/step.go @@ -18,6 +18,7 @@ type Input struct { Paths string `env:"paths,required"` IsKeyUnique bool `env:"is_key_unique"` CompressionLevel int `env:"compression_level,range[1..19]"` + CustomTarArgs string `env:"custom_tar_args"` } type SaveCacheStep struct { @@ -52,7 +53,7 @@ func (step SaveCacheStep) Run() error { step.logger.EnableDebugLog(input.Verbose) - saver := cache.NewSaver(step.envRepo, step.logger, step.pathProvider, step.pathModifier, step.pathChecker) + saver := cache.NewSaver(step.envRepo, step.logger, step.pathProvider, step.pathModifier, step.pathChecker, nil) return saver.Save(cache.SaveCacheInput{ StepId: "save-cache", Verbose: input.Verbose, @@ -60,5 +61,6 @@ func (step SaveCacheStep) Run() error { Paths: strings.Split(input.Paths, "\n"), IsKeyUnique: input.IsKeyUnique, CompressionLevel: input.CompressionLevel, + CustomTarArgs: strings.Fields(input.CustomTarArgs), }) } diff --git a/vendor/github.com/bitrise-io/go-steputils/v2/cache/compression/compression.go b/vendor/github.com/bitrise-io/go-steputils/v2/cache/compression/compression.go index 9444dec..fac5356 100644 --- a/vendor/github.com/bitrise-io/go-steputils/v2/cache/compression/compression.go +++ b/vendor/github.com/bitrise-io/go-steputils/v2/cache/compression/compression.go @@ -67,7 +67,7 @@ func NewArchiver(logger log.Logger, envRepo env.Repository, archiveDependencyChe } // Compress creates a compressed archive from the provided files and folders using absolute paths. -func (a *Archiver) Compress(archivePath string, includePaths []string, compressionLevel int) error { +func (a *Archiver) Compress(archivePath string, includePaths []string, compressionLevel int, customTarArgs []string) error { haveZstdAndTar := a.archiveDependencyChecker.CheckDependencies() if !haveZstdAndTar { @@ -79,7 +79,7 @@ func (a *Archiver) Compress(archivePath string, includePaths []string, compressi } a.logger.Infof("Using installed zstd binary") - if err := a.compressWithBinary(archivePath, includePaths, compressionLevel); err != nil { + if err := a.compressWithBinary(archivePath, includePaths, compressionLevel, customTarArgs); err != nil { return fmt.Errorf("compress files: %w", err) } return nil @@ -183,7 +183,7 @@ func (a *Archiver) compressWithGoLib(archivePath string, includePaths []string, return nil } -func (a *Archiver) compressWithBinary(archivePath string, includePaths []string, compressionLevel int) error { +func (a *Archiver) compressWithBinary(archivePath string, includePaths []string, compressionLevel int, customTarArgs []string) error { cmdFactory := command.NewFactory(a.envRepo) /* @@ -207,6 +207,7 @@ func (a *Archiver) compressWithBinary(archivePath string, includePaths []string, "-c", "-f", archivePath, } + tarArgs = append(tarArgs, customTarArgs...) tarArgs = append(tarArgs, includePaths...) cmd := cmdFactory.Create("tar", tarArgs, nil) diff --git a/vendor/github.com/bitrise-io/go-steputils/v2/cache/network/download.go b/vendor/github.com/bitrise-io/go-steputils/v2/cache/network/download.go index d94cdda..e23ca05 100644 --- a/vendor/github.com/bitrise-io/go-steputils/v2/cache/network/download.go +++ b/vendor/github.com/bitrise-io/go-steputils/v2/cache/network/download.go @@ -14,6 +14,9 @@ import ( "github.com/melbahja/got" ) +// DefaultDownloader ... +type DefaultDownloader struct{} + // DownloadParams ... type DownloadParams struct { APIBaseURL string @@ -21,6 +24,7 @@ type DownloadParams struct { CacheKeys []string DownloadPath string NumFullRetries int + MaxConcurrency uint } // ErrCacheNotFound ... @@ -28,8 +32,10 @@ var ErrCacheNotFound = errors.New("no cache archive found for the provided keys" // Download archive from the cache API based on the provided keys in params. // If there is no match for any of the keys, the error is ErrCacheNotFound. -func Download(ctx context.Context, params DownloadParams, logger log.Logger) (string, error) { +func (d DefaultDownloader) Download(ctx context.Context, params DownloadParams, logger log.Logger) (string, error) { retryableHTTPClient := retryhttp.NewClient(logger) + retryableHTTPClient.HTTPClient.Transport.(*http.Transport).MaxIdleConns = 10 + return downloadWithClient(ctx, retryableHTTPClient, params, logger) } @@ -66,7 +72,7 @@ func downloadWithClient(ctx context.Context, httpClient *retryablehttp.Client, p } logger.Debugf("Downloading archive...") - downloadErr := downloadFile(ctx, httpClient.StandardClient(), restoreResponse.URL, params.DownloadPath) + downloadErr := downloadFile(ctx, httpClient.StandardClient(), restoreResponse.URL, params.DownloadPath, params.MaxConcurrency) if downloadErr != nil { logger.Debugf("Failed to download archive: %s", downloadErr) return fmt.Errorf("failed to download archive: %w", downloadErr), false @@ -79,7 +85,7 @@ func downloadWithClient(ctx context.Context, httpClient *retryablehttp.Client, p return matchedKey, err } -func downloadFile(ctx context.Context, client *http.Client, url string, dest string) error { +func downloadFile(ctx context.Context, client *http.Client, url string, dest string, maxConcurrency uint) error { downloader := got.New() downloader.Client = client @@ -88,6 +94,7 @@ func downloadFile(ctx context.Context, client *http.Client, url string, dest str // as depending on how downloader is called // either the Client from the downloader or from the Download will be used. gDownload.Client = client + gDownload.Concurrency = maxConcurrency err := downloader.Do(gDownload) diff --git a/vendor/github.com/bitrise-io/go-steputils/v2/cache/network/interface.go b/vendor/github.com/bitrise-io/go-steputils/v2/cache/network/interface.go new file mode 100644 index 0000000..505df6a --- /dev/null +++ b/vendor/github.com/bitrise-io/go-steputils/v2/cache/network/interface.go @@ -0,0 +1,17 @@ +package network + +import ( + "context" + + "github.com/bitrise-io/go-utils/v2/log" +) + +// Uploader ... +type Uploader interface { + Upload(context.Context, UploadParams, log.Logger) error +} + +// Downloader ... +type Downloader interface { + Download(context.Context, DownloadParams, log.Logger) (string, error) +} diff --git a/vendor/github.com/bitrise-io/go-steputils/v2/cache/network/upload.go b/vendor/github.com/bitrise-io/go-steputils/v2/cache/network/upload.go index a4f4133..585f6c3 100644 --- a/vendor/github.com/bitrise-io/go-steputils/v2/cache/network/upload.go +++ b/vendor/github.com/bitrise-io/go-steputils/v2/cache/network/upload.go @@ -1,6 +1,7 @@ package network import ( + "context" "fmt" "path/filepath" "strings" @@ -9,17 +10,21 @@ import ( "github.com/bitrise-io/go-utils/v2/retryhttp" ) +// DefaultUploader ... +type DefaultUploader struct{} + // UploadParams ... type UploadParams struct { - APIBaseURL string - Token string - ArchivePath string - ArchiveSize int64 - CacheKey string + APIBaseURL string + Token string + ArchivePath string + ArchiveChecksum string + ArchiveSize int64 + CacheKey string } // Upload a cache archive and associate it with the provided cache key -func Upload(params UploadParams, logger log.Logger) error { +func (u DefaultUploader) Upload(ctx context.Context, params UploadParams, logger log.Logger) error { validatedKey, err := validateKey(params.CacheKey, logger) if err != nil { return err diff --git a/vendor/github.com/bitrise-io/go-steputils/v2/cache/restore.go b/vendor/github.com/bitrise-io/go-steputils/v2/cache/restore.go index 592c596..26c66d5 100644 --- a/vendor/github.com/bitrise-io/go-steputils/v2/cache/restore.go +++ b/vendor/github.com/bitrise-io/go-steputils/v2/cache/restore.go @@ -6,6 +6,7 @@ import ( "fmt" "os" "path/filepath" + "strconv" "time" "github.com/bitrise-io/go-steputils/v2/cache/compression" @@ -39,12 +40,14 @@ type restoreCacheConfig struct { APIBaseURL stepconf.Secret APIAccessToken stepconf.Secret NumFullRetries int + MaxConcurrency uint } type restorer struct { envRepo env.Repository logger log.Logger cmdFactory command.Factory + downloader network.Downloader } type downloadResult struct { @@ -52,9 +55,19 @@ type downloadResult struct { matchedKey string } -// NewRestorer ... -func NewRestorer(envRepo env.Repository, logger log.Logger, cmdFactory command.Factory) *restorer { - return &restorer{envRepo: envRepo, logger: logger, cmdFactory: cmdFactory} +// NewRestorer creates a new cache restorer instance. `downloader` can be nil, unless you want to provide a custom `Downloader` implementation. +func NewRestorer( + envRepo env.Repository, + logger log.Logger, + cmdFactory command.Factory, + downloader network.Downloader, +) *restorer { + var downloaderImpl network.Downloader = downloader + if downloader == nil { + downloaderImpl = network.DefaultDownloader{} + } + + return &restorer{envRepo: envRepo, logger: logger, cmdFactory: cmdFactory, downloader: downloaderImpl} } // Restore ... @@ -129,6 +142,17 @@ func (r *restorer) createConfig(input RestoreCacheInput) (restoreCacheConfig, er return restoreCacheConfig{}, fmt.Errorf("the secret 'BITRISEIO_BITRISE_SERVICES_ACCESS_TOKEN' is not defined") } + maxConcurrency := uint(0) + maxConcurrencyStr := r.envRepo.Get("BITRISEIO_DEPENDENCY_CACHE_MAX_CONCURRENCY") + if maxConcurrencyStr != "" { + parsedConcurrency, err := strconv.ParseUint(maxConcurrencyStr, 10, 32) + if err != nil { + r.logger.Warnf("Failed to parse BITRISEIO_DEPENDENCY_CACHE_MAX_CONCURRENCY: %s", err) + } + + maxConcurrency = uint(parsedConcurrency) + } + keys, err := r.evaluateKeys(input.Keys) if err != nil { return restoreCacheConfig{}, fmt.Errorf("failed to evaluate keys: %w", err) @@ -140,6 +164,7 @@ func (r *restorer) createConfig(input RestoreCacheInput) (restoreCacheConfig, er APIBaseURL: stepconf.Secret(apiBaseURL), APIAccessToken: stepconf.Secret(apiAccessToken), NumFullRetries: input.NumFullRetries, + MaxConcurrency: maxConcurrency, }, nil } @@ -179,8 +204,9 @@ func (r *restorer) download(ctx context.Context, config restoreCacheConfig) (dow CacheKeys: config.Keys, DownloadPath: downloadPath, NumFullRetries: config.NumFullRetries, + MaxConcurrency: config.MaxConcurrency, } - matchedKey, err := network.Download(ctx, params, r.logger) + matchedKey, err := r.downloader.Download(ctx, params, r.logger) if err != nil { return downloadResult{}, err } diff --git a/vendor/github.com/bitrise-io/go-steputils/v2/cache/save.go b/vendor/github.com/bitrise-io/go-steputils/v2/cache/save.go index 7d9db3b..983eae2 100644 --- a/vendor/github.com/bitrise-io/go-steputils/v2/cache/save.go +++ b/vendor/github.com/bitrise-io/go-steputils/v2/cache/save.go @@ -1,6 +1,7 @@ package cache import ( + "context" "fmt" "os" "path/filepath" @@ -28,6 +29,9 @@ type SaveCacheInput struct { // CompressionLevel is the zstd compression level used. Valid values are between 1 and 19. // If not provided (0), the default value (3) will be used. CompressionLevel int + // CustomTarArgs is a list of custom arguments to pass to the tar command. These are appended to the default arguments. + // Example: []string{"--format", "posix"} + CustomTarArgs []string // IsKeyUnique indicates that the cache key is enough for knowing the cache archive is different from // another cache archive. // This can be set to true if the cache key contains a checksum that changes when any of the cached files change. @@ -46,6 +50,7 @@ type saveCacheConfig struct { Key string Paths []string CompressionLevel int + CustomTarArgs []string APIBaseURL stepconf.Secret APIAccessToken stepconf.Secret } @@ -56,22 +61,29 @@ type saver struct { pathProvider pathutil.PathProvider pathModifier pathutil.PathModifier pathChecker pathutil.PathChecker + uploader network.Uploader } -// NewSaver ... +// NewSaver creates a new cache saver instance. `uploader` can be nil, unless you want to provide a custom `Uploader` implementation. func NewSaver( envRepo env.Repository, logger log.Logger, pathProvider pathutil.PathProvider, pathModifier pathutil.PathModifier, pathChecker pathutil.PathChecker, + uploader network.Uploader, ) *saver { + var uploaderImpl network.Uploader = uploader + if uploader == nil { + uploaderImpl = network.DefaultUploader{} + } return &saver{ envRepo: envRepo, logger: logger, pathProvider: pathProvider, pathModifier: pathModifier, pathChecker: pathChecker, + uploader: uploaderImpl, } } @@ -101,7 +113,7 @@ func (s *saver) Save(input SaveCacheInput) error { s.logger.Println() s.logger.Infof("Creating archive...") compressionStartTime := time.Now() - archivePath, err := s.compress(config.Paths, config.CompressionLevel) + archivePath, err := s.compress(config.Paths, config.CompressionLevel, config.CustomTarArgs) if err != nil { return fmt.Errorf("compression failed: %s", err) } @@ -133,7 +145,7 @@ func (s *saver) Save(input SaveCacheInput) error { s.logger.Println() s.logger.Infof("Uploading archive...") uploadStartTime := time.Now() - err = s.upload(archivePath, fileInfo.Size(), config) + err = s.upload(archivePath, fileInfo.Size(), archiveChecksum, config) if err != nil { return fmt.Errorf("cache upload failed: %w", err) } @@ -183,6 +195,7 @@ func (s *saver) createConfig(input SaveCacheInput) (saveCacheConfig, error) { Key: evaluatedKey, Paths: finalPaths, CompressionLevel: input.CompressionLevel, + CustomTarArgs: input.CustomTarArgs, APIBaseURL: stepconf.Secret(apiBaseURL), APIAccessToken: stepconf.Secret(apiAccessToken), }, nil @@ -246,7 +259,7 @@ func (s *saver) evaluateKey(keyTemplate string) (string, error) { return model.Evaluate(keyTemplate) } -func (s *saver) compress(paths []string, compressionLevel int) (string, error) { +func (s *saver) compress(paths []string, compressionLevel int, customTarArgs []string) (string, error) { if compression.AreAllPathsEmpty(paths) { s.logger.Warnf("The provided paths are all empty, skipping compression and upload.") os.Exit(0) @@ -264,7 +277,7 @@ func (s *saver) compress(paths []string, compressionLevel int) (string, error) { s.envRepo, compression.NewDependencyChecker(s.logger, s.envRepo)) - err = archiver.Compress(archivePath, paths, compressionLevel) + err = archiver.Compress(archivePath, paths, compressionLevel, customTarArgs) if err != nil { return "", err } @@ -272,13 +285,14 @@ func (s *saver) compress(paths []string, compressionLevel int) (string, error) { return archivePath, nil } -func (s *saver) upload(archivePath string, archiveSize int64, config saveCacheConfig) error { +func (s *saver) upload(archivePath string, archiveSize int64, archiveChecksum string, config saveCacheConfig) error { params := network.UploadParams{ - APIBaseURL: string(config.APIBaseURL), - Token: string(config.APIAccessToken), - ArchivePath: archivePath, - ArchiveSize: archiveSize, - CacheKey: config.Key, + APIBaseURL: string(config.APIBaseURL), + Token: string(config.APIAccessToken), + ArchivePath: archivePath, + ArchiveChecksum: archiveChecksum, + ArchiveSize: archiveSize, + CacheKey: config.Key, } - return network.Upload(params, s.logger) + return s.uploader.Upload(context.Background(), params, s.logger) } diff --git a/vendor/modules.txt b/vendor/modules.txt index 20bcc63..def91cc 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1,4 +1,4 @@ -# github.com/bitrise-io/go-steputils/v2 v2.0.0-alpha.31 +# github.com/bitrise-io/go-steputils/v2 v2.0.0-alpha.33 ## explicit; go 1.17 github.com/bitrise-io/go-steputils/v2/cache github.com/bitrise-io/go-steputils/v2/cache/compression