From 062cac9af6e6e2b33af529c8cb1241a67f781bf5 Mon Sep 17 00:00:00 2001 From: Ben van B <030@users.noreply.github.com> Date: Sat, 2 Dec 2023 16:26:14 +0100 Subject: [PATCH] fix: [#384] Create groups. (#386) * fix: [#384] Create groups. * fix: [#384] Create groups. * fix: [#384] Create groups. * fix: [#384] Create groups. * fix: [#384] Create groups. * fix: [#384] Create groups. * fix: [#384] Create groups. * fix: [#384] Create groups. * fix: [#384] Create groups. * fix: [#384] Create groups. * fix: [#384] Create groups. * fix: [#384] Create groups. * fix: [#384] Create groups. * fix: [#384] Create groups. * fix: [#384] Create groups. * fix: [#384] Create groups. * fix: [#384] Create groups. * fix: [#384] Create groups. * fix: [#384] Create groups. * fix: [#384] Create groups. * feat: Determine what images are running in a k8s cluster. * fix: [#384] Create groups. * fix: [#384] Create groups. * fix: [#384] Create groups. * fix: [#384] Create groups. * fix: [#384] Create groups. * fix: [#384] Create groups. --- .dive-ci | 13 ++ .github/dependabot.yml | 15 ++ .github/workflows/trivy.yml | 12 +- .trivyignore | 1 + Dockerfile | 13 +- README.md | 2 +- build/package/snap/snapcraft.yaml | 2 +- cmd/n3dr/configRepository.go | 185 ++++++++++++------ cmd/n3dr/root.go | 2 +- docs/CHANGELOG.md | 15 +- docs/README.md | 6 +- docs/quickstarts/snippets/n3dr/DOWNLOAD.md | 6 +- go.mod | 17 +- go.sum | 46 +++-- internal/app/n3dr/artifactsv2/download.go | 2 +- .../upload/maven2/snapshot/upload.go | 5 +- .../app/n3dr/artifactsv2/upload/upload.go | 139 ++++++++----- .../app/n3dr/config/repository/repository.go | 109 +++++++++++ test/integration-tests.sh | 2 +- 19 files changed, 435 insertions(+), 157 deletions(-) create mode 100644 .dive-ci create mode 100644 .github/dependabot.yml create mode 100644 .trivyignore diff --git a/.dive-ci b/.dive-ci new file mode 100644 index 00000000..75f947d3 --- /dev/null +++ b/.dive-ci @@ -0,0 +1,13 @@ +rules: + # If the efficiency is measured below X%, mark as failed. + # Expressed as a ratio between 0-1. + lowestEfficiency: 0.90 + + # If the amount of wasted space is at least X or larger than X, mark as failed. + # Expressed in B, KB, MB, and GB. + highestWastedBytes: 20MB + + # If the amount of wasted space makes up for X% or more of the image, mark as failed. + # Note: the base image layer is NOT included in the total image size. + # Expressed as a ratio between 0-1; fails if the threshold is met or crossed. + highestUserWastedPercent: 0.25 \ No newline at end of file diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..75bc82fd --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,15 @@ +--- +version: 2 +updates: + - package-ecosystem: 'docker' + directory: '/' + schedule: + interval: 'weekly' + - package-ecosystem: 'github-actions' + directory: '/' + schedule: + interval: 'weekly' + - package-ecosystem: 'gomod' + directory: '/' + schedule: + interval: 'weekly' diff --git a/.github/workflows/trivy.yml b/.github/workflows/trivy.yml index b4f1d01a..db8e9111 100644 --- a/.github/workflows/trivy.yml +++ b/.github/workflows/trivy.yml @@ -12,7 +12,7 @@ jobs: run: | docker build -t utrecht/n3dr:${{ github.sha }} . - name: Run Trivy vulnerability scanner - uses: aquasecurity/trivy-action@master + uses: aquasecurity/trivy-action@0.14.0 with: image-ref: 'utrecht/n3dr:${{ github.sha }}' format: 'table' @@ -20,3 +20,13 @@ jobs: ignore-unfixed: true vuln-type: 'os,library' severity: 'CRITICAL,HIGH' + trivyignores: .trivyignore + - name: Run Trivy vulnerability scanner in fs mode + uses: aquasecurity/trivy-action@0.14.0 + with: + scan-type: 'fs' + scan-ref: '.' + exit-code: '1' + ignore-unfixed: true + severity: 'CRITICAL,HIGH' + trivyignores: .trivyignore diff --git a/.trivyignore b/.trivyignore new file mode 100644 index 00000000..a1cc6963 --- /dev/null +++ b/.trivyignore @@ -0,0 +1 @@ +#CVE-2023-5363 diff --git a/Dockerfile b/Dockerfile index 9872e566..9c948db3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,13 +5,16 @@ RUN adduser -D -g '' $USERNAME COPY . /go/${USERNAME}/ WORKDIR /go/${USERNAME}/cmd/${USERNAME} RUN apk add --no-cache \ - git=~2 && \ - CGO_ENABLED=0 go build -ldflags "-X main.Version=${VERSION}" -buildvcs=false && \ - cp n3dr /n3dr + git=~2 && \ + CGO_ENABLED=0 go build -ldflags "-X main.Version=${VERSION}" -buildvcs=false && \ + cp n3dr /n3dr -FROM alpine:3.18.3 +FROM alpine:3.18.4 COPY --from=builder /etc/passwd /etc/passwd COPY --from=builder /n3dr /usr/local/bin/n3dr COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ +RUN apk add --no-cache \ + libcrypto3=3.1.4-r1 \ + libssl3=3.1.4-r1 USER n3dr -ENTRYPOINT ["/usr/local/bin/n3dr"] +ENTRYPOINT ["n3dr"] diff --git a/README.md b/README.md index a6f52f8d..d16b4daf 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ [![DevOps SE Questions](https://img.shields.io/stackexchange/devops/t/n3dr.svg?logo=stackexchange)](https://devops.stackexchange.com/tags/n3dr) [![ServerFault SE Questions](https://img.shields.io/stackexchange/serverfault/t/n3dr.svg?logo=serverfault)](https://serverfault.com/tags/n3dr) [![Docker Pulls](https://img.shields.io/docker/pulls/utrecht/n3dr?logo=docker&logoColor=white)](https://hub.docker.com/r/utrecht/n3dr) -![Docker Image Size (latest semver)](https://img.shields.io/docker/image-size/utrecht/n3dr?logo=docker&logoColor=white&sort=semver) +[![Docker Image Size (latest semver)](https://img.shields.io/docker/image-size/utrecht/n3dr?logo=docker&logoColor=white&sort=semver)](https://hub.docker.com/r/utrecht/n3dr) ![Issues](https://img.shields.io/github/issues-raw/030/n3dr.svg) ![Pull requests](https://img.shields.io/github/issues-pr-raw/030/n3dr.svg) ![Total downloads](https://img.shields.io/github/downloads/030/n3dr/total.svg) diff --git a/build/package/snap/snapcraft.yaml b/build/package/snap/snapcraft.yaml index 5de84cd6..b526b3e2 100644 --- a/build/package/snap/snapcraft.yaml +++ b/build/package/snap/snapcraft.yaml @@ -1,7 +1,7 @@ --- name: n3dr base: core20 -version: 7.3.1 +version: 7.3.2 summary: Nexus3 Disaster Recovery description: | Download all artifacts at once or migrate automatically from Nexus to Nexus. diff --git a/cmd/n3dr/configRepository.go b/cmd/n3dr/configRepository.go index d8e78d4e..7d8f9f44 100644 --- a/cmd/n3dr/configRepository.go +++ b/cmd/n3dr/configRepository.go @@ -1,6 +1,7 @@ package main import ( + "fmt" "os" "github.com/030/n3dr/internal/app/n3dr/config/repository" @@ -13,8 +14,109 @@ var ( configRepoDockerPortSecure, configRepoDelete, snapshot, strictContentTypeValidation bool configRepoDockerPort int32 configRepoName, configRepoRecipe, configRepoType, configRepoProxyURL string + configRepoGroupMemberNames []string ) +type repo struct { + conn repository.Repository + kind, name, recipe string + snapshot bool +} + +var repoRecipeAndKindNotSupported = "repoRecipe: '%s' not supported in conjunction with repoKind: '%s'" + +func (r *repo) createByType() error { + switch configRepoType { + case "apt": + return r.Apt() + case "docker": + return r.Docker() + case "gem": + return r.Gem() + case "maven2": + return r.Maven2() + case "npm": + return r.Npm() + case "raw": + return r.Raw() + case "yum": + return r.Yum() + default: + return fmt.Errorf("configRepoType should not be empty, but: 'apt', 'docker', 'gem', 'maven2', 'npm' 'raw' or 'yum' and not: '%s'. Did you populate the --configRepoType parameter?", configRepoType) + } +} + +func (r *repo) Apt() error { + switch r.recipe { + case "proxy": + return r.conn.CreateAptProxied(r.name) + default: + return fmt.Errorf(repoRecipeAndKindNotSupported, r.recipe, r.kind) + } +} + +func (r *repo) Docker() error { + switch r.recipe { + case "hosted": + return r.conn.CreateDockerHosted(configRepoDockerPortSecure, configRepoDockerPort, r.name) + default: + return fmt.Errorf(repoRecipeAndKindNotSupported, r.recipe, r.kind) + } +} + +func (r *repo) Gem() error { + switch r.recipe { + case "hosted": + return r.conn.CreateGemHosted(r.name) + default: + return fmt.Errorf(repoRecipeAndKindNotSupported, r.recipe, r.kind) + } +} + +func (r *repo) Maven2() error { + switch r.recipe { + case "group": + return r.conn.CreateMavenGroup(configRepoGroupMemberNames, r.name) + case "hosted": + return r.conn.CreateMavenHosted(r.name, snapshot) + case "proxy": + return r.conn.CreateMavenProxied(r.name) + default: + return fmt.Errorf(repoRecipeAndKindNotSupported, r.recipe, r.kind) + } +} + +func (r *repo) Npm() error { + switch r.recipe { + case "hosted": + return r.conn.CreateNpmHosted(r.name, snapshot) + case "proxy": + return r.conn.CreateNpmProxied(r.name) + default: + return fmt.Errorf(repoRecipeAndKindNotSupported, r.recipe, r.kind) + } +} + +func (r *repo) Raw() error { + switch r.recipe { + case "hosted": + return r.conn.CreateRawHosted(r.name) + default: + return fmt.Errorf(repoRecipeAndKindNotSupported, r.recipe, r.kind) + } +} + +func (r *repo) Yum() error { + switch r.recipe { + case "hosted": + return r.conn.CreateYumHosted(r.name) + case "proxy": + return r.conn.CreateYumProxied(r.name) + default: + return fmt.Errorf(repoRecipeAndKindNotSupported, r.recipe, r.kind) + } +} + // configRepositoryCmd represents the configRepository command. var configRepositoryCmd = &cobra.Command{ Use: "configRepository", @@ -26,6 +128,9 @@ Examples: # Create a Docker repository: n3dr configRepository -u some-user -p some-pass -n localhost:9000 --https=false --configRepoName some-name --configRepoType docker + # Create a Maven2 repository if credentials and FQDN have been set in a '~/.n3dr/config.yml' file: + n3dr configRepository --configRepoName some-name --configRepoType maven2 + # Create a Maven2 repository: n3dr configRepository -u some-user -p some-pass -n localhost:9000 --https=false --configRepoName some-name --configRepoType maven2 @@ -40,6 +145,16 @@ Examples: # Create a Rubygems repository: n3dr configRepository -u admin -p some-pass -n localhost:9000 --https=false --configRepoName 3rdparty-rubygems --configRepoType gem + + # Create Maven2 proxies: + n3dr configRepository --configRepoType maven2 --configRepoName 3rdparty-maven --configRepoRecipe proxy --configRepoProxyURL https://repo.maven.apache.org/maven2/ + n3dr configRepository --configRepoType maven2 --configRepoName 3rdparty-maven-gradle-plugins --configRepoRecipe proxy --configRepoProxyURL https://plugins.gradle.org/m2/ + + # Create a NPM proxy: + n3dr configRepository --configRepoType npm --configRepoName 3rdparty-npm --configRepoRecipe proxy --configRepoProxyURL https://registry.npmjs.org/ + + # Create a Maven2 group: + n3dr configRepository --configRepoType maven2 --configRepoRecipe group --configRepoName some-group --configRepoGroupMemberNames releases,snapshots `, Run: func(cmd *cobra.Command, args []string) { n := connection.Nexus3{ @@ -49,10 +164,10 @@ Examples: StrictContentTypeValidation: strictContentTypeValidation, User: n3drUser, } - r := repository.Repository{Nexus3: n} + rr := repository.Repository{Nexus3: n} if configRepoDelete { - if err := r.Delete(configRepoName); err != nil { + if err := rr.Delete(configRepoName); err != nil { log.Fatal(err) } os.Exit(0) @@ -62,64 +177,17 @@ Examples: log.Fatal("configRepoReceipe should not be empty") } - if configRepoRecipe == "proxy" { - if configRepoProxyURL == "" { - log.Fatal("configRepoProxyURL should not be empty") - } - r.ProxyRemoteURL = configRepoProxyURL + if configRepoRecipe == "proxy" && configRepoProxyURL == "" { + log.Fatal("configRepoProxyURL should not be empty") + } else { + rr.ProxyRemoteURL = configRepoProxyURL + log.Infof("configRepoProxyURL has been set to: '%s'", rr.ProxyRemoteURL) } - switch configRepoType { - case "apt": - if configRepoRecipe == "proxy" { - if err := r.CreateAptProxied(configRepoName); err != nil { - log.Fatal(err) - } - } - case "docker": - if configRepoRecipe == "hosted" { - if err := r.CreateDockerHosted(configRepoDockerPortSecure, configRepoDockerPort, configRepoName); err != nil { - log.Fatal(err) - } - } - case "gem": - if configRepoRecipe == "hosted" { - if err := r.CreateGemHosted(configRepoName); err != nil { - log.Fatal(err) - } - } - case "maven2": - if configRepoRecipe == "hosted" { - if err := r.CreateMavenHosted(configRepoName, snapshot); err != nil { - log.Fatal(err) - } - } - case "npm": - if configRepoRecipe == "hosted" { - if err := r.CreateNpmHosted(configRepoName, snapshot); err != nil { - log.Fatal(err) - } - } - case "raw": - if configRepoRecipe == "hosted" { - if err := r.CreateRawHosted(configRepoName); err != nil { - log.Fatal(err) - } - } - case "yum": - if configRepoRecipe == "hosted" { - if err := r.CreateYumHosted(configRepoName); err != nil { - log.Fatal(err) - } - } else if configRepoRecipe == "proxy" { - if err := r.CreateYumProxied(configRepoName); err != nil { - log.Fatal(err) - } - } else { - log.Fatalf("configRepoRecipe: '%s' not supported in conjunction with configRepoType: '%s'", configRepoRecipe, configRepoType) - } - default: - log.Fatalf("configRepoType should not be empty, but: 'apt', 'docker', 'maven2', 'raw' or 'yum' and not: '%s'. Did you populate the --configRepoType parameter?", configRepoType) + log.Infof("creating repo: '%s' of type: '%s'", configRepoName, configRepoType) + r := repo{conn: rr, kind: configRepoType, name: configRepoName, recipe: configRepoRecipe, snapshot: snapshot} + if err := r.createByType(); err != nil { + log.Fatalf("repo not created. Error: '%v'", err) } }, } @@ -140,4 +208,5 @@ func init() { configRepositoryCmd.Flags().Int32Var(&configRepoDockerPort, "configRepoDockerPort", 8082, "The docker connector port, e.g. 8082") configRepositoryCmd.Flags().BoolVar(&configRepoDockerPortSecure, "configRepoDockerPortSecure", false, "Whether the docker connector port should be secure") configRepositoryCmd.Flags().BoolVar(&strictContentTypeValidation, "strictContentTypeValidation", true, "whether strictContentTypeValidation should be enabled") + configRepositoryCmd.Flags().StringSliceVar(&configRepoGroupMemberNames, "configRepoGroupMemberNames", []string{}, "The repository type, e.g.: 'apt', 'raw'") } diff --git a/cmd/n3dr/root.go b/cmd/n3dr/root.go index b7374e67..245f3425 100644 --- a/cmd/n3dr/root.go +++ b/cmd/n3dr/root.go @@ -219,7 +219,7 @@ func parseConfig(cfgFile string) error { return err } } else { - log.Warnf("Looked for config file: '%v', but found: '%v' including err: '%v'. Check whether it exists, the YAML is correct and the content is valid", cfgFile, viper.ConfigFileUsed(), err) + log.Debugf("Looked for config file: '%v', but found: '%v' including err: '%v'. Check whether it exists, the YAML is correct and the content is valid", cfgFile, viper.ConfigFileUsed(), err) } return nil diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 9dc51e5c..c2b5bd70 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -2,6 +2,18 @@ ## [Unreleased] + +## [7.3.2] - 2023-12-02 +### Build +- **deps:** bump golang.org/x/net from 0.15.0 to 0.17.0 + +### Feat +- Determine what images are running in a k8s cluster. + +### Fix +- [[#384](https://github.com/030/n3dr/issues/384)] Create groups. + + ## [7.3.1] - 2023-10-14 ### Fix @@ -394,7 +406,8 @@ The `backup`, `upload` and `repositories` commands have been removed. ## 1.0.0 - 2019-05-12 -[Unreleased]: https://github.com/030/n3dr/compare/7.3.1...HEAD +[Unreleased]: https://github.com/030/n3dr/compare/7.3.2...HEAD +[7.3.2]: https://github.com/030/n3dr/compare/7.3.1...7.3.2 [7.3.1]: https://github.com/030/n3dr/compare/7.3.0...7.3.1 [7.3.0]: https://github.com/030/n3dr/compare/7.2.5...7.3.0 [7.2.5]: https://github.com/030/n3dr/compare/7.2.4...7.2.5 diff --git a/docs/README.md b/docs/README.md index 931db96a..3049604e 100644 --- a/docs/README.md +++ b/docs/README.md @@ -238,7 +238,7 @@ n3dr config \ ### Build ```bash -docker build -t utrecht/n3dr:7.3.1 . +docker build -t utrecht/n3dr:7.3.2 . ``` [![dockeri.co](https://dockeri.co/image/utrecht/n3dr)](https://hub.docker.com/r/utrecht/n3dr) @@ -248,7 +248,7 @@ docker build -t utrecht/n3dr:7.3.1 . ```bash docker run -it \ -v /home/${USER}/.n3dr:/root/.n3dr \ - -v /tmp/n3dr:/tmp/n3dr utrecht/n3dr:7.3.1 + -v /tmp/n3dr:/tmp/n3dr utrecht/n3dr:7.3.2 ``` ### Upload @@ -257,7 +257,7 @@ docker run -it \ docker run -it \ --entrypoint=/bin/ash \ -v /home/${USER}/.n3dr:/root/.n3dr \ - -v /tmp/n3dr:/tmp/n3dr utrecht/n3dr:7.3.1 + -v /tmp/n3dr:/tmp/n3dr utrecht/n3dr:7.3.2 ``` navigate to the repository folder, e.g. `/tmp/n3dr/download*/` and upload: diff --git a/docs/quickstarts/snippets/n3dr/DOWNLOAD.md b/docs/quickstarts/snippets/n3dr/DOWNLOAD.md index 8e8047aa..b9140867 100644 --- a/docs/quickstarts/snippets/n3dr/DOWNLOAD.md +++ b/docs/quickstarts/snippets/n3dr/DOWNLOAD.md @@ -1,12 +1,12 @@ # Download -Download the [latest N3DR binary](https://github.com/030/n3dr/releases/tag/7.3.1): +Download the [latest N3DR binary](https://github.com/030/n3dr/releases/tag/7.3.2): ```bash cd /tmp && \ -curl -L https://github.com/030/n3dr/releases/download/7.3.1/n3dr-ubuntu-latest \ +curl -L https://github.com/030/n3dr/releases/download/7.3.2/n3dr-ubuntu-latest \ -o n3dr-ubuntu-latest && \ -curl -L https://github.com/030/n3dr/releases/download/7.3.1/\ +curl -L https://github.com/030/n3dr/releases/download/7.3.2/\ n3dr-ubuntu-latest.sha512.txt \ -o n3dr-ubuntu-latest.sha512.txt && \ sha512sum -c n3dr-ubuntu-latest.sha512.txt && \ diff --git a/go.mod b/go.mod index 9848dd04..a995ae2a 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/030/n3dr -go 1.19 +go 1.21 require ( github.com/030/logging v0.1.2 @@ -14,7 +14,7 @@ require ( github.com/go-openapi/validate v0.22.1 github.com/go-playground/validator/v10 v10.15.5 github.com/hashicorp/go-retryablehttp v0.7.4 - github.com/mholt/archiver v3.1.1+incompatible + github.com/mholt/archiver/v3 v3.5.1 github.com/mitchellh/go-homedir v1.1.0 github.com/samber/lo v1.38.1 github.com/sirupsen/logrus v1.9.3 @@ -25,7 +25,7 @@ require ( ) require ( - github.com/andybalholm/brotli v1.0.4 // indirect + github.com/andybalholm/brotli v1.0.6 // indirect github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 // indirect @@ -46,20 +46,19 @@ require ( github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect - github.com/klauspost/compress v1.17.0 // indirect - github.com/klauspost/pgzip v1.2.5 // indirect + github.com/klauspost/compress v1.17.2 // indirect + github.com/klauspost/pgzip v1.2.6 // indirect github.com/leodido/go-urn v1.2.4 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mholt/archiver/v4 v4.0.0-alpha.7 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect - github.com/nwaples/rardecode v1.1.3 // indirect + github.com/nwaples/rardecode v1.1.0 // indirect github.com/nwaples/rardecode/v2 v2.0.0-beta.2 // indirect github.com/oklog/ulid v1.3.1 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect github.com/pelletier/go-toml/v2 v2.1.0 // indirect - github.com/pierrec/lz4 v2.6.1+incompatible // indirect - github.com/pierrec/lz4/v4 v4.1.17 // indirect + github.com/pierrec/lz4/v4 v4.1.18 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/sagikazarmark/locafero v0.3.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect @@ -83,7 +82,7 @@ require ( golang.org/x/crypto v0.14.0 // indirect golang.org/x/net v0.17.0 // indirect golang.org/x/sys v0.13.0 // indirect - golang.org/x/text v0.13.0 // indirect + golang.org/x/text v0.14.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index 04dcd21d..d721742e 100644 --- a/go.sum +++ b/go.sum @@ -46,8 +46,9 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= -github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= -github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= +github.com/andybalholm/brotli v1.0.1/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= +github.com/andybalholm/brotli v1.0.6 h1:Yf9fFpf49Zrxb9NlQaluyE92/+X7UVHlhMNJN2sxfOI= +github.com/andybalholm/brotli v1.0.6/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= @@ -77,7 +78,9 @@ github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5y github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.14.1 h1:qfhVLaG5s+nCROl1zJsZRxFeYrHLqWroPOQ8BWiNb4w= +github.com/fatih/color v1.14.1/go.mod h1:2oHN61fhTpgcxD3TSWCgKDiH1+x4OiDVVGH8WlgGZGg= github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= +github.com/frankban/quicktest v1.14.4/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU= @@ -126,6 +129,7 @@ github.com/go-openapi/swag v0.22.4/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+ github.com/go-openapi/validate v0.22.1 h1:G+c2ub6q47kfX1sOBLwIQwzBVt8qmOAARyo/9Fqs9NU= github.com/go-openapi/validate v0.22.1/go.mod h1:rjnrwK57VJ7A8xqfpAOEKRH8yQSGUriMu5/zuPSQ1hg= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= +github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= @@ -183,6 +187,7 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.2/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -198,6 +203,7 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -215,6 +221,7 @@ github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm4 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= @@ -222,6 +229,7 @@ github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9n github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= github.com/hashicorp/go-hclog v1.5.0 h1:bI2ocEMgcVlz55Oj1xZNBsVi900c7II+fWDyV9o+13c= +github.com/hashicorp/go-hclog v1.5.0/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= github.com/hashicorp/go-retryablehttp v0.7.4 h1:ZQgVdpTdAL7WpMIwLzCfbalOcSUdkDZnpUv3/+BxzFA= github.com/hashicorp/go-retryablehttp v0.7.4/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= @@ -246,18 +254,21 @@ github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaR github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.11.4/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/compress v1.17.0 h1:Rnbp4K9EjcDuVuHtd0dgA4qNuv9yKDYKK1ulpJwgrqM= -github.com/klauspost/compress v1.17.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4= +github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= -github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE= github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= +github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU= +github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -274,9 +285,11 @@ github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJ github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= -github.com/mholt/archiver v3.1.1+incompatible h1:1dCVxuqs0dJseYEhi5pl7MYPH9zDa1wBi7mF09cbNkU= -github.com/mholt/archiver v3.1.1+incompatible/go.mod h1:Dh2dOXnSdiLxRiPoVfIr/fI1TwETms9B8CTWfeh7ROU= +github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mholt/archiver/v3 v3.5.1 h1:rDjOBX9JSF5BvoJGvjqK479aL70qh9DIpZCl+k7Clwo= +github.com/mholt/archiver/v3 v3.5.1/go.mod h1:e3dqJ7H78uzsRSEACH1joayhuSyhnonssnDhppzS1L4= github.com/mholt/archiver/v4 v4.0.0-alpha.7 h1:xzByj8G8tj0Oq7ZYYU4+ixL/CVb5ruWCm0EZQ1PjOkE= github.com/mholt/archiver/v4 v4.0.0-alpha.7/go.mod h1:Fs8qUkO74HHaidabihzYephJH8qmGD/nCP6tE5xC9BM= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= @@ -287,8 +300,8 @@ github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyua github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/nwaples/rardecode v1.1.3 h1:cWCaZwfM5H7nAD6PyEdcVnczzV8i/JtotnyW/dD9lEc= -github.com/nwaples/rardecode v1.1.3/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0= +github.com/nwaples/rardecode v1.1.0 h1:vSxaY8vQhOcVr4mm5e8XllHWTiM4JF507A0Katqw7MQ= +github.com/nwaples/rardecode v1.1.0/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0= github.com/nwaples/rardecode/v2 v2.0.0-beta.2 h1:e3mzJFJs4k83GXBEiTaQ5HgSc/kOK8q0rDaRO0MPaOk= github.com/nwaples/rardecode/v2 v2.0.0-beta.2/go.mod h1:yntwv/HfMc/Hbvtq9I19D1n58te3h6KsqCf3GxyfBGY= github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= @@ -298,10 +311,9 @@ github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYr github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= -github.com/pierrec/lz4 v2.6.1+incompatible h1:9UY3+iC23yxF0UfGaYrGplQ+79Rg+h/q9FV9ix19jjM= -github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= -github.com/pierrec/lz4/v4 v4.1.17 h1:kV4Ip+/hUBC+8T6+2EgburRtkE9ef4nbY3f4dFhGjMc= -github.com/pierrec/lz4/v4 v4.1.17/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pierrec/lz4/v4 v4.1.2/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ= +github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -314,6 +326,7 @@ github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sagikazarmark/locafero v0.3.0 h1:zT7VEGWC2DTflmccN/5T1etyKvxSxpHsjb9cJvm4SvQ= github.com/sagikazarmark/locafero v0.3.0/go.mod h1:w+v7UsPNFwzF1cHuOajOOzoq4U7v/ig1mpRjqV+Bu1U= @@ -371,6 +384,7 @@ github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhso github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= +github.com/ulikunitz/xz v0.5.9/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8= github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= @@ -400,6 +414,7 @@ go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opentelemetry.io/otel v1.14.0 h1:/79Huy8wbf5DnIPhemGB+zEPVwnN6fuQybr/SRXa6hM= go.opentelemetry.io/otel v1.14.0/go.mod h1:o4buv+dJzx8rohcUeRmWUZhqupFvzWis188WlggnNeU= go.opentelemetry.io/otel/sdk v1.14.0 h1:PDCppFRDq8A1jL9v6KMI6dYesaq+DFcDZvjsoGvxGzY= +go.opentelemetry.io/otel/sdk v1.14.0/go.mod h1:bwIC5TjrNG6QDCHNWvW4HLHtUQ4I+VQDsnjhvyZCALM= go.opentelemetry.io/otel/trace v1.14.0 h1:wp2Mmvj41tDsyAJXiWDWpfNsOiIyd38fy85pyKcFq/M= go.opentelemetry.io/otel/trace v1.14.0/go.mod h1:8avnQLK+CG77yNLUae4ea2JDQ6iT+gozhnZjy/rw9G8= go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= @@ -576,8 +591,8 @@ golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= -golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -729,6 +744,7 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= diff --git a/internal/app/n3dr/artifactsv2/download.go b/internal/app/n3dr/artifactsv2/download.go index 73c9896e..863262ae 100644 --- a/internal/app/n3dr/artifactsv2/download.go +++ b/internal/app/n3dr/artifactsv2/download.go @@ -16,7 +16,7 @@ import ( "github.com/030/n3dr/internal/app/n3dr/s3" "github.com/030/p2iwd/pkg/p2iwd" "github.com/hashicorp/go-retryablehttp" - "github.com/mholt/archiver" + "github.com/mholt/archiver/v3" log "github.com/sirupsen/logrus" ) diff --git a/internal/app/n3dr/artifactsv2/upload/maven2/snapshot/upload.go b/internal/app/n3dr/artifactsv2/upload/maven2/snapshot/upload.go index 2c0075be..bf27b580 100644 --- a/internal/app/n3dr/artifactsv2/upload/maven2/snapshot/upload.go +++ b/internal/app/n3dr/artifactsv2/upload/maven2/snapshot/upload.go @@ -9,7 +9,6 @@ import ( "path/filepath" "regexp" - "github.com/030/n3dr/internal/app/n3dr/artifactsv2/artifacts" "github.com/hashicorp/go-retryablehttp" log "github.com/sirupsen/logrus" ) @@ -22,8 +21,6 @@ type Nexus3 struct { func (n *Nexus3) statusCode(resp *http.Response) error { if resp.StatusCode == http.StatusCreated { log.Trace("file has been uploaded") - artifacts.PrintType(n.RepoFormat) - return nil } else { bodyBytes, err := io.ReadAll(resp.Body) if err != nil { @@ -65,7 +62,7 @@ func (n *Nexus3) readRetryAndUpload(path string) error { log.Tracef("uri: '%s'", uri) u := protocol + "://" + n.FQDN + "/repository/" + n.RepoName + "/" + uri - log.Tracef("upload url: '%s'", u) + log.Tracef("snapshot upload url: '%s'", u) req, err := http.NewRequest("PUT", u, f) if err != nil { return err diff --git a/internal/app/n3dr/artifactsv2/upload/upload.go b/internal/app/n3dr/artifactsv2/upload/upload.go index c47766e0..da4d8fff 100644 --- a/internal/app/n3dr/artifactsv2/upload/upload.go +++ b/internal/app/n3dr/artifactsv2/upload/upload.go @@ -40,6 +40,10 @@ type Nexus3 struct { *connection.Nexus3 } +type repoFormatAndType struct { + format, repoType string +} + func uploadStatus(err error) (int, error) { re := regexp.MustCompile(`status (\d{3})`) match := re.FindStringSubmatch(err.Error()) @@ -70,10 +74,11 @@ func (n *Nexus3) reposOnDisk() (localDiskRepos []string, err error) { return nil, err } log.Infof("found the following localDiskRepos: '%v'", localDiskRepos) + return localDiskRepos, nil } -func (n *Nexus3) repoFormatLocalDiskRepo(localDiskRepo string) (string, error) { +func (n *Nexus3) repoFormatLocalDiskRepo(localDiskRepo string) (repoFormatAndType, error) { cn := connection.Nexus3{ BasePathPrefix: n.BasePathPrefix, FQDN: n.FQDN, @@ -85,19 +90,21 @@ func (n *Nexus3) repoFormatLocalDiskRepo(localDiskRepo string) (string, error) { a := artifacts.Nexus3{Nexus3: &cn} repos, err := a.Repos() if err != nil { - return "", err + return repoFormatAndType{}, err } var repoFormat string + var repoType string for _, repo := range repos { repoName := repo.Name if repoName == localDiskRepo { repoFormat = repo.Format + repoType = repo.Type } } - log.Infof("format of repo: '%s' is: '%s'", localDiskRepo, repoFormat) + log.Infof("format of repo: '%s' is: '%s' and repoType: '%s'", localDiskRepo, repoFormat, repoType) - return repoFormat, nil + return repoFormatAndType{repoFormat, repoType}, nil } func maven(path string, skipErrors bool) (mp mavenParts, err error) { @@ -298,7 +305,7 @@ func (n *Nexus3) checkLocalChecksumAndCompareWithOneInRemote(f, localDiskRepo, d if err != nil { return false, err } - log.Info(f, downloadedFileChecksum) + log.Debugf("checksum of file: '%s' is '%s'", f, downloadedFileChecksum) retryClient := retryablehttp.NewClient() retryClient.Logger = nil @@ -311,7 +318,7 @@ func (n *Nexus3) checkLocalChecksumAndCompareWithOneInRemote(f, localDiskRepo, d } u := scheme + "://" + n.FQDN + "/repository/" + localDiskRepo + "/" + dir + "/" + filename + ".sha512" - log.Info("URL: ", u) + log.Debugf("upload URL: '%s'", u) req, err := http.NewRequest("GET", u, nil) if err != nil { @@ -335,10 +342,10 @@ func (n *Nexus3) checkLocalChecksumAndCompareWithOneInRemote(f, localDiskRepo, d }() bodyBytes, err := io.ReadAll(resp.Body) if err != nil { - return identical, err + return false, err } bodyString := string(bodyBytes) - log.Info(bodyString) + log.Debugf("checksum of artifact in nexus3: '%s'", bodyString) if bodyString == downloadedFileChecksum { identical = true @@ -347,7 +354,7 @@ func (n *Nexus3) checkLocalChecksumAndCompareWithOneInRemote(f, localDiskRepo, d return identical, nil } -func (n *Nexus3) UploadSingleArtifact(client *client.Nexus3, path, localDiskRepo, localDiskRepoHome, repoFormat string, skipErrors bool) error { +func (n *Nexus3) UploadSingleArtifact(client *client.Nexus3, path, localDiskRepo, localDiskRepoHome, repoFormat string, skipErrors bool) (bool, error) { dir := strings.Replace(filepath.Dir(path), localDiskRepoHome+"/", "", -1) filename := filepath.Base(path) @@ -355,8 +362,8 @@ func (n *Nexus3) UploadSingleArtifact(client *client.Nexus3, path, localDiskRepo af := artifactFiles{} if identical, _ := n.checkLocalChecksumAndCompareWithOneInRemote(filepath.Clean(path), localDiskRepo, dir, filename); identical { - log.Info("Already uploaded") - return nil + log.Debugf("artifact: '%s' has already been uploaded", filename) + return true, nil } c := components.UploadComponentParams{} @@ -365,7 +372,7 @@ func (n *Nexus3) UploadSingleArtifact(client *client.Nexus3, path, localDiskRepo c.Repository = localDiskRepo f, err := os.Open(filepath.Clean(path)) if err != nil { - return err + return false, err } c.AptAsset = f case "maven2": @@ -375,7 +382,7 @@ func (n *Nexus3) UploadSingleArtifact(client *client.Nexus3, path, localDiskRepo filePathPom := fileNameWithoutExtIncludingDir + ".pom" if slices.Contains(checkedMavenFolders, dirPath) { - return nil + return false, nil } if _, err := os.Stat(filePathPom); err == nil { @@ -384,13 +391,13 @@ func (n *Nexus3) UploadSingleArtifact(client *client.Nexus3, path, localDiskRepo c.Repository = localDiskRepo if err := af.mavenJarAndOtherExtensions(&c, fileNameWithoutExtIncludingDir, skipErrors); err != nil { - return err + return false, err } var err error f, err = os.Open(filepath.Clean(filePathPom)) if err != nil { - return err + return false, err } c.Maven2Asset1 = f ext1 := "pom" @@ -441,13 +448,13 @@ func (n *Nexus3) UploadSingleArtifact(client *client.Nexus3, path, localDiskRepo c.Repository = localDiskRepo if err := af.mavenJarAndOtherExtensions(&c, fileNameWithoutExtIncludingDir, skipErrors); err != nil { - return err + return false, err } // mp, err := maven(path, skipErrors) if err != nil { - return err + return false, err } c.Maven2ArtifactID = &mp.artifact c.Maven2Version = &mp.version @@ -463,7 +470,7 @@ func (n *Nexus3) UploadSingleArtifact(client *client.Nexus3, path, localDiskRepo groupID = match[1] groupID = strings.ReplaceAll(groupID, `/`, `.`) } else { - return fmt.Errorf("groupID should not be empty, path: '%s' and regex: '%s'", path, regex) + return false, fmt.Errorf("groupID should not be empty, path: '%s' and regex: '%s'", path, regex) } c.Maven2GroupID = &groupID generatePOM := true @@ -516,21 +523,21 @@ func (n *Nexus3) UploadSingleArtifact(client *client.Nexus3, path, localDiskRepo c.Repository = localDiskRepo f, err := os.Open(filepath.Clean(path)) if err != nil { - return err + return false, err } c.NpmAsset = f case "nuget": c.Repository = localDiskRepo f, err := os.Open(filepath.Clean(path)) if err != nil { - return err + return false, err } c.NugetAsset = f case "raw": c.Repository = localDiskRepo f, err := os.Open(filepath.Clean(path)) if err != nil { - return err + return false, err } c.RawAsset1 = f c.RawDirectory = &dir @@ -541,7 +548,7 @@ func (n *Nexus3) UploadSingleArtifact(client *client.Nexus3, path, localDiskRepo c.Repository = localDiskRepo f, err := os.Open(filepath.Clean(path)) if err != nil { - return err + return false, err } c.RubygemsAsset = f } @@ -549,20 +556,20 @@ func (n *Nexus3) UploadSingleArtifact(client *client.Nexus3, path, localDiskRepo c.Repository = localDiskRepo f, err := os.Open(filepath.Clean(path)) if err != nil { - return err + return false, err } c.YumAsset = f c.YumAssetFilename = &filename default: - return nil + return false, nil } files := []*os.File{f, af.f2, af.f3, af.f4, af.f5, af.f6, af.f2, af.f7} if err := upload(c, client, path, files); err != nil { - return err + return false, err } - return nil + return false, nil } func upload(c components.UploadComponentParams, client *client.Nexus3, path string, files []*os.File) error { @@ -595,6 +602,35 @@ func upload(c components.UploadComponentParams, client *client.Nexus3, path stri return nil } +func (n *Nexus3) uploadAndPrintRepoFormat(c *client.Nexus3, path, localDiskRepo, localDiskRepoHome, repoFormat string, skipErrors bool) error { + identical, err := n.UploadSingleArtifact(c, path, localDiskRepo, localDiskRepoHome, repoFormat, skipErrors) + if err != nil { + uploaded, errRegex := regexp.MatchString("status 400", err.Error()) + if errRegex != nil { + return err + } + if uploaded { + log.Debugf("artifact: '%s' has already been uploaded", path) + return nil + } + + errString := fmt.Errorf("could not upload artifact: '%s', err: '%w'", path, err) + if n.SkipErrors { + log.Error(errString) + } else { + return errString + } + } else { + artifacts.PrintType(repoFormat) + } + if identical { + log.Debugf("checksum file: '%s' locally is identical compared to one in nexus", path) + return nil + } + + return nil +} + func (n *Nexus3) ReadLocalDirAndUploadArtifacts(localDiskRepoHome, localDiskRepo, repoFormat string) error { var wg sync.WaitGroup @@ -617,24 +653,8 @@ func (n *Nexus3) ReadLocalDirAndUploadArtifacts(localDiskRepoHome, localDiskRepo go func(path, localDiskRepo, localDiskRepoHome, repoFormat string, skipErrors bool) { defer wg.Done() - if err := n.UploadSingleArtifact(c, path, localDiskRepo, localDiskRepoHome, repoFormat, skipErrors); err != nil { - uploaded, errRegex := regexp.MatchString("status 400", err.Error()) - if errRegex != nil { - panic(err) - } - if uploaded { - log.Debugf("artifact: '%s' has already been uploaded", path) - return - } - - errString := fmt.Errorf("could not upload artifact: '%s', err: '%w'", path, err) - if n.SkipErrors { - log.Error(errString) - } else { - panic(errString) - } - } else { - artifacts.PrintType(repoFormat) + if err := n.uploadAndPrintRepoFormat(c, path, localDiskRepo, localDiskRepoHome, repoFormat, skipErrors); err != nil { + panic(err) } }(path, localDiskRepo, localDiskRepoHome, repoFormat, n.SkipErrors) } @@ -676,8 +696,17 @@ func (n *Nexus3) maven2SnapshotsUpload(localDiskRepo string) { log.Tracef("VersionPolicy: '%s'", vp) if strings.EqualFold(vp, "snapshot") { - s := snapshot.Nexus3{DownloadDirName: n.DownloadDirName, FQDN: n.FQDN, Pass: n.Pass, RepoFormat: "maven2", RepoName: localDiskRepo, SkipErrors: n.SkipErrors, User: n.User} + s := snapshot.Nexus3{DownloadDirName: n.DownloadDirName, FQDN: n.FQDN, HTTPS: *n.HTTPS, Pass: n.Pass, RepoFormat: "maven2", RepoName: localDiskRepo, SkipErrors: n.SkipErrors, User: n.User} + if err := s.Upload(); err != nil { + uploaded, errRegex := regexp.MatchString("bad status: 400 Repository does not allow updating assets", err.Error()) + if errRegex != nil { + panic(err) + } + if uploaded { + log.Debugf("artifact from localDiskRepo: '%s' has been uploaded already", localDiskRepo) + return + } if !n.SkipErrors { panic(err) } @@ -698,22 +727,26 @@ func (n *Nexus3) uploadArtifactsSingleDir(localDiskRepo string) { return } - repoFormat, err := n.repoFormatLocalDiskRepo(localDiskRepo) + repoFormatAndType, err := n.repoFormatLocalDiskRepo(localDiskRepo) if err != nil { panic(err) } - if repoFormat == "" { + if repoFormatAndType.format == "" { log.Errorf("repoFormat not detected. Verify whether repo: '%s' resides in Nexus", localDiskRepo) return } - if repoFormat == "maven2" { - n.maven2SnapshotsUpload(localDiskRepo) - } + log.Warnf("only uploads to 'hosted' repositories are supported. Current: '%v'", repoFormatAndType) + if repoFormatAndType.repoType == "hosted" { + if repoFormatAndType.format == "maven2" { + log.Info("upload to snapshot repo") + n.maven2SnapshotsUpload(localDiskRepo) + } - localDiskRepoHome := filepath.Join(n.DownloadDirName, localDiskRepo) - if err := n.ReadLocalDirAndUploadArtifacts(localDiskRepoHome, localDiskRepo, repoFormat); err != nil { - panic(err) + localDiskRepoHome := filepath.Join(n.DownloadDirName, localDiskRepo) + if err := n.ReadLocalDirAndUploadArtifacts(localDiskRepoHome, localDiskRepo, repoFormatAndType.format); err != nil { + panic(err) + } } } diff --git a/internal/app/n3dr/config/repository/repository.go b/internal/app/n3dr/config/repository/repository.go index f2867971..23cdd58b 100644 --- a/internal/app/n3dr/config/repository/repository.go +++ b/internal/app/n3dr/config/repository/repository.go @@ -31,6 +31,42 @@ func created(name string, err error) error { return fmt.Errorf("could not create repository: '%v', err: '%w'", name, err) } +func (r *Repository) CreateMavenGroup(memberNames []string, name string) error { + log.Infof("creating maven group: '%s'...", name) + client, err := r.Nexus3.Client() + if err != nil { + return err + } + if name == "" { + return fmt.Errorf("repo name should not be empty") + } + if len(memberNames) == 0 { + return fmt.Errorf("memberNames should not be empty") + } + + online := true + mhsa := models.StorageAttributes{BlobStoreName: "default", StrictContentTypeValidation: &r.StrictContentTypeValidation} + group := models.GroupAttributes{MemberNames: memberNames} + body := models.MavenGroupRepositoryAPIRequest{ + Group: &group, + Name: &name, + Online: &online, + Storage: &mhsa, + } + createMavenGroup := repository_management.CreateRepositoryParams{Body: &body} + createMavenGroup.WithTimeout(time.Second * 30) + if createRepositoryCreated, err := client.RepositoryManagement.CreateRepository(&createMavenGroup); err != nil { + log.Debugf("createRepositoryCreated: '%v'", createRepositoryCreated) + log.Tracef("createRepositoryCreatedError '%v'", err) + if err := created(name, err); err != nil { + return err + } + } + log.Infof("created the following maven group: '%v'", name) + + return nil +} + func (r *Repository) CreateAptProxied(name string) error { log.Infof("Creating proxied apt repository: '%s'...", name) client, err := r.Nexus3.Client() @@ -68,6 +104,42 @@ func (r *Repository) CreateAptProxied(name string) error { return nil } +func (r *Repository) CreateNpmProxied(name string) error { + log.Infof("Creating npm proxy: '%s'...", name) + client, err := r.Nexus3.Client() + if err != nil { + return err + } + if name == "" { + return fmt.Errorf("repo name should not be empty") + } + + httpClientBlocked := false + httpClientAutoBlocked := true + httpClient := models.HTTPClientAttributes{AutoBlock: &httpClientAutoBlocked, Blocked: &httpClientBlocked} + negativeCacheEnabled := true + var negativeCacheTimeToLive int32 = 1440 + negativeCache := models.NegativeCacheAttributes{Enabled: &negativeCacheEnabled, TimeToLive: &negativeCacheTimeToLive} + var contentMaxAge int32 = 1440 + var metadataMaxAge int32 = 1440 + remoteURL := r.ProxyRemoteURL + proxy := models.ProxyAttributes{ContentMaxAge: &contentMaxAge, MetadataMaxAge: &metadataMaxAge, RemoteURL: remoteURL} + online := true + npm := models.NpmAttributes{} + mhsa := models.StorageAttributes{BlobStoreName: "default", StrictContentTypeValidation: &r.StrictContentTypeValidation} + ma := models.NpmProxyRepositoryAPIRequest{Npm: &npm, Name: &name, Online: &online, Storage: &mhsa, Proxy: &proxy, NegativeCache: &negativeCache, HTTPClient: &httpClient} + createNpmProxy := repository_management.CreateRepository10Params{Body: &ma} + createNpmProxy.WithTimeout(time.Second * 30) + if _, err := client.RepositoryManagement.CreateRepository10(&createNpmProxy); err != nil { + if err := created(name, err); err != nil { + return err + } + } + log.Infof("created the following repository: '%v'", name) + + return nil +} + func (r *Repository) CreateYumProxied(name string) error { log.Infof("Creating proxied yum repository: '%s'...", name) client, err := r.Nexus3.Client() @@ -103,6 +175,43 @@ func (r *Repository) CreateYumProxied(name string) error { return nil } +func (r *Repository) CreateMavenProxied(name string) error { + log.Infof("creating the following maven proxy: '%s'...", name) + client, err := r.Nexus3.Client() + if err != nil { + return err + } + remoteURL := r.ProxyRemoteURL + log.Infof("remoteURL: '%s'", remoteURL) + if name == "" || remoteURL == "" { + return fmt.Errorf("repo name of proxy url should not be empty") + } + + httpClientBlocked := false + httpClientAutoBlocked := true + httpClient := models.HTTPClientAttributesWithPreemptiveAuth{AutoBlock: &httpClientAutoBlocked, Blocked: &httpClientBlocked} + negativeCacheEnabled := true + var negativeCacheTimeToLive int32 = 1440 + negativeCache := models.NegativeCacheAttributes{Enabled: &negativeCacheEnabled, TimeToLive: &negativeCacheTimeToLive} + var contentMaxAge int32 = 1440 + var metadataMaxAge int32 = 1440 + proxy := models.ProxyAttributes{ContentMaxAge: &contentMaxAge, MetadataMaxAge: &metadataMaxAge, RemoteURL: remoteURL} + online := true + maven := models.MavenAttributes{LayoutPolicy: "STRICT", VersionPolicy: "MIXED"} + mhsa := models.StorageAttributes{BlobStoreName: "default", StrictContentTypeValidation: &r.StrictContentTypeValidation} + ma := models.MavenProxyRepositoryAPIRequest{Maven: &maven, Name: &name, Online: &online, Storage: &mhsa, Proxy: &proxy, NegativeCache: &negativeCache, HTTPClient: &httpClient} + createMavenProxy := repository_management.CreateRepository2Params{Body: &ma} + createMavenProxy.WithTimeout(time.Second * 30) + if _, err := client.RepositoryManagement.CreateRepository2(&createMavenProxy); err != nil { + if err := created(name, err); err != nil { + return err + } + } + log.Infof("created the following maven proxy: '%v'", name) + + return nil +} + func (r *Repository) CreateDockerHosted(secure bool, port int32, name string) error { log.Infof("Creating docker hosted repository: '%s'...", name) client, err := r.Nexus3.Client() diff --git a/test/integration-tests.sh b/test/integration-tests.sh index 16535d9a..534d82b6 100755 --- a/test/integration-tests.sh +++ b/test/integration-tests.sh @@ -409,7 +409,7 @@ repositories() { --dockerHost ${DOCKER_URL} count_downloads 354 ${testZipSizeDir} - test_zip 1399 ${testZipSizeDir} + test_zip "1403[0-9]\{2\}" ${testZipSizeDir} cleanup_downloads }