From 5c72852e8d4299252be0fd1c81d8280fb24f4a7c Mon Sep 17 00:00:00 2001 From: ben Date: Wed, 24 Nov 2021 14:15:19 +0100 Subject: [PATCH] [GH-215] Improve performance. --- .github/workflows/dip.yml | 2 +- .github/workflows/dive.yml | 2 +- .github/workflows/dockle.yml | 2 +- .github/workflows/go.yml | 16 +- .github/workflows/hadolint.yml | 2 +- .github/workflows/integration.yml | 40 +++ .github/workflows/trivy.yml | 2 +- .gitignore | 1 + CHANGELOG.md | 18 +- Dockerfile | 4 +- README.md | 70 ++++- build/package/snap/snapcraft.yaml | 2 +- cmd/n3dr/backup.go | 2 +- cmd/n3dr/config.go | 4 +- cmd/n3dr/configRepository.go | 4 +- cmd/n3dr/configUser.go | 4 +- cmd/n3dr/repositories.go | 2 +- cmd/n3dr/repositoriesV2.go | 63 ++++ cmd/n3dr/root.go | 6 +- cmd/n3dr/sync.go | 78 +++++ internal/artifacts/common.go | 23 +- internal/artifacts/repositories.go | 2 +- internal/artifactsv2/download.go | 188 ++++++++++++ internal/artifactsv2/upload/upload.go | 283 ++++++++++++++++++ internal/config/repository/repository.go | 4 +- internal/config/security/anonymous.go | 4 +- internal/config/user/user.go | 6 +- internal/pkg/artifactsv2/artifacts/file.go | 74 +++++ .../pkg/artifactsv2/artifacts/repository.go | 25 ++ .../pkg/{http => connection}/connection.go | 11 +- test/integration-tests.sh | 160 +++++++--- 31 files changed, 998 insertions(+), 106 deletions(-) create mode 100644 .github/workflows/integration.yml create mode 100644 cmd/n3dr/repositoriesV2.go create mode 100644 cmd/n3dr/sync.go create mode 100644 internal/artifactsv2/download.go create mode 100644 internal/artifactsv2/upload/upload.go create mode 100644 internal/pkg/artifactsv2/artifacts/file.go create mode 100644 internal/pkg/artifactsv2/artifacts/repository.go rename internal/pkg/{http => connection}/connection.go (56%) diff --git a/.github/workflows/dip.yml b/.github/workflows/dip.yml index 5bf6254a..2a43aeea 100644 --- a/.github/workflows/dip.yml +++ b/.github/workflows/dip.yml @@ -1,5 +1,5 @@ --- -name: "DIP" +name: DIP on: [push] jobs: dive: diff --git a/.github/workflows/dive.yml b/.github/workflows/dive.yml index 7aa747fc..6d857d8e 100644 --- a/.github/workflows/dive.yml +++ b/.github/workflows/dive.yml @@ -1,5 +1,5 @@ --- -name: "Dive CI" +name: Dive CI on: [push] jobs: dive: diff --git a/.github/workflows/dockle.yml b/.github/workflows/dockle.yml index 829a1ff5..f15fb707 100644 --- a/.github/workflows/dockle.yml +++ b/.github/workflows/dockle.yml @@ -1,5 +1,5 @@ --- -name: "Dockle" +name: Dockle on: [push] jobs: dive: diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 53aaea1e..458cb893 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -16,7 +16,7 @@ jobs: nexus-api-version: beta shasum: shasum -a 512 - os: ubuntu-20.04 - nexus-version: 3.34.1 + nexus-version: 3.37.0 nexus-api-version: v1 shasum: sha512sum - os: ubuntu-20.04 @@ -36,12 +36,6 @@ jobs: - name: Unit tests run: go test ./... -cover if: ${{ startsWith(matrix.os, 'ubuntu') }} - - name: Install bats - run: | - set -x - sudo apt-get update - sudo apt-get install bats -y - if: ${{ startsWith(matrix.os, 'ubuntu') }} - name: Set N3DR deliverable environment variable run: echo "n3dr-deliverable=n3dr-${{ matrix.os }}" >> $GITHUB_ENV - name: Use the value @@ -61,11 +55,3 @@ jobs: docker run --rm -v $(pwd):/app -w /app \ golangci/golangci-lint:v1.39.0-alpine golangci-lint run -v if: ${{ startsWith(matrix.os, 'ubuntu') }} - - name: Integration test - run: ./test/integration-tests.sh - env: - N3DR_DELIVERABLE: ${{ env.n3dr-deliverable }} - NEXUS_VERSION: ${{ matrix.nexus-version }} - NEXUS_API_VERSION: ${{ matrix.nexus-api-version }} - N3DR_APT_GPG_SECRET: ${{ secrets.N3DR_APT_GPG_SECRET }} - if: ${{ startsWith(matrix.os, 'ubuntu') }} diff --git a/.github/workflows/hadolint.yml b/.github/workflows/hadolint.yml index 5e595720..17ef0f21 100644 --- a/.github/workflows/hadolint.yml +++ b/.github/workflows/hadolint.yml @@ -1,4 +1,4 @@ -name: "Hadolint" +name: Hadolint on: [push] jobs: dive: diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml new file mode 100644 index 00000000..b6717b70 --- /dev/null +++ b/.github/workflows/integration.yml @@ -0,0 +1,40 @@ +--- +name: Integration +on: + push: + branches: [master] + pull_request: + branches: [master] +jobs: + build: + runs-on: ${{ matrix.os }} + strategy: + matrix: + include: + - os: ubuntu-20.04 + nexus-version: 3.37.0 + nexus-api-version: v1 + shasum: sha512sum + - os: ubuntu-20.04 + nexus-version: 3.9.0 + nexus-api-version: beta + shasum: sha512sum + steps: + - uses: actions/checkout@v2 + - name: Set up Go + uses: actions/setup-go@v2 + with: + go-version: 1.17.0 + - name: Install bats + run: | + set -x + sudo apt-get update + sudo apt-get install bats -y + - name: Integration test + run: | + ./test/integration-tests.sh + env: + N3DR_DELIVERABLE: ${{ env.n3dr-deliverable }} + NEXUS_VERSION: ${{ matrix.nexus-version }} + NEXUS_API_VERSION: ${{ matrix.nexus-api-version }} + N3DR_APT_GPG_SECRET: ${{ secrets.N3DR_APT_GPG_SECRET }} diff --git a/.github/workflows/trivy.yml b/.github/workflows/trivy.yml index fb147a1a..153d1d0d 100644 --- a/.github/workflows/trivy.yml +++ b/.github/workflows/trivy.yml @@ -1,5 +1,5 @@ --- -name: trivy +name: Trivy on: [push] jobs: build: diff --git a/.gitignore b/.gitignore index 9c9a7e36..a080487a 100644 --- a/.gitignore +++ b/.gitignore @@ -23,3 +23,4 @@ virtualbox* *.exe swagger swagger.json +*.log diff --git a/CHANGELOG.md b/CHANGELOG.md index e4a9cae5..3546b65c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [6.2.0] - 2021-11-30 + +### Added + +- Nuget downloads. +- Sync between multiple nexus repositories. + +### Changed + +- Moved the integration test to a separate github workflow file. + +### Fixed + +- Inconsistent file zip size while running `./test/integration.sh`. + ## [6.1.0] - 2021-11-23 ### Added @@ -566,7 +581,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Download all artifacts from a certain Nexus3 repository. -[Unreleased]: https://github.com/030/n3dr/compare/6.1.0...HEAD +[Unreleased]: https://github.com/030/n3dr/compare/6.2.0...HEAD +[6.2.0]: https://github.com/030/n3dr/compare/6.1.0...6.2.0 [6.1.0]: https://github.com/030/n3dr/compare/6.0.13...6.1.0 [6.0.13]: https://github.com/030/n3dr/compare/6.0.12...6.0.13 [6.0.12]: https://github.com/030/n3dr/compare/6.0.11...6.0.12 diff --git a/Dockerfile b/Dockerfile index 9e72f2bb..ea2217f5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.17.3-alpine3.14 as builder +FROM golang:1.17.3-alpine3.15 as builder ENV USERNAME n3dr RUN adduser -D -g '' $USERNAME COPY . /go/${USERNAME}/ @@ -7,7 +7,7 @@ RUN apk add --no-cache git=~2 && \ CGO_ENABLED=0 go build && \ cp n3dr /n3dr -FROM alpine:3.14.3 +FROM alpine:3.15.0 COPY --from=builder /etc/passwd /etc/passwd COPY --from=builder /n3dr /usr/local/bin/n3dr COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ diff --git a/README.md b/README.md index 62cf02be..85b52a85 100644 --- a/README.md +++ b/README.md @@ -301,7 +301,7 @@ n3dr config \ ### Build ```bash -docker build -t utrecht/n3dr:6.1.0 . +docker build -t utrecht/n3dr:6.2.0 . ``` [![dockeri.co](https://dockeri.co/image/utrecht/n3dr)](https://hub.docker.com/r/utrecht/n3dr) @@ -311,7 +311,7 @@ docker build -t utrecht/n3dr:6.1.0 . ```bash docker run -it \ -v /home/${USER}/.n3dr:/root/.n3dr \ - -v /tmp/n3dr:/tmp/n3dr utrecht/n3dr:6.1.0 + -v /tmp/n3dr:/tmp/n3dr utrecht/n3dr:6.2.0 ``` ### Upload @@ -320,7 +320,7 @@ docker run -it \ docker run -it \ --entrypoint=/bin/ash \ -v /home/${USER}/.n3dr:/root/.n3dr \ - -v /tmp/n3dr:/tmp/n3dr utrecht/n3dr:6.1.0 + -v /tmp/n3dr:/tmp/n3dr utrecht/n3dr:6.2.0 ``` navigate to the repository folder, e.g. `/tmp/n3dr/download*/` and upload: @@ -519,6 +519,70 @@ a single command. | npm | x | x | `*` | | nuget | | x | | +### repositoriesV2 + +| type | backup | upload | label | +|-----------|--------|--------|-------| +| apt | x | x | `^` | +| bower | | | | +| cocoapods | | | | +| conan | | | | +| docker | | | | +| gitlfs | | | | +| go | | | | +| helm | | | | +| maven2 | x | | `+` | +| npm | x | x | `*` | +| nuget | x | x | `$` | +| p2 | | | | +| pypi | | | | +| r | | | | +| raw | x | x | `%` | +| rubygems | | | | +| yum | | | | +| unknown | x | x | `?` | + +#### backup + +`repositoriesV2` command in conjunction with the `--backup` subcommand ensures +that all artifacts are downloaded from Nexus3 and stored in a folder that is +defined with the `--directory-prefix` parameter. + +```bash +n3dr repositoriesV2 \ + --backup \ + -n some-url \ + -u some-user \ + -p some-pass \ + --directory-prefix /tmp/some-dir +``` + +#### upload + +```bash +n3dr repositoriesV2 \ + --upload \ + -n some-url \ + -u some-user \ + -p some-pass \ + --directory-prefix /tmp/some-dir +``` + +#### sync + +```bash +n3dr sync \ + --otherNexus3Passwords=some-pass-B,some-pass-C \ + --otherNexus3Users=admin,admin \ + --otherNexus3URLs=localhost:9998,localhost:9997 \ + --directory-prefix /some/dir \ + -p some-pass-A \ + -n localhost:9999 \ + -u admin +``` + +Note: use `--https=false` in order to connect to a . + ## Stargazers over time [![Stargazers over time](https://starchart.cc/030/n3dr.svg)](https://starchart.cc/030/n3dr) diff --git a/build/package/snap/snapcraft.yaml b/build/package/snap/snapcraft.yaml index 7606a20e..35aa89f4 100644 --- a/build/package/snap/snapcraft.yaml +++ b/build/package/snap/snapcraft.yaml @@ -1,7 +1,7 @@ --- name: n3dr base: core20 -version: 6.1.0 +version: 6.2.0 summary: Nexus3 Disaster Recovery description: | Download all artifacts at once or migrate automatically from Nexus to Nexus. diff --git a/cmd/n3dr/backup.go b/cmd/n3dr/backup.go index cc8fded9..5d062af0 100644 --- a/cmd/n3dr/backup.go +++ b/cmd/n3dr/backup.go @@ -43,7 +43,7 @@ reside in a certain Nexus3 repository`, } } - if err := n.CreateZip(dir); err != nil { + if err := n.CreateZip(dir, downloadDirNameZip); err != nil { log.Fatal(err) } } diff --git a/cmd/n3dr/config.go b/cmd/n3dr/config.go index e85d70e2..0f5490ea 100644 --- a/cmd/n3dr/config.go +++ b/cmd/n3dr/config.go @@ -4,7 +4,7 @@ import ( "fmt" "github.com/030/n3dr/internal/config/security" - "github.com/030/n3dr/internal/pkg/http" + "github.com/030/n3dr/internal/pkg/connection" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" ) @@ -19,7 +19,7 @@ var configCmd = &cobra.Command{ Run: func(cmd *cobra.Command, args []string) { fmt.Println("config called") - n := http.Nexus3{FQDN: n3drURL, Pass: n3drPass, User: n3drUser} + n := connection.Nexus3{FQDN: n3drURL, Pass: n3drPass, User: n3drUser} s := security.Security{Nexus3: n} if err := s.Anonymous(configUserAnonymous); err != nil { log.Fatal(err) diff --git a/cmd/n3dr/configRepository.go b/cmd/n3dr/configRepository.go index aacedf1d..aac63ba4 100644 --- a/cmd/n3dr/configRepository.go +++ b/cmd/n3dr/configRepository.go @@ -5,7 +5,7 @@ import ( "os" "github.com/030/n3dr/internal/config/repository" - "github.com/030/n3dr/internal/pkg/http" + "github.com/030/n3dr/internal/pkg/connection" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" ) @@ -25,7 +25,7 @@ var configRepositoryCmd = &cobra.Command{ Run: func(cmd *cobra.Command, args []string) { fmt.Println("configRepository called") - n := http.Nexus3{FQDN: n3drURL, Pass: n3drPass, User: n3drUser} + n := connection.Nexus3{FQDN: n3drURL, Pass: n3drPass, User: n3drUser} r := repository.Repository{Nexus3: n} if configRepoDelete { diff --git a/cmd/n3dr/configUser.go b/cmd/n3dr/configUser.go index 94b398ba..639545b3 100644 --- a/cmd/n3dr/configUser.go +++ b/cmd/n3dr/configUser.go @@ -5,7 +5,7 @@ import ( "github.com/030/n3dr/internal/config/user" "github.com/030/n3dr/internal/goswagger/models" - "github.com/030/n3dr/internal/pkg/http" + "github.com/030/n3dr/internal/pkg/connection" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" ) @@ -39,7 +39,7 @@ to quickly create a Cobra application.`, Password: pass, UserID: id, } - n := http.Nexus3{FQDN: n3drURL, Pass: n3drPass, User: n3drUser} + n := connection.Nexus3{FQDN: n3drURL, Pass: n3drPass, User: n3drUser} u := user.User{APICreateUser: acu, Nexus3: n} if admin { diff --git a/cmd/n3dr/repositories.go b/cmd/n3dr/repositories.go index 1075a919..39599c83 100644 --- a/cmd/n3dr/repositories.go +++ b/cmd/n3dr/repositories.go @@ -23,7 +23,7 @@ download artifacts from all repositories`, } log.Fatal("One of the required flags \"names\", \"count\" or \"backup\" not set") } - n := cli.Nexus3{URL: n3drURL, User: n3drUser, Pass: n3drPass, APIVersion: apiVersion, ZIP: zip, ZipName: zipName, DownloadDirName: downloadDirName} + n := cli.Nexus3{URL: n3drURL, User: n3drUser, Pass: n3drPass, APIVersion: apiVersion, ZIP: zip, ZipName: zipName, DownloadDirName: downloadDirName, DownloadDirNameZip: downloadDirNameZip} if err := n.ValidateNexusURL(); err != nil { log.Fatal(err) } diff --git a/cmd/n3dr/repositoriesV2.go b/cmd/n3dr/repositoriesV2.go new file mode 100644 index 00000000..7fb2b646 --- /dev/null +++ b/cmd/n3dr/repositoriesV2.go @@ -0,0 +1,63 @@ +package main + +import ( + "github.com/030/n3dr/internal/artifactsv2" + "github.com/030/n3dr/internal/artifactsv2/upload" + "github.com/030/n3dr/internal/pkg/connection" + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" +) + +var ( + countV2, namesV2, backupV2, uploadV2 bool +) + +// repositoriesCmd represents the repositories command +var repositoriesV2Cmd = &cobra.Command{ + Use: "repositoriesV2", + Short: "Count the number of repositories or return their names", + Long: `Count the number of repositories, count the total or +download artifacts from all repositories`, + Run: func(cmd *cobra.Command, args []string) { + if !(namesV2 || countV2 || backupV2 || uploadV2) { + if err := cmd.Help(); err != nil { + log.Fatal(err) + } + log.Fatal("One of the required flags \"names\", \"count\" or \"backup\" not set") + } + n := connection.Nexus3{FQDN: n3drURL, Pass: n3drPass, User: n3drUser, DownloadDirName: downloadDirName, HTTPS: https} + a := artifactsv2.Nexus3{Nexus3: &n} + if namesV2 { + if err := a.RepositoryNamesV2(); err != nil { + log.Fatal(err) + } + } + if countV2 { + if err := a.CountRepositoriesV2(); err != nil { + log.Fatal(err) + } + } + if backupV2 { + if err := a.Backup(); err != nil { + log.Fatal(err) + } + } + if uploadV2 { + u := upload.Nexus3{Nexus3: &n} + if err := u.Upload(); err != nil { + log.Fatal(err) + } + } + }, + Version: rootCmd.Version, +} + +func init() { + repositoriesV2Cmd.Flags().BoolVarP(&namesV2, "names", "", false, "print all repository names") + repositoriesV2Cmd.Flags().BoolVarP(&countV2, "count", "", false, "count the number of repositories") + repositoriesV2Cmd.Flags().BoolVarP(&backupV2, "backup", "", false, "backup artifacts from all repositories") + repositoriesV2Cmd.Flags().BoolVarP(&uploadV2, "upload", "", false, "upload artifacts from all repositories") + repositoriesV2Cmd.Flags().StringVarP(®ex, "regex", "x", ".*", "only download artifacts that match a regular expression, e.g. 'some/group42'") + + rootCmd.AddCommand(repositoriesV2Cmd) +} diff --git a/cmd/n3dr/root.go b/cmd/n3dr/root.go index f118021a..bbf594a3 100644 --- a/cmd/n3dr/root.go +++ b/cmd/n3dr/root.go @@ -19,8 +19,8 @@ import ( ) var ( - apiVersion, cfgFile, n3drRepo, n3drURL, n3drPass, n3drUser, Version, zipName, downloadDirName string - anonymous, debug, insecureSkipVerify, skipErrors, zip bool + apiVersion, cfgFile, n3drRepo, n3drURL, n3drPass, n3drUser, Version, zipName, downloadDirName, downloadDirNameZip string + anonymous, debug, https, insecureSkipVerify, skipErrors, zip bool ) var rootCmd = &cobra.Command{ @@ -50,9 +50,11 @@ func init() { rootCmd.PersistentFlags().StringVarP(&n3drURL, "n3drURL", "n", "", "nexus3 URL") rootCmd.PersistentFlags().StringVarP(&n3drUser, "n3drUser", "u", "", "nexus3 user") rootCmd.PersistentFlags().StringVar(&downloadDirName, "directory-prefix", "", "directory to store downloaded artifacts") + rootCmd.PersistentFlags().StringVar(&downloadDirNameZip, "directory-prefix-zip", "", "directory to store the zipped artifacts") rootCmd.PersistentFlags().StringVarP(&apiVersion, "apiVersion", "v", "v1", "nexus3 APIVersion, e.g. v1 or beta") rootCmd.PersistentFlags().BoolVar(&anonymous, "anonymous", false, "Skip authentication") rootCmd.PersistentFlags().BoolVarP(&skipErrors, "skipErrors", "s", false, "Skip errors") + rootCmd.PersistentFlags().BoolVarP(&https, "https", "", true, "https true or false") } func n3drHiddenHome() (string, error) { diff --git a/cmd/n3dr/sync.go b/cmd/n3dr/sync.go new file mode 100644 index 00000000..2648fbf5 --- /dev/null +++ b/cmd/n3dr/sync.go @@ -0,0 +1,78 @@ +package main + +import ( + "fmt" + "sync" + + "github.com/030/n3dr/internal/artifactsv2" + "github.com/030/n3dr/internal/artifactsv2/upload" + "github.com/030/n3dr/internal/pkg/connection" + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" +) + +var otherNexus3URLs, otherNexus3Users, otherNexus3Passwords []string + +// syncCmd represents the sync command +var syncCmd = &cobra.Command{ + Use: "sync", + Short: "sync", + Long: `sync`, + Run: func(cmd *cobra.Command, args []string) { + fmt.Println("sync called") + numberOfURLs := len(otherNexus3URLs) + numberOfPasswords := len(otherNexus3Passwords) + numberOfUsers := len(otherNexus3Users) + if numberOfURLs != numberOfPasswords || numberOfURLs != numberOfUsers { + log.Fatal("incorrect number of elements. Ensure that the number of elements is identical") + } + + n := connection.Nexus3{FQDN: n3drURL, Pass: n3drPass, User: n3drUser, DownloadDirName: downloadDirName} + a := artifactsv2.Nexus3{Nexus3: &n} + if err := a.Backup(); err != nil { + log.Fatal(err) + } + + var errs []error + var wg sync.WaitGroup + for i, otherNexus3User := range otherNexus3Users { + n.User = otherNexus3User + n.Pass = otherNexus3Passwords[i] + n.FQDN = otherNexus3URLs[i] + + wg.Add(1) + go func(n connection.Nexus3) { + defer wg.Done() + + u := upload.Nexus3{Nexus3: &n} + if err := u.Upload(); err != nil { + errs = append(errs, err) + return + } + }(n) + } + wg.Wait() + for _, err := range errs { + if err != nil { + log.Fatal(err) + } + } + }, +} + +func init() { + rootCmd.AddCommand(syncCmd) + + syncCmd.PersistentFlags().StringSliceVarP(&otherNexus3URLs, "otherNexus3URLs", "", nil, "specify the other Nexus3 URLs in a comma separated list") + if err := syncCmd.MarkPersistentFlagRequired("otherNexus3URLs"); err != nil { + log.Fatal(err) + } + syncCmd.PersistentFlags().StringSliceVarP(&otherNexus3Users, "otherNexus3Users", "", nil, "specify the other Nexus3 users in a comma separated list") + if err := syncCmd.MarkPersistentFlagRequired("otherNexus3Users"); err != nil { + log.Fatal(err) + } + syncCmd.PersistentFlags().StringSliceVarP(&otherNexus3Passwords, "otherNexus3Passwords", "", nil, "specify the other Nexus3 passwords in a comma separated list") + if err := syncCmd.MarkPersistentFlagRequired("otherNexus3Passwords"); err != nil { + log.Fatal(err) + } +} diff --git a/internal/artifacts/common.go b/internal/artifacts/common.go index adfa39e7..fcb8289d 100644 --- a/internal/artifacts/common.go +++ b/internal/artifacts/common.go @@ -5,6 +5,7 @@ import ( "io/ioutil" "net/http" "os" + "path/filepath" "strconv" "strings" "time" @@ -30,9 +31,9 @@ const ( // Nexus3 contains the attributes that are used by several functions type Nexus3 struct { - URL string `validate:"nonzero,regexp=^http(s)?://[a-z0-9\\.-]+(:[0-9]+)?(/[a-z0-9\\.-]+)*$"` - APIVersion, ArtifactType, DownloadDirName, Pass, Repository, User, ZipName string - ZIP bool + URL string `validate:"nonzero,regexp=^http(s)?://[a-z0-9\\.-]+(:[0-9]+)?(/[a-z0-9\\.-]+)*$"` + APIVersion, ArtifactType, DownloadDirName, DownloadDirNameZip, Pass, Repository, User, ZipName string + ZIP bool } // RetryLogAdaptor adapts the retryablehttp.Logger interface to the logrus logger. @@ -128,21 +129,23 @@ func (n Nexus3) responseBodyString(resp *http.Response) ([]byte, string, error) } // CreateZip adds all artifacts to a ZIP archive -func (n Nexus3) CreateZip(dir string) error { +func (n Nexus3) CreateZip(dir string, zipDirDest string) (err error) { if n.ZIP { if n.ZipName == "" { n.ZipName = "n3dr-backup-" + time.Now().Format("01-02-2006T15-04-05") + ".zip" } - cwd, err := os.Getwd() - if err != nil { - return err + if zipDirDest == "" { + zipDirDest, err = os.Getwd() + if err != nil { + return err + } } - log.Warnf("Trying to create a zip file in: '%v'. Note that this could result in a 'permission denied' issue if N3DR has been installed using snap and is run in a different directory than your own home folder.", cwd) - err = archiver.Archive([]string{dir}, n.ZipName) + log.Warnf("Trying to create a zip file in: '%v'. Note that this could result in a 'permission denied' issue if N3DR has been installed using snap and is run in a different directory than your own home folder.", zipDirDest) + err = archiver.Archive([]string{dir}, filepath.Join(zipDirDest, n.ZipName)) if err != nil { return err } - log.Infof("Zipfile: '%v' created in '%v'", n.ZipName, cwd) + log.Infof("Zipfile: '%v' created in '%v'", n.ZipName, zipDirDest) } return nil } diff --git a/internal/artifacts/repositories.go b/internal/artifacts/repositories.go index f44b7a5b..82b88559 100644 --- a/internal/artifacts/repositories.go +++ b/internal/artifacts/repositories.go @@ -129,7 +129,7 @@ func (n *Nexus3) Downloads(regex string) error { if err := n.downloadAllArtifactsFromRepositories(dir, regex); err != nil { return err } - if err := n.CreateZip(dir); err != nil { + if err := n.CreateZip(dir, n.DownloadDirNameZip); err != nil { return err } return nil diff --git a/internal/artifactsv2/download.go b/internal/artifactsv2/download.go new file mode 100644 index 00000000..7bcec22d --- /dev/null +++ b/internal/artifactsv2/download.go @@ -0,0 +1,188 @@ +package artifactsv2 + +import ( + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "sync" + "time" + + "github.com/030/n3dr/internal/goswagger/client/components" + "github.com/030/n3dr/internal/goswagger/models" + "github.com/030/n3dr/internal/pkg/artifactsv2/artifacts" + "github.com/030/n3dr/internal/pkg/connection" + "github.com/hashicorp/go-retryablehttp" + + log "github.com/sirupsen/logrus" +) + +const chmodDir = 0750 + +type Nexus3 struct { + *connection.Nexus3 +} + +func (n *Nexus3) RepositoryNamesV2() error { + cn := connection.Nexus3{FQDN: n.FQDN, DownloadDirName: n.DownloadDirName, Pass: n.Pass, User: n.User, HTTPS: n.HTTPS} + a := artifacts.Nexus3{Nexus3: &cn} + repos, err := a.Repos() + if err != nil { + return err + } + for _, repo := range repos { + fmt.Println(repo.Name) + } + return nil +} + +func (n *Nexus3) CountRepositoriesV2() error { + cn := connection.Nexus3{FQDN: n.FQDN, DownloadDirName: n.DownloadDirName, Pass: n.Pass, User: n.User, HTTPS: n.HTTPS} + a := artifacts.Nexus3{Nexus3: &cn} + repos, err := a.Repos() + if err != nil { + return err + } + fmt.Println(len(repos)) + return nil +} + +func (n *Nexus3) download(checksum, downloadedFileChecksum string, asset *models.AssetXO) (errs []error) { + if checksum != downloadedFileChecksum { + req, err := http.NewRequest("GET", asset.DownloadURL, nil) + if err != nil { + errs = append(errs, err) + } + req.SetBasicAuth(n.User, n.Pass) + + retryClient := retryablehttp.NewClient() + retryClient.Logger = nil + retryClient.RetryMax = 10 + standardClient := retryClient.StandardClient() + + dir := filepath.Dir(asset.Path) + if err := os.MkdirAll(filepath.Join(n.DownloadDirName, asset.Repository, dir), chmodDir); err != nil { + errs = append(errs, err) + } + + out, err := os.Create(filepath.Join(n.DownloadDirName, asset.Repository, asset.Path)) + if err != nil { + errs = append(errs, err) + } + defer func() { + if err := out.Close(); err != nil { + errs = append(errs, err) + } + }() + resp, err := standardClient.Do(req) + if err != nil { + errs = append(errs, err) + } + defer func() { + if err := resp.Body.Close(); err != nil { + errs = append(errs, err) + } + }() + + if resp.StatusCode != http.StatusOK { + err := fmt.Errorf("bad status: %s", resp.Status) + errs = append(errs, err) + } + _, err = io.Copy(out, resp.Body) + if err != nil { + errs = append(errs, err) + } + + artifacts.PrintType(asset.Format) + } + return nil +} + +func (n *Nexus3) downloadIfChecksumMismatchLocalFile(continuationToken, repo string) error { + var errs []error + var wg sync.WaitGroup + client := n.Nexus3.Client() + c := components.GetComponentsParams{ContinuationToken: &continuationToken, Repository: repo} + c.WithTimeout(time.Second * 60) + resp, err := client.Components.GetComponents(&c) + if err != nil { + return fmt.Errorf("cannot get components: '%v'", err) + } + continuationToken = resp.GetPayload().ContinuationToken + for _, item := range resp.GetPayload().Items { + for _, asset := range item.Assets { + wg.Add(1) + go func(asset *models.AssetXO) { + defer wg.Done() + shaType, checksum := artifacts.Checksum(asset) + + log.Debugf("downloadURL: '%s', format: '%s', checksum: '%s'", asset.DownloadURL, asset.Format, checksum) + + file := filepath.Join(n.DownloadDirName, repo, asset.Path) + downloadedFileChecksum, checksumLocalFileErrs := artifacts.ChecksumLocalFile(file, shaType) + for _, checksumLocalFileErr := range checksumLocalFileErrs { + if checksumLocalFileErr != nil { + errs = append(errs, checksumLocalFileErr) + return + } + } + + downloadErrs := n.download(checksum, downloadedFileChecksum, asset) + for _, downloadErr := range downloadErrs { + if downloadErr != nil { + errs = append(errs, downloadErr) + return + } + } + }(asset) + } + } + wg.Wait() + for _, err := range errs { + if err != nil { + return err + } + } + if continuationToken == "" { + return nil + } + if err := n.downloadIfChecksumMismatchLocalFile(continuationToken, repo); err != nil { + return err + } + + return nil +} + +func (n *Nexus3) Backup() error { + var errs []error + var wg sync.WaitGroup + + cn := connection.Nexus3{FQDN: n.FQDN, DownloadDirName: n.DownloadDirName, Pass: n.Pass, User: n.User, HTTPS: n.HTTPS} + a := artifacts.Nexus3{Nexus3: &cn} + repos, err := a.Repos() + if err != nil { + return err + } + for _, repo := range repos { + wg.Add(1) + go func(repo *models.AbstractAPIRepository) { + defer wg.Done() + log.Infof("backing up '%s', '%s', %s", repo.Name, repo.Type, repo.Format) + if err := os.MkdirAll(filepath.Join(n.DownloadDirName, repo.Name), chmodDir); err != nil { + errs = append(errs, err) + } + if err := n.downloadIfChecksumMismatchLocalFile("", repo.Name); err != nil { + errs = append(errs, err) + } + }(repo) + } + wg.Wait() + for _, err := range errs { + if err != nil { + return err + } + } + + return nil +} diff --git a/internal/artifactsv2/upload/upload.go b/internal/artifactsv2/upload/upload.go new file mode 100644 index 00000000..b72bda33 --- /dev/null +++ b/internal/artifactsv2/upload/upload.go @@ -0,0 +1,283 @@ +package upload + +import ( + "fmt" + "io/fs" + "os" + "path/filepath" + "reflect" + "regexp" + "strconv" + "strings" + "sync" + "time" + + "github.com/030/n3dr/internal/goswagger/client" + "github.com/030/n3dr/internal/goswagger/client/components" + "github.com/030/n3dr/internal/pkg/artifactsv2/artifacts" + "github.com/030/n3dr/internal/pkg/connection" + + log "github.com/sirupsen/logrus" +) + +type Nexus3 struct { + *connection.Nexus3 +} + +func uploadStatus(err error) (int, error) { + re := regexp.MustCompile(`status (\d{3})`) + match := re.FindStringSubmatch(err.Error()) + if len(match) != 2 { + return 0, fmt.Errorf("http status code not found") + } + statusCode := match[1] + statusCodeInt, err := strconv.Atoi(statusCode) + if err != nil { + return 0, err + } + + return statusCodeInt, nil +} + +func (n *Nexus3) reposOnDisk() (localDiskRepos []string, errs []error) { + file, err := os.Open(n.DownloadDirName) + if err != nil { + errs = append(errs, err) + return + } + defer func() { + if err := file.Close(); err != nil { + errs = append(errs, err) + return + } + }() + localDiskRepos, err = file.Readdirnames(0) + if err != nil { + log.Fatal(err) + } + return localDiskRepos, nil +} + +func (n *Nexus3) repoFormatLocalDiskRepo(localDiskRepo string) (string, error) { + cn := connection.Nexus3{FQDN: n.FQDN, DownloadDirName: n.DownloadDirName, Pass: n.Pass, User: n.User, HTTPS: n.HTTPS} + a := artifacts.Nexus3{Nexus3: &cn} + repos, err := a.Repos() + if err != nil { + return "", err + } + var repoFormat string + for _, repo := range repos { + if repo.Name == localDiskRepo { + repoFormat = repo.Format + } + } + return repoFormat, nil +} + +func checkIfLocalArtifactResidesInNexus(continuationToken, localDiskRepo, path string, client *client.Nexus3) (bool, error) { + g := components.GetComponentsParams{ContinuationToken: &continuationToken, Repository: localDiskRepo} + g.WithTimeout(time.Second * 60) + resp, err := client.Components.GetComponents(&g) + if err != nil { + return false, fmt.Errorf("cannot get components from repo: '%s'. Error: '%v'. Does the repo exist in Nexus3?", localDiskRepo, err) + } + continuationToken = resp.GetPayload().ContinuationToken + for _, item := range resp.GetPayload().Items { + for _, asset := range item.Assets { + if filepath.Base(asset.Path) == filepath.Base(path) { + shaType, nexusFileChecksum := artifacts.Checksum(asset) + + localFileChecksum, checksumLocalFileErrs := artifacts.ChecksumLocalFile(path, shaType) + for _, checksumLocalFileErr := range checksumLocalFileErrs { + if checksumLocalFileErr != nil { + return false, checksumLocalFileErr + } + } + + if nexusFileChecksum == localFileChecksum { + log.Debugf("file: '%v' has already been uploaded", path) + return true, nil + } + } + } + } + if continuationToken == "" { + return false, err + } + bestaat, err := checkIfLocalArtifactResidesInNexus(continuationToken, localDiskRepo, path, client) + if err != nil { + return false, err + } + if bestaat { + return true, nil + } + return false, err +} + +func UploadSingleArtifact(client *client.Nexus3, path, localDiskRepo, localDiskRepoHome, repoFormat string) error { + dir := strings.Replace(filepath.Dir(path), localDiskRepoHome+"/", "", -1) + filename := filepath.Base(path) + + var f, f2, f3 *os.File + c := components.UploadComponentParams{} + switch rf := repoFormat; rf { + case "apt": + c.Repository = localDiskRepo + f, err := os.Open(filepath.Clean(path)) + if err != nil { + return err + } + c.AptAsset = f + case "npm": + fmt.Print("*") + c.Repository = localDiskRepo + f, err := os.Open(filepath.Clean(path)) + if err != nil { + return err + } + c.NpmAsset = f + case "nuget": + fmt.Print("$") + c.Repository = localDiskRepo + f, err := os.Open(filepath.Clean(path)) + if err != nil { + return err + } + c.NugetAsset = f + case "raw": + fmt.Print("%") + c.Repository = localDiskRepo + f, err := os.Open(filepath.Clean(path)) + if err != nil { + return err + } + c.RawAsset1 = f + c.RawDirectory = &dir + c.RawAsset1Filename = &filename + default: + fmt.Print("?") + return nil + } + + if reflect.ValueOf(c).IsZero() { + log.Debug("no files to be uploaded") + return nil + } + c.WithTimeout(time.Second * 600) + if err := client.Components.UploadComponent(&c); err != nil { + statusCode, uploadStatusErr := uploadStatus(err) + if uploadStatusErr != nil { + return uploadStatusErr + } + if statusCode == 204 { + log.Infof("artifact: '%v' has been uploaded", path) + return nil + } + if statusCode == 400 { + log.Debugf("artifact: '%v' has already been uploaded, perhaps 'redeploy' is disabled?", path) + return nil + } + + return fmt.Errorf("cannot upload component: '%s', error: '%v'", path, err) + } + + files := []*os.File{f, f2, f3} + for _, file := range files { + if err := boek(file); err != nil { + return err + } + } + return nil +} + +func (n *Nexus3) ReadLocalDirAndUploadArtifacts(localDiskRepoHome, localDiskRepo, repoFormat string) error { + var errs []error + var wg sync.WaitGroup + client := n.Nexus3.Client() + if err := filepath.WalkDir(localDiskRepoHome, + func(path string, info fs.DirEntry, err error) error { + if err != nil { + return err + } + + if !info.IsDir() && !(filepath.Ext(path) == ".sha512") && !(filepath.Ext(path) == ".sha256") && !(filepath.Ext(path) == ".sha1") && !(filepath.Ext(path) == ".md5") { + wg.Add(1) + go func() { + defer wg.Done() + exists, err := checkIfLocalArtifactResidesInNexus("", localDiskRepo, path, client) + if err != nil { + errs = append(errs, err) + return + } + if exists { + return + } + if err := UploadSingleArtifact(client, path, localDiskRepo, localDiskRepoHome, repoFormat); err != nil { + errs = append(errs, err) + return + } + }() + } + return nil + }); err != nil { + return err + } + wg.Wait() + for _, err := range errs { + if err != nil { + return err + } + } + return nil +} + +func boek(f *os.File) error { + fi, err := f.Stat() + if err != nil { + return err + } + if fi.Size() != 0 { + if err := f.Close(); err != nil { + return err + } + } + return err +} + +func (n *Nexus3) Upload() error { + localDiskRepos, repoOnDiskErrs := n.reposOnDisk() + for _, repoOnDiskErr := range repoOnDiskErrs { + if repoOnDiskErr != nil { + return repoOnDiskErr + } + } + + var errs []error + var wg sync.WaitGroup + for _, localDiskRepo := range localDiskRepos { + wg.Add(1) + go func(localDiskRepo string) { + defer wg.Done() + log.Infof("Uploading files to Nexus: '%s' repository: '%s'...", n.FQDN, localDiskRepo) + repoFormat, err := n.repoFormatLocalDiskRepo(localDiskRepo) + if err != nil { + errs = append(errs, err) + return + } + localDiskRepoHome := filepath.Join(n.DownloadDirName, localDiskRepo) + if err := n.ReadLocalDirAndUploadArtifacts(localDiskRepoHome, localDiskRepo, repoFormat); err != nil { + errs = append(errs, err) + return + } + }(localDiskRepo) + } + + wg.Wait() + for _, err := range errs { + if err != nil { + return err + } + } + + return nil +} diff --git a/internal/config/repository/repository.go b/internal/config/repository/repository.go index 0aca1468..f5b7991d 100644 --- a/internal/config/repository/repository.go +++ b/internal/config/repository/repository.go @@ -7,12 +7,12 @@ import ( "github.com/030/n3dr/internal/goswagger/client/repository_management" "github.com/030/n3dr/internal/goswagger/models" - "github.com/030/n3dr/internal/pkg/http" + "github.com/030/n3dr/internal/pkg/connection" log "github.com/sirupsen/logrus" ) type Repository struct { - http.Nexus3 + connection.Nexus3 ProxyRemoteURL string } diff --git a/internal/config/security/anonymous.go b/internal/config/security/anonymous.go index b7054cdb..6f19b76f 100644 --- a/internal/config/security/anonymous.go +++ b/internal/config/security/anonymous.go @@ -6,12 +6,12 @@ import ( "github.com/030/n3dr/internal/goswagger/client/security_management_anonymous_access" "github.com/030/n3dr/internal/goswagger/models" - "github.com/030/n3dr/internal/pkg/http" + "github.com/030/n3dr/internal/pkg/connection" log "github.com/sirupsen/logrus" ) type Security struct { - http.Nexus3 + connection.Nexus3 } func (s *Security) Anonymous(enabled bool) error { diff --git a/internal/config/user/user.go b/internal/config/user/user.go index 07cbbba0..69b24f48 100644 --- a/internal/config/user/user.go +++ b/internal/config/user/user.go @@ -8,17 +8,17 @@ import ( "github.com/030/n3dr/internal/goswagger/client/security_management_roles" "github.com/030/n3dr/internal/goswagger/client/security_management_users" "github.com/030/n3dr/internal/goswagger/models" - "github.com/030/n3dr/internal/pkg/http" + "github.com/030/n3dr/internal/pkg/connection" log "github.com/sirupsen/logrus" ) type Role struct { - http.Nexus3 + connection.Nexus3 models.RoleXORequest } type User struct { - http.Nexus3 + connection.Nexus3 models.APICreateUser } diff --git a/internal/pkg/artifactsv2/artifacts/file.go b/internal/pkg/artifactsv2/artifacts/file.go new file mode 100644 index 00000000..45521269 --- /dev/null +++ b/internal/pkg/artifactsv2/artifacts/file.go @@ -0,0 +1,74 @@ +package artifacts + +import ( + "crypto/sha1" // #nosec + "crypto/sha256" + "crypto/sha512" + "fmt" + "io" + "os" + "path/filepath" + + "github.com/030/n3dr/internal/goswagger/models" + log "github.com/sirupsen/logrus" +) + +func ChecksumLocalFile(file, shaType string) (checksum string, errs []error) { + f, err := os.Open(filepath.Clean(file)) + if err != nil { + log.Debugf("file: '%v' not found on local disk", f) + return "", nil + } + if err == nil { + defer func() { + if err := f.Close(); err != nil { + errs = append(errs, err) + } + }() + h := sha512.New() + if shaType == "sha256" { + h = sha256.New() + } + if shaType == "sha1" { + /* #nosec */ + h = sha1.New() + } + if _, err := io.Copy(h, f); err != nil { + return "", errs + } + checksum = fmt.Sprintf("%x", h.Sum(nil)) + } + return checksum, errs +} + +func PrintType(assetFormat string) { + switch af := assetFormat; af { + case "apt": + fmt.Print("^") + case "maven2": + fmt.Print("+") + case "npm": + fmt.Print("*") + case "nuget": + fmt.Print("$") + case "raw": + fmt.Print("%") + default: + fmt.Print("?") + log.Debugf("Unknown type: '%s'", af) + } +} + +func Checksum(asset *models.AssetXO) (string, string) { + shaType := "sha512" + checksum := fmt.Sprintf("%s", asset.Checksum[shaType]) + if len(checksum) != 128 { + shaType = "sha256" + checksum = fmt.Sprintf("%s", asset.Checksum[shaType]) + } + if len(checksum) != 64 { + shaType = "sha1" + checksum = fmt.Sprintf("%s", asset.Checksum[shaType]) + } + return shaType, checksum +} diff --git a/internal/pkg/artifactsv2/artifacts/repository.go b/internal/pkg/artifactsv2/artifacts/repository.go new file mode 100644 index 00000000..b6fc4ecb --- /dev/null +++ b/internal/pkg/artifactsv2/artifacts/repository.go @@ -0,0 +1,25 @@ +package artifacts + +import ( + "fmt" + "time" + + "github.com/030/n3dr/internal/goswagger/client/repository_management" + "github.com/030/n3dr/internal/goswagger/models" + "github.com/030/n3dr/internal/pkg/connection" +) + +type Nexus3 struct { + *connection.Nexus3 +} + +func (n *Nexus3) Repos() ([]*models.AbstractAPIRepository, error) { + client := n.Nexus3.Client() + r := repository_management.GetRepositoriesParams{} + r.WithTimeout(time.Second * 30) + resp, err := client.RepositoryManagement.GetRepositories(&r) + if err != nil { + return nil, fmt.Errorf("cannot get repository names: '%v'", err) + } + return resp.Payload, nil +} diff --git a/internal/pkg/http/connection.go b/internal/pkg/connection/connection.go similarity index 56% rename from internal/pkg/http/connection.go rename to internal/pkg/connection/connection.go index 18bd6f74..cb82a6eb 100644 --- a/internal/pkg/http/connection.go +++ b/internal/pkg/connection/connection.go @@ -1,4 +1,4 @@ -package http +package connection import ( apiclient "github.com/030/n3dr/internal/goswagger/client" @@ -7,11 +7,16 @@ import ( ) type Nexus3 struct { - FQDN, Pass, User string + DownloadDirName, FQDN, Pass, User string + HTTPS bool } func (n *Nexus3) Client() *apiclient.Nexus3 { - r := httptransport.New(n.FQDN, apiclient.DefaultBasePath, apiclient.DefaultSchemes) + schemes := apiclient.DefaultSchemes + if n.HTTPS { + schemes = []string{"http", "https"} + } + r := httptransport.New(n.FQDN, apiclient.DefaultBasePath, schemes) r.DefaultAuthentication = httptransport.BasicAuth(n.User, n.Pass) return apiclient.New(r, strfmt.Default) } diff --git a/test/integration-tests.sh b/test/integration-tests.sh index 2804568f..4c7091ae 100755 --- a/test/integration-tests.sh +++ b/test/integration-tests.sh @@ -16,7 +16,7 @@ fi if [ -z "${NEXUS_VERSION}" ]; then echo "NEXUS_VERSION empty, setting it to the default value" - NEXUS_VERSION=3.36.0 + NEXUS_VERSION=3.37.0 fi if [ -z "${NEXUS_API_VERSION}" ]; then @@ -30,7 +30,17 @@ if [ -z "${N3DR_DELIVERABLE}" ]; then fi readonly DOWNLOAD_LOCATION=/tmp/n3dr -readonly NEXUS_URL=http://localhost:9999 +readonly DOWNLOAD_LOCATION_PASS=${DOWNLOAD_LOCATION}-pass +readonly DOWNLOAD_LOCATION_SYNC=${DOWNLOAD_LOCATION}-sync +readonly PORT_NEXUS_A=9999 +readonly PORT_NEXUS_B=9998 +readonly PORT_NEXUS_C=9997 +readonly URL_NEXUS_A=http://localhost:${PORT_NEXUS_A} +readonly URL_NEXUS_B=http://localhost:${PORT_NEXUS_B} +readonly URL_NEXUS_C=http://localhost:${PORT_NEXUS_C} +readonly URL_NEXUS_A_V2="${URL_NEXUS_A/http:\/\//}" +readonly URL_NEXUS_B_V2="${URL_NEXUS_B/http:\/\//}" +readonly URL_NEXUS_C_V2="${URL_NEXUS_C/http:\/\//}" validate() { if [ -z "${N3DR_DELIVERABLE}" ]; then @@ -52,13 +62,27 @@ build() { cd cmd/n3dr } +startNexus() { + # shellcheck disable=SC1091 + # as nexus-docker.sh is retrieved remotely + source ./start.sh "${NEXUS_VERSION}" "${NEXUS_API_VERSION}" "nexus-${1}" "${2}" "${DOWNLOAD_LOCATION_PASS}" &>/dev/null +} + nexus() { - curl -L https://gist.githubusercontent.com/030/666c99d8fc86e9f1cc0ad216e0190574/raw/47056b970df25334edf8f9a86bd6b2cb02a2b816/nexus-docker.sh -o start.sh + curl -sL https://gist.githubusercontent.com/030/666c99d8fc86e9f1cc0ad216e0190574/raw/d6e02230d86bce0ad555fdb0330ed4162c735d09/nexus-docker.sh -o start.sh chmod +x start.sh - # shellcheck disable=SC1091 - # as nexus-docker.sh is retrieved remotely - source ./start.sh "${NEXUS_VERSION}" "${NEXUS_API_VERSION}" + startNexus a ${PORT_NEXUS_A} & + startNexus b ${PORT_NEXUS_B} & + startNexus c ${PORT_NEXUS_C} & + wait + + PASSWORD_NEXUS_A=$(cat ${DOWNLOAD_LOCATION_PASS}/nexus-a.txt) + readonly PASSWORD_NEXUS_A + PASSWORD_NEXUS_B=$(cat ${DOWNLOAD_LOCATION_PASS}/nexus-b.txt) + readonly PASSWORD_NEXUS_B + PASSWORD_NEXUS_C=$(cat ${DOWNLOAD_LOCATION_PASS}/nexus-c.txt) + readonly PASSWORD_NEXUS_C } artifact() { @@ -75,39 +99,45 @@ files() { upload() { echo "#134 archetype-catalog download issue" echo "URL:" - echo "${NEXUS_URL}/repository/maven-releases/archetype-catalog.xml" + echo "${URL_NEXUS_A}/repository/maven-releases/archetype-catalog.xml" echo "does not seem to contain a Maven artifact" - curl -f ${NEXUS_URL}/repository/maven-releases/archetype-catalog.xml + curl -f ${URL_NEXUS_A}/repository/maven-releases/archetype-catalog.xml echo "Testing upload..." ./"${N3DR_DELIVERABLE}" upload -u admin \ -p "${PASSWORD}" \ -r maven-releases \ - -n http://127.0.0.1:9999 \ + -n ${URL_NEXUS_A} \ -v "${NEXUS_API_VERSION}" echo } +createHostedAPT() { + echo "Creating apt repo..." + curl "${2}/service/rest/beta/repositories/apt/hosted" \ + -s \ + -f \ + -u "admin:${1}" \ + -H "accept: application/json" \ + -H "Content-Type: application/json" \ + -d "{\"name\":\"REPO_NAME_HOSTED_APT\",\"online\":true,\"proxy\":{\"remoteUrl\":\"http://nl.archive.ubuntu.com/ubuntu/\"},\"storage\":{\"blobStoreName\":\"default\",\"strictContentTypeValidation\":true,\"writePolicy\":\"ALLOW_ONCE\"},\"apt\": {\"distribution\": \"bionic\"},\"aptSigning\": {\"keypair\": \"${N3DR_APT_GPG_SECRET}\",\"passphrase\": \"abc\"}}" +} + uploadDeb() { if [ "${NEXUS_API_VERSION}" != "beta" ]; then - echo "Creating apt repo..." - curl -u "admin:${PASSWORD}" \ - -X POST "${NEXUS_URL}/service/rest/beta/repositories/apt/hosted" \ - -H "accept: application/json" \ - -H "Content-Type: application/json" \ - --data "{\"name\":\"REPO_NAME_HOSTED_APT\",\"online\":true,\"proxy\":{\"remoteUrl\":\"http://nl.archive.ubuntu.com/ubuntu/\"},\"storage\":{\"blobStoreName\":\"default\",\"strictContentTypeValidation\":true,\"writePolicy\":\"ALLOW_ONCE\"},\"apt\": {\"distribution\": \"bionic\"},\"aptSigning\": {\"keypair\": \"${N3DR_APT_GPG_SECRET}\",\"passphrase\": \"abc\"}}" + createHostedAPT "${PASSWORD_NEXUS_A}" "${URL_NEXUS_A}" mkdir REPO_NAME_HOSTED_APT cd REPO_NAME_HOSTED_APT - curl -L https://github.com/030/a2deb/releases/download/1.0.0/a2deb_1.0.0-0.deb -o a2deb.deb - curl -L https://github.com/030/n3dr/releases/download/5.0.1/n3dr_5.0.1-0.deb -o n3dr.deb - curl -L https://github.com/030/informado/releases/download/1.4.0/informado_1.4.0-0.deb -o informado.deb + curl -sL https://github.com/030/a2deb/releases/download/1.0.0/a2deb_1.0.0-0.deb -o a2deb.deb + curl -sL https://github.com/030/n3dr/releases/download/5.0.1/n3dr_5.0.1-0.deb -o n3dr.deb + curl -sL https://github.com/030/informado/releases/download/1.4.0/informado_1.4.0-0.deb -o informado.deb cd .. echo "Testing deb upload..." ./"${N3DR_DELIVERABLE}" upload -u=admin -p="${PASSWORD}" \ -r=REPO_NAME_HOSTED_APT \ - -n=${NEXUS_URL} -v="${NEXUS_API_VERSION}" \ + -n=${URL_NEXUS_A} -v="${NEXUS_API_VERSION}" \ -t=apt echo else @@ -115,16 +145,20 @@ uploadDeb() { fi } +createHostedNPM() { + echo "Creating npm repo..." + curl "${2}/service/rest/v1/repositories/npm/hosted" \ + -s \ + -f \ + -u "admin:${1}" \ + -H "accept: application/json" \ + -H "Content-Type: application/json" \ + -d "{\"name\":\"REPO_NAME_HOSTED_NPM\",\"online\":true,\"storage\":{\"blobStoreName\":\"default\",\"strictContentTypeValidation\":true,\"writePolicy\":\"ALLOW_ONCE\"}}" +} + uploadNPM() { if [ "${NEXUS_API_VERSION}" != "beta" ]; then - echo "Creating npm repo..." - curl -f \ - -v \ - -u "admin:${PASSWORD}" \ - -X POST "${NEXUS_URL}/service/rest/v1/repositories/npm/hosted" \ - -H "accept: application/json" \ - -H "Content-Type: application/json" \ - --data "{\"name\":\"REPO_NAME_HOSTED_NPM\",\"online\":true,\"storage\":{\"blobStoreName\":\"default\",\"strictContentTypeValidation\":true,\"writePolicy\":\"ALLOW_ONCE\"}}" + createHostedNPM "${PASSWORD_NEXUS_A}" "${URL_NEXUS_A}" mkdir REPO_NAME_HOSTED_NPM cd REPO_NAME_HOSTED_NPM @@ -133,11 +167,11 @@ uploadNPM() { echo "Testing NPM upload..." ./"${N3DR_DELIVERABLE}" upload -u=admin -p="${PASSWORD}" -r=REPO_NAME_HOSTED_NPM \ - -n=${NEXUS_URL} -v="${NEXUS_API_VERSION}" \ + -n=${URL_NEXUS_A} -v="${NEXUS_API_VERSION}" \ -t=npm echo else - echo "Nuget upload not supported in beta API" + echo "NPM upload not supported in beta API" fi } @@ -145,12 +179,12 @@ uploadNuget() { if [ "${NEXUS_API_VERSION}" != "beta" ]; then mkdir nuget-hosted cd nuget-hosted - curl -L https://chocolatey.org/api/v2/package/n3dr/5.2.6 -o n3dr.5.2.6.nupkg + curl -sL https://chocolatey.org/api/v2/package/n3dr/5.2.6 -o n3dr.5.2.6.nupkg cd .. echo "Testing nuget upload..." ./"${N3DR_DELIVERABLE}" upload -u=admin -p="${PASSWORD}" -r=nuget-hosted \ - -n=${NEXUS_URL} -v="${NEXUS_API_VERSION}" \ + -n=${URL_NEXUS_A} -v="${NEXUS_API_VERSION}" \ -t=nuget echo else @@ -171,19 +205,19 @@ backupHelper() { anonymous() { echo "Testing backup by anonymous user..." - ./"${N3DR_DELIVERABLE}" backup -n ${NEXUS_URL} -r maven-releases -v "${NEXUS_API_VERSION}" -z --anonymous + ./"${N3DR_DELIVERABLE}" backup -n ${URL_NEXUS_A} -r maven-releases -v "${NEXUS_API_VERSION}" -z --anonymous backupHelper } backup() { echo "Testing backup..." - ./"${N3DR_DELIVERABLE}" backup -n ${NEXUS_URL} -u admin -p "${PASSWORD}" -r maven-releases -v "${NEXUS_API_VERSION}" -z + ./"${N3DR_DELIVERABLE}" backup -n ${URL_NEXUS_A} -u admin -p "${PASSWORD}" -r maven-releases -v "${NEXUS_API_VERSION}" -z backupHelper } regex() { echo "Testing backup regex..." - ./"${N3DR_DELIVERABLE}" backup -n ${NEXUS_URL} -u admin -p "${PASSWORD}" -r maven-releases -v "${NEXUS_API_VERSION}" -x 'some/group42' -z + ./"${N3DR_DELIVERABLE}" backup -n ${URL_NEXUS_A} -u admin -p "${PASSWORD}" -r maven-releases -v "${NEXUS_API_VERSION}" -x 'some/group42' -z if [ "${NEXUS_VERSION}" == "3.9.0" ]; then count_downloads 3 test_zip 4 @@ -193,7 +227,7 @@ regex() { fi cleanup_downloads echo -e "\nTesting repositories regex..." - ./"${N3DR_DELIVERABLE}" repositories -n ${NEXUS_URL} -u admin -p "${PASSWORD}" -v "${NEXUS_API_VERSION}" -b -x 'some/group42' -z + ./"${N3DR_DELIVERABLE}" repositories -n ${URL_NEXUS_A} -u admin -p "${PASSWORD}" -v "${NEXUS_API_VERSION}" -b -x 'some/group42' -z if [ "${NEXUS_VERSION}" == "3.9.0" ]; then count_downloads 3 test_zip 4 @@ -205,7 +239,7 @@ regex() { } repositories() { - local cmd="./$N3DR_DELIVERABLE repositories -n ${NEXUS_URL} -u admin -p $PASSWORD -v ${NEXUS_API_VERSION}" + local cmd="./$N3DR_DELIVERABLE repositories -n ${URL_NEXUS_A} -u admin -p ${PASSWORD} -v ${NEXUS_API_VERSION}" echo "Testing repositories..." $cmd -a | grep maven-releases @@ -220,26 +254,29 @@ repositories() { [ "${actual_number}" == "${expected_number}" ] echo "> Testing zip functionality..." - $cmd -b -z + testZipSizeDir=/tmp/n3dr/testZipSize/ + $cmd -b -z \ + --directory-prefix ${testZipSizeDir} \ + --directory-prefix-zip ${testZipSizeDir} if [ "${NEXUS_VERSION}" == "3.9.0" ]; then - count_downloads 300 - test_zip 152 + count_downloads 301 + test_zip 132 ${testZipSizeDir} else - count_downloads 401 - test_zip 236 + count_downloads 402 + test_zip 208 ${testZipSizeDir} fi cleanup_downloads } zipName() { echo "Testing zipName..." - ./"${N3DR_DELIVERABLE}" backup -n=${NEXUS_URL} -u=admin -p="${PASSWORD}" -r=maven-releases -v="${NEXUS_API_VERSION}" -z -i=helloZipFile.zip - ./"${N3DR_DELIVERABLE}" repositories -n ${NEXUS_URL} -u admin -p "${PASSWORD}" -v "${NEXUS_API_VERSION}" -b -z -i=helloZipRepositoriesFile.zip + ./"${N3DR_DELIVERABLE}" backup -n=${URL_NEXUS_A} -u=admin -p="${PASSWORD}" -r=maven-releases -v="${NEXUS_API_VERSION}" -z -i=helloZipFile.zip + ./"${N3DR_DELIVERABLE}" repositories -n ${URL_NEXUS_A} -u admin -p "${PASSWORD}" -v "${NEXUS_API_VERSION}" -b -z -i=helloZipRepositoriesFile.zip find . -name "helloZip*" -type f | wc -l | grep 2 } clean() { - cleanup + for r in a b; do NEXUS_DOCKER_NAME=nexus-${r} cleanup; done cleanup_downloads } @@ -253,7 +290,7 @@ count_downloads() { test_zip() { local size - size=$(du n3dr-backup-*zip) + size=$(du "${2}"n3dr-backup-*zip) echo "Actual ZIP size: ${size}" echo "Expected ZIP size: ${1}" echo "${size}" | grep "^${1}" @@ -265,6 +302,7 @@ cleanup_downloads() { rm -rf REPO_NAME_HOSTED_NPM rm -rf maven-releases rm -rf "${DOWNLOAD_LOCATION}" + rm -rf "${DOWNLOAD_LOCATION_SYNC}" rm -f n3dr-backup-*zip rm -f helloZip*zip } @@ -279,13 +317,34 @@ cac() { echo "Configuration as code" } +sync() { + if [ "${NEXUS_API_VERSION}" != "beta" ]; then + echo "Testing the sync between NexusA and followers NexusB and C..." + createHostedAPT "${PASSWORD_NEXUS_B}" "${URL_NEXUS_B}" + createHostedAPT "${PASSWORD_NEXUS_C}" "${URL_NEXUS_C}" + createHostedNPM "${PASSWORD_NEXUS_B}" "${URL_NEXUS_B}" + createHostedNPM "${PASSWORD_NEXUS_C}" "${URL_NEXUS_C}" + + ./"${N3DR_DELIVERABLE}" sync \ + --otherNexus3Passwords="${PASSWORD_NEXUS_B}","${PASSWORD_NEXUS_C}" \ + --otherNexus3Users=admin,admin \ + --otherNexus3URLs="${URL_NEXUS_B_V2}","${URL_NEXUS_C_V2}" \ + --directory-prefix "${DOWNLOAD_LOCATION_SYNC}" \ + -p "${PASSWORD_NEXUS_A}" \ + -n "${URL_NEXUS_A_V2}" \ + -u admin + else + echo "RepositoriesV2 sync not supported in beta API" + fi +} + main() { validate build nexus - readiness - password files + + export PASSWORD=${PASSWORD_NEXUS_A} upload anonymous backup @@ -296,8 +355,13 @@ main() { regex zipName version + sync bats --tap ../../test/tests.bats - echo "In order to debug, comment out the 'trap clean EXIT', run this script again and login to http://localhost:9999 and login as admin and ${PASSWORD}" + echo " +In order to debug, comment out the 'trap clean EXIT', run this script again and +login to ${URL_NEXUS_A}, ${URL_NEXUS_B} or ${URL_NEXUS_C} and login as admin +and respectively ${PASSWORD_NEXUS_A}, ${PASSWORD_NEXUS_B} or +${PASSWORD_NEXUS_C}" } # trap clean EXIT