Merge pull request #226 from 030/GH-215-performance

[GH-215] Improve performance.
030 · Nov 30, 2021 · d778f66 · d778f66
2 parents cd240f7 + 5c72852
commit d778f66
Show file tree

Hide file tree

Showing 31 changed files with 998 additions and 106 deletions.
diff --git a/.github/workflows/dip.yml b/.github/workflows/dip.yml
@@ -1,5 +1,5 @@
 ---
-name: "DIP"
+name: DIP
 on: [push]
 jobs:
   dive:

diff --git a/.github/workflows/dive.yml b/.github/workflows/dive.yml
@@ -1,5 +1,5 @@
 ---
-name: "Dive CI"
+name: Dive CI
 on: [push]
 jobs:
   dive:

diff --git a/.github/workflows/dockle.yml b/.github/workflows/dockle.yml
@@ -1,5 +1,5 @@
 ---
-name: "Dockle"
+name: Dockle
 on: [push]
 jobs:
   dive:

diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml
@@ -16,7 +16,7 @@ jobs:
             nexus-api-version: beta
             shasum: shasum -a 512
           - os: ubuntu-20.04
-            nexus-version: 3.34.1
+            nexus-version: 3.37.0
             nexus-api-version: v1
             shasum: sha512sum
           - os: ubuntu-20.04
@@ -36,12 +36,6 @@ jobs:
       - name: Unit tests
         run: go test ./... -cover
         if: ${{ startsWith(matrix.os, 'ubuntu') }}
-      - name: Install bats
-        run: |
-          set -x
-          sudo apt-get update
-          sudo apt-get install bats -y
-        if: ${{ startsWith(matrix.os, 'ubuntu') }}
       - name: Set N3DR deliverable environment variable
         run: echo "n3dr-deliverable=n3dr-${{ matrix.os }}" >> $GITHUB_ENV
       - name: Use the value
@@ -61,11 +55,3 @@ jobs:
           docker run --rm -v $(pwd):/app -w /app \
             golangci/golangci-lint:v1.39.0-alpine golangci-lint run -v
         if: ${{ startsWith(matrix.os, 'ubuntu') }}
-      - name: Integration test
-        run: ./test/integration-tests.sh
-        env:
-          N3DR_DELIVERABLE: ${{ env.n3dr-deliverable }}
-          NEXUS_VERSION: ${{ matrix.nexus-version }}
-          NEXUS_API_VERSION: ${{ matrix.nexus-api-version }}
-          N3DR_APT_GPG_SECRET: ${{ secrets.N3DR_APT_GPG_SECRET }}
-        if: ${{ startsWith(matrix.os, 'ubuntu') }}
diff --git a/.github/workflows/hadolint.yml b/.github/workflows/hadolint.yml
@@ -1,4 +1,4 @@
-name: "Hadolint"
+name: Hadolint
 on: [push]
 jobs:
   dive:

diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml
@@ -0,0 +1,40 @@
+---
+name: Integration
+on:
+  push:
+    branches: [master]
+  pull_request:
+    branches: [master]
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        include:
+          - os: ubuntu-20.04
+            nexus-version: 3.37.0
+            nexus-api-version: v1
+            shasum: sha512sum
+          - os: ubuntu-20.04
+            nexus-version: 3.9.0
+            nexus-api-version: beta
+            shasum: sha512sum
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.17.0
+      - name: Install bats
+        run: |
+          set -x
+          sudo apt-get update
+          sudo apt-get install bats -y
+      - name: Integration test
+        run: |
+          ./test/integration-tests.sh
+        env:
+          N3DR_DELIVERABLE: ${{ env.n3dr-deliverable }}
+          NEXUS_VERSION: ${{ matrix.nexus-version }}
+          NEXUS_API_VERSION: ${{ matrix.nexus-api-version }}
+          N3DR_APT_GPG_SECRET: ${{ secrets.N3DR_APT_GPG_SECRET }}
diff --git a/.github/workflows/trivy.yml b/.github/workflows/trivy.yml
@@ -1,5 +1,5 @@
 ---
-name: trivy
+name: Trivy
 on: [push]
 jobs:
   build:

diff --git a/.gitignore b/.gitignore
@@ -23,3 +23,4 @@ virtualbox*
 *.exe
 swagger
 swagger.json
+*.log
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -7,6 +7,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [6.2.0] - 2021-11-30
+
+### Added
+
+- Nuget downloads.
+- Sync between multiple nexus repositories.
+
+### Changed
+
+- Moved the integration test to a separate github workflow file.
+
+### Fixed
+
+- Inconsistent file zip size while running `./test/integration.sh`.
+
 ## [6.1.0] - 2021-11-23
 
 ### Added
@@ -566,7 +581,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - Download all artifacts from a certain Nexus3 repository.
 
-[Unreleased]: https://github.com/030/n3dr/compare/6.1.0...HEAD
+[Unreleased]: https://github.com/030/n3dr/compare/6.2.0...HEAD
+[6.2.0]: https://github.com/030/n3dr/compare/6.1.0...6.2.0
 [6.1.0]: https://github.com/030/n3dr/compare/6.0.13...6.1.0
 [6.0.13]: https://github.com/030/n3dr/compare/6.0.12...6.0.13
 [6.0.12]: https://github.com/030/n3dr/compare/6.0.11...6.0.12

diff --git a/Dockerfile b/Dockerfile
@@ -1,4 +1,4 @@
-FROM golang:1.17.3-alpine3.14 as builder
+FROM golang:1.17.3-alpine3.15 as builder
 ENV USERNAME n3dr
 RUN adduser -D -g '' $USERNAME
 COPY . /go/${USERNAME}/
@@ -7,7 +7,7 @@ RUN apk add --no-cache git=~2 && \
     CGO_ENABLED=0 go build && \
     cp n3dr /n3dr
 
-FROM alpine:3.14.3
+FROM alpine:3.15.0
 COPY --from=builder /etc/passwd /etc/passwd
 COPY --from=builder /n3dr /usr/local/bin/n3dr
 COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/

diff --git a/README.md b/README.md
@@ -301,7 +301,7 @@ n3dr config \
 ### Build
 
 ```bash
-docker build -t utrecht/n3dr:6.1.0 .
+docker build -t utrecht/n3dr:6.2.0 .
 ```
 
 [![dockeri.co](https://dockeri.co/image/utrecht/n3dr)](https://hub.docker.com/r/utrecht/n3dr)
@@ -311,7 +311,7 @@ docker build -t utrecht/n3dr:6.1.0 .
 ```bash
 docker run -it \
   -v /home/${USER}/.n3dr:/root/.n3dr \
-  -v /tmp/n3dr:/tmp/n3dr utrecht/n3dr:6.1.0
+  -v /tmp/n3dr:/tmp/n3dr utrecht/n3dr:6.2.0
 ```
 
 ### Upload
@@ -320,7 +320,7 @@ docker run -it \
 docker run -it \
   --entrypoint=/bin/ash \
   -v /home/${USER}/.n3dr:/root/.n3dr \
-  -v /tmp/n3dr:/tmp/n3dr utrecht/n3dr:6.1.0
+  -v /tmp/n3dr:/tmp/n3dr utrecht/n3dr:6.2.0
 ```
 
 navigate to the repository folder, e.g. `/tmp/n3dr/download*/` and upload:
@@ -519,6 +519,70 @@ a single command.
 | npm    | x      | x      | `*`   |
 | nuget  |        | x      |       |
 
+### repositoriesV2
+
+| type      | backup | upload | label |
+|-----------|--------|--------|-------|
+| apt       | x      | x      | `^`   |
+| bower     |        |        |       |
+| cocoapods |        |        |       |
+| conan     |        |        |       |
+| docker    |        |        |       |
+| gitlfs    |        |        |       |
+| go        |        |        |       |
+| helm      |        |        |       |
+| maven2    | x      |        | `+`   |
+| npm       | x      | x      | `*`   |
+| nuget     | x      | x      | `$`   |
+| p2        |        |        |       |
+| pypi      |        |        |       |
+| r         |        |        |       |
+| raw       | x      | x      | `%`   |
+| rubygems  |        |        |       |
+| yum       |        |        |       |
+| unknown   | x      | x      | `?`   |
+
+#### backup
+
+`repositoriesV2` command in conjunction with the `--backup` subcommand ensures
+that all artifacts are downloaded from Nexus3 and stored in a folder that is
+defined with the `--directory-prefix` parameter.
+
+```bash
+n3dr repositoriesV2 \
+  --backup \
+  -n some-url \
+  -u some-user \
+  -p some-pass \
+  --directory-prefix /tmp/some-dir
+```
+
+#### upload
+
+```bash
+n3dr repositoriesV2 \
+  --upload \
+  -n some-url \
+  -u some-user \
+  -p some-pass \
+  --directory-prefix /tmp/some-dir
+```
+
+#### sync
+
+```bash
+n3dr sync \
+  --otherNexus3Passwords=some-pass-B,some-pass-C \
+  --otherNexus3Users=admin,admin \
+  --otherNexus3URLs=localhost:9998,localhost:9997 \
+  --directory-prefix /some/dir \
+  -p some-pass-A \
+  -n localhost:9999 \
+  -u admin
+```
+
+Note: use `--https=false` in order to connect to a <http://nexus-url/>.
+
 ## Stargazers over time
 
 [![Stargazers over time](https://starchart.cc/030/n3dr.svg)](https://starchart.cc/030/n3dr)

diff --git a/build/package/snap/snapcraft.yaml b/build/package/snap/snapcraft.yaml
@@ -1,7 +1,7 @@
 ---
 name: n3dr
 base: core20
-version: 6.1.0
+version: 6.2.0
 summary: Nexus3 Disaster Recovery
 description: |
   Download all artifacts at once or migrate automatically from Nexus to Nexus.

diff --git a/cmd/n3dr/backup.go b/cmd/n3dr/backup.go
@@ -43,7 +43,7 @@ reside in a certain Nexus3 repository`,
 				}
 			}
 
-			if err := n.CreateZip(dir); err != nil {
+			if err := n.CreateZip(dir, downloadDirNameZip); err != nil {
 				log.Fatal(err)
 			}
 		}

diff --git a/cmd/n3dr/config.go b/cmd/n3dr/config.go
@@ -4,7 +4,7 @@ import (
 	"fmt"
 
 	"github.com/030/n3dr/internal/config/security"
-	"github.com/030/n3dr/internal/pkg/http"
+	"github.com/030/n3dr/internal/pkg/connection"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 )
@@ -19,7 +19,7 @@ var configCmd = &cobra.Command{
 	Run: func(cmd *cobra.Command, args []string) {
 		fmt.Println("config called")
 
-		n := http.Nexus3{FQDN: n3drURL, Pass: n3drPass, User: n3drUser}
+		n := connection.Nexus3{FQDN: n3drURL, Pass: n3drPass, User: n3drUser}
 		s := security.Security{Nexus3: n}
 		if err := s.Anonymous(configUserAnonymous); err != nil {
 			log.Fatal(err)

diff --git a/cmd/n3dr/configRepository.go b/cmd/n3dr/configRepository.go
@@ -5,7 +5,7 @@ import (
 	"os"
 
 	"github.com/030/n3dr/internal/config/repository"
-	"github.com/030/n3dr/internal/pkg/http"
+	"github.com/030/n3dr/internal/pkg/connection"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 )
@@ -25,7 +25,7 @@ var configRepositoryCmd = &cobra.Command{
 	Run: func(cmd *cobra.Command, args []string) {
 		fmt.Println("configRepository called")
 
-		n := http.Nexus3{FQDN: n3drURL, Pass: n3drPass, User: n3drUser}
+		n := connection.Nexus3{FQDN: n3drURL, Pass: n3drPass, User: n3drUser}
 		r := repository.Repository{Nexus3: n}
 
 		if configRepoDelete {

diff --git a/cmd/n3dr/configUser.go b/cmd/n3dr/configUser.go
@@ -5,7 +5,7 @@ import (
 
 	"github.com/030/n3dr/internal/config/user"
 	"github.com/030/n3dr/internal/goswagger/models"
-	"github.com/030/n3dr/internal/pkg/http"
+	"github.com/030/n3dr/internal/pkg/connection"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 )
@@ -39,7 +39,7 @@ to quickly create a Cobra application.`,
 			Password:     pass,
 			UserID:       id,
 		}
-		n := http.Nexus3{FQDN: n3drURL, Pass: n3drPass, User: n3drUser}
+		n := connection.Nexus3{FQDN: n3drURL, Pass: n3drPass, User: n3drUser}
 		u := user.User{APICreateUser: acu, Nexus3: n}
 
 		if admin {

diff --git a/cmd/n3dr/repositories.go b/cmd/n3dr/repositories.go
@@ -23,7 +23,7 @@ download artifacts from all repositories`,
 			}
 			log.Fatal("One of the required flags \"names\", \"count\" or \"backup\" not set")
 		}
-		n := cli.Nexus3{URL: n3drURL, User: n3drUser, Pass: n3drPass, APIVersion: apiVersion, ZIP: zip, ZipName: zipName, DownloadDirName: downloadDirName}
+		n := cli.Nexus3{URL: n3drURL, User: n3drUser, Pass: n3drPass, APIVersion: apiVersion, ZIP: zip, ZipName: zipName, DownloadDirName: downloadDirName, DownloadDirNameZip: downloadDirNameZip}
 		if err := n.ValidateNexusURL(); err != nil {
 			log.Fatal(err)
 		}
-Original file line number
+Diff line change
@@ Expand Up / @@ -23,3 +23,4 @@ virtualbox* @@
     *.exe
     swagger
     swagger.json
+    *.log