Merge pull request #255 from 030/172-s3

[#172] Upload backup archive to AWS S3.

.github/workflows/dip.yml

@@ -18,9 +18,11 @@ jobs:
       - name: Check Nexus3
         run: |
           NEXUS_VERSION=$(./dip image --name=sonatype/nexus3 --regex=^3\.[0-9]+\.[0-9]+$)
+          echo "Check whether the latest Nexus3 version: '${NEXUS_VERSION}' is used..."
           grep "${NEXUS_VERSION}" ./test/integration-tests.sh
           grep "${NEXUS_VERSION}" ./.github/workflows/integration.yml
       - name: Check golangci-lint
         run: |
           GOLANGCI_LINT_VERSION=$(./dip image --name=golangci/golangci-lint --regex=^v1\.[0-9]+\.[0-9]+-alpine$)
+          echo "Check whether the latest GolangCI version: '${GOLANGCI_LINT_VERSION}' is used..."
           grep "golangci-lint:${GOLANGCI_LINT_VERSION}" ./.github/workflows/go.yml

.github/workflows/go.yml

@@ -40,6 +40,6 @@ jobs:
           docker run --rm -v ${PWD}:/data markdownlint/markdownlint:0.11.0 \
             README.md -s /data/.markdownlint.rb
           docker run --rm -v $(pwd):/data cytopia/yamllint:1.26-0.8 .
-          docker run --rm -v $(pwd):/app -w /app \
-            golangci/golangci-lint:v1.45.2-alpine golangci-lint run -v
+          docker run --rm -v $(pwd):/app -w /app -e GOFLAGS=-buildvcs=false \
+            golangci/golangci-lint:v1.46.2-alpine golangci-lint run -v
         if: ${{ startsWith(matrix.os, 'ubuntu') }}

.github/workflows/integration.yml

@@ -8,7 +8,7 @@ jobs:
       matrix:
         include:
           - os: ubuntu-20.04
-            nexus-version: 3.38.1
+            nexus-version: 3.39.0
             nexus-api-version: v1
             shasum: sha512sum
           - os: ubuntu-20.04

.github/workflows/release.yml

@@ -3,7 +3,7 @@ name: Release
 on:
   push:
     tags:
-      - "*"
+      - '*'
 jobs:
   release:
     name: Create Release
@@ -29,7 +29,7 @@ jobs:
           ${{ startsWith(matrix.os, 'mac') || startsWith(matrix.os, 'ubuntu') }}
       - name: Set N3DR deliverable environment variable on Windows
         run: echo "n3dr-deliverable=n3dr-${{ matrix.os }}" >> $env:GITHUB_ENV
-        if: ${{ startsWith(matrix.os, 'windows') }}
+        if: ${{ startsWith(matrix.os, 'windows') }}go
       - name: Use the value
         run: |
           echo "${{ env.n3dr-deliverable }}"
@@ -56,7 +56,7 @@ jobs:
           echo "GITHUB_TAG: '${{ github.ref }}'"
           echo "N3DR_DELIVERABLE: '${{ env.n3dr-deliverable }}'"
           cd cmd/n3dr
-          go build -ldflags "-X main.Version=${{ github.ref }}" -o "${{ env.n3dr-deliverable }}"
+          go build -buildvcs=false -ldflags "-X main.Version=${{ github.ref }}" -o "${{ env.n3dr-deliverable }}"
           sha512sum "${{ env.n3dr-deliverable }}" > "${{ env.n3dr-deliverable }}.sha512.txt"
           chmod +x "${{ env.n3dr-deliverable }}"
           ls n3dr-windows-2019

Dockerfile

@@ -1,17 +1,17 @@
-FROM golang:1.18.1-alpine3.15 as builder
+FROM golang:1.18.2-alpine3.16 as builder
 ARG VERSION
 ENV USERNAME n3dr
 RUN adduser -D -g '' $USERNAME
 COPY . /go/${USERNAME}/
 WORKDIR /go/${USERNAME}/cmd/${USERNAME}
 RUN apk add --no-cache git=~2 && \
-    CGO_ENABLED=0 go build -ldflags "-X main.Version=${VERSION}" && \
-    cp n3dr /n3dr
+  CGO_ENABLED=0 go build -ldflags "-X main.Version=${VERSION}" -buildvcs=false && \
+  cp n3dr /n3dr
 
-FROM alpine:3.15.4
+FROM alpine:3.16.0
 RUN apk update && \
-    apk upgrade && \
-    rm -rf /var/cache/apk/*
+  apk upgrade && \
+  rm -rf /var/cache/apk/*
 COPY --from=builder /etc/passwd /etc/passwd
 COPY --from=builder /n3dr /usr/local/bin/n3dr
 COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/

README.md

@@ -315,7 +315,7 @@ n3dr config \
 ### Build
 
 ```bash
-docker build -t utrecht/n3dr:6.6.2 .
+docker build -t utrecht/n3dr:6.7.0 .
 ```
 
 [![dockeri.co](https://dockeri.co/image/utrecht/n3dr)](https://hub.docker.com/r/utrecht/n3dr)
@@ -325,7 +325,7 @@ docker build -t utrecht/n3dr:6.6.2 .
 ```bash
 docker run -it \
   -v /home/${USER}/.n3dr:/root/.n3dr \
-  -v /tmp/n3dr:/tmp/n3dr utrecht/n3dr:6.6.2
+  -v /tmp/n3dr:/tmp/n3dr utrecht/n3dr:6.7.0
 ```
 
 ### Upload
@@ -334,7 +334,7 @@ docker run -it \
 docker run -it \
   --entrypoint=/bin/ash \
   -v /home/${USER}/.n3dr:/root/.n3dr \
-  -v /tmp/n3dr:/tmp/n3dr utrecht/n3dr:6.6.2
+  -v /tmp/n3dr:/tmp/n3dr utrecht/n3dr:6.7.0
 ```
 
 navigate to the repository folder, e.g. `/tmp/n3dr/download*/` and upload:

build/package/snap/snapcraft.yaml

@@ -1,7 +1,7 @@
 ---
 name: n3dr
 base: core20
-version: 6.6.2
+version: 6.7.0
 summary: Nexus3 Disaster Recovery
 description: |
   Download all artifacts at once or migrate automatically from Nexus to Nexus.

cmd/n3dr/repositoriesV2.go

@@ -25,7 +25,7 @@ download artifacts from all repositories`,
 			}
 			log.Fatal("One of the required flags \"names\", \"count\" or \"backup\" not set")
 		}
-		n := connection.Nexus3{BasePathPrefix: basePathPrefix, FQDN: n3drURL, Pass: n3drPass, User: n3drUser, DownloadDirName: downloadDirName, HTTPS: https, DockerHost: dockerHost, DockerPort: dockerPort, DockerPortSecure: dockerPortSecure}
+		n := connection.Nexus3{AwsBucket: awsBucket, AwsId: awsId, AwsRegion: awsRegion, AwsSecret: awsSecret, BasePathPrefix: basePathPrefix, FQDN: n3drURL, Pass: n3drPass, User: n3drUser, DownloadDirName: downloadDirName, HTTPS: https, DockerHost: dockerHost, DockerPort: dockerPort, DockerPortSecure: dockerPortSecure}
 		a := artifactsv2.Nexus3{Nexus3: &n}
 		if namesV2 {
 			if err := a.RepositoryNamesV2(); err != nil {

cmd/n3dr/root.go

@@ -23,9 +23,9 @@ import (
 var logo string
 
 var (
-	apiVersion, basePathPrefix, cfgFile, n3drRepo, n3drURL, n3drPass, n3drUser, Version, zipName, downloadDirName, downloadDirNameZip, dockerHost string
-	anonymous, debug, dockerPortSecure, https, insecureSkipVerify, showLogo, skipErrors, zip                                                      bool
-	dockerPort                                                                                                                                    int32
+	apiVersion, awsBucket, awsId, awsRegion, awsSecret, basePathPrefix, cfgFile, n3drRepo, n3drURL, n3drPass, n3drUser, Version, zipName, downloadDirName, downloadDirNameZip, dockerHost string
+	anonymous, awsS3, debug, dockerPortSecure, https, insecureSkipVerify, showLogo, skipErrors, zip                                                                                       bool
+	dockerPort                                                                                                                                                                            int32
 )
 
 var rootCmd = &cobra.Command{
@@ -46,6 +46,7 @@ func execute() {
 func init() {
 	cobra.OnInitialize(initConfig)
 
+	rootCmd.PersistentFlags().BoolVarP(&awsS3, "awsS3", "", false, "whether the backup zip should be uploaded to AWS S3")
 	rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/"+cli.DefaultCfgFileWithExt+")")
 	rootCmd.PersistentFlags().BoolVarP(&debug, "debug", "d", false, "enable debug logging")
 	rootCmd.PersistentFlags().BoolVarP(&zip, "zip", "z", false, "add downloaded artifacts to a ZIP archive")
@@ -114,15 +115,15 @@ func ascii() error {
 }
 
 func initConfig() {
+	parseConfig(configFilePath())
+	viper.AutomaticEnv()
 	if err := ascii(); err != nil {
 		log.Fatal(err)
 	}
 	enableDebug()
 	if err := insecureCerts(); err != nil {
 		log.Fatal(err)
 	}
-	parseConfig(configFilePath())
-	viper.AutomaticEnv()
 }
 
 func valueInConfigFile(key string) (string, error) {
@@ -159,6 +160,28 @@ func parseVarsFromConfig() {
 			log.Fatal(err)
 		}
 	}
+
+	showLogo = viper.GetBool("showLogo")
+	log.Info(showLogo)
+
+	if awsS3 {
+		awsBucket, err = valueInConfigFile("awsBucket")
+		if err != nil {
+			log.Fatal(err)
+		}
+		awsId, err = valueInConfigFile("awsId")
+		if err != nil {
+			log.Fatal(err)
+		}
+		awsRegion, err = valueInConfigFile("awsRegion")
+		if err != nil {
+			log.Fatal(err)
+		}
+		awsSecret, err = valueInConfigFile("awsSecret")
+		if err != nil {
+			log.Fatal(err)
+		}
+	}
 }
 
 func parseConfig(cfgFile string) {

docs/CHANGELOG.md

@@ -7,6 +7,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [6.7.0] - 2022-05-27
+
+### Added
+
+- support for uploading backup archives to AWS S3.
+- example how to upload a backup archive to AWS S3.
+- option to disable logo using the config file.
+
+### Changed
+
+- Alpine version to 3.16.2.
+- Golang version to 1.18.2-alpine3.16.
+- GolangCI version to v1.46.2-alpine.
+- Nexus3 and API version to 3.39.0.
+
 ## [6.6.2] - 2022-05-07
 
 ### Changed
@@ -697,7 +712,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - Download all artifacts from a certain Nexus3 repository.
 
-[Unreleased]: https://github.com/030/n3dr/compare/6.6.2...HEAD
+[Unreleased]: https://github.com/030/n3dr/compare/6.7.0...HEAD
+[6.7.0]: https://github.com/030/n3dr/compare/6.6.2...6.7.0
 [6.6.2]: https://github.com/030/n3dr/compare/6.6.1...6.6.2
 [6.6.1]: https://github.com/030/n3dr/compare/6.6.0...6.6.1
 [6.6.0]: https://github.com/030/n3dr/compare/6.5.1...6.6.0

examples/awsS3Upload/README.md

@@ -0,0 +1,12 @@
+# awsS3Upload
+
+Add the following parameters to `~/.n3dr/config.yml`:
+
+```bash
+awsBucket: someAwsBucket
+awsId: someAwsId
+awsRegion: someAwsRegion
+awsSecret: someAwsSecret
+```
+
+and use the `--awsS3` subcommand.

examples/showLogo/README.md

@@ -0,0 +1,15 @@
+# showLogo
+
+Add the following parameter to the config file:
+
+```bash
+showLogo: true
+```
+
+or
+
+```bash
+showLogo: false
+```
+
+to disable it.

go.mod

@@ -8,12 +8,13 @@ require (
 	github.com/PuerkitoBio/goquery v1.8.0
 	github.com/andybalholm/cascadia v1.3.1
 	github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d
+	github.com/aws/aws-sdk-go v1.44.23
 	github.com/go-openapi/errors v0.20.2
-	github.com/go-openapi/runtime v0.23.3
+	github.com/go-openapi/runtime v0.24.1
 	github.com/go-openapi/strfmt v0.21.2
 	github.com/go-openapi/swag v0.21.1
 	github.com/go-openapi/validate v0.21.0
-	github.com/go-playground/validator/v10 v10.10.1
+	github.com/go-playground/validator/v10 v10.11.0
 	github.com/hashicorp/go-retryablehttp v0.7.1
 	github.com/levigross/grequests v0.0.0-20190908174114-253788527a1a
 	github.com/mholt/archiver v3.1.1+incompatible
@@ -22,25 +23,24 @@ require (
 	github.com/sirupsen/logrus v1.8.1
 	github.com/spf13/cobra v1.4.0
 	github.com/spf13/jwalterweatherman v1.1.0
-	github.com/spf13/viper v1.11.0
+	github.com/spf13/viper v1.12.0
 	github.com/stretchr/testify v1.7.1
 	github.com/svenfuchs/jq v0.0.0-20180603193138-b038733a5990
 	github.com/thedevsaddam/gojsonq v2.3.0+incompatible
-	golang.org/x/net v0.0.0-20220421235706-1d1ef9303861
+	golang.org/x/net v0.0.0-20220526153639-5463443f8c37
 	gopkg.in/validator.v2 v2.0.1
 )
 
 require (
 	github.com/andybalholm/brotli v1.0.4 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 // indirect
-	github.com/frankban/quicktest v1.14.2 // indirect
-	github.com/fsnotify/fsnotify v1.5.3 // indirect
+	github.com/fsnotify/fsnotify v1.5.4 // indirect
 	github.com/go-openapi/analysis v0.21.3 // indirect
 	github.com/go-openapi/jsonpointer v0.19.5 // indirect
 	github.com/go-openapi/jsonreference v0.20.0 // indirect
 	github.com/go-openapi/loads v0.21.1 // indirect
-	github.com/go-openapi/spec v0.20.5 // indirect
+	github.com/go-openapi/spec v0.20.6 // indirect
 	github.com/go-playground/locales v0.14.0 // indirect
 	github.com/go-playground/universal-translator v0.18.0 // indirect
 	github.com/go-stack/stack v1.8.1 // indirect
@@ -49,38 +49,39 @@ require (
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
+	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
-	github.com/klauspost/compress v1.15.1 // indirect
+	github.com/klauspost/compress v1.15.5 // indirect
 	github.com/klauspost/pgzip v1.2.5 // indirect
 	github.com/leodido/go-urn v1.2.1 // indirect
 	github.com/magiconair/properties v1.8.6 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
-	github.com/mholt/archiver/v4 v4.0.0-alpha.6 // indirect
+	github.com/mholt/archiver/v4 v4.0.0-alpha.7 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/nwaples/rardecode v1.1.3 // indirect
 	github.com/nwaples/rardecode/v2 v2.0.0-beta.2 // indirect
 	github.com/oklog/ulid v1.3.1 // indirect
 	github.com/opentracing/opentracing-go v1.2.0 // indirect
 	github.com/pelletier/go-toml v1.9.5 // indirect
-	github.com/pelletier/go-toml/v2 v2.0.0-beta.8 // indirect
+	github.com/pelletier/go-toml/v2 v2.0.1 // indirect
 	github.com/pierrec/lz4 v2.6.1+incompatible // indirect
 	github.com/pierrec/lz4/v4 v4.1.14 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/spf13/afero v1.8.2 // indirect
-	github.com/spf13/cast v1.4.1 // indirect
+	github.com/spf13/cast v1.5.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/subosito/gotenv v1.2.0 // indirect
+	github.com/subosito/gotenv v1.3.0 // indirect
 	github.com/therootcompany/xz v1.0.1 // indirect
 	github.com/tidwall/gjson v1.14.1 // indirect
 	github.com/tidwall/match v1.1.1 // indirect
 	github.com/tidwall/pretty v1.2.0 // indirect
 	github.com/ulikunitz/xz v0.5.10 // indirect
 	github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect
-	go.mongodb.org/mongo-driver v1.9.0 // indirect
-	golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 // indirect
-	golang.org/x/sys v0.0.0-20220422013727-9388b58f7150 // indirect
+	go.mongodb.org/mongo-driver v1.9.1 // indirect
+	golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e // indirect
+	golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a // indirect
 	golang.org/x/text v0.3.7 // indirect
 	gopkg.in/ini.v1 v1.66.4 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
-	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )