Merge pull request #234 from 030/228-maven-upload-repositoriesv2

[GH-228] Upload maven artifacts using repositoriesV2.

CHANGELOG.md

@@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [6.4.0] - 2021-01-02
+
+### Added
+
+- Upload of Maven2 artifacts using `repositoriesV2`.
+
 ## [6.3.0] - 2021-12-31
 
 ### Added
@@ -591,7 +597,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - Download all artifacts from a certain Nexus3 repository.
 
-[Unreleased]: https://github.com/030/n3dr/compare/6.3.0...HEAD
+[Unreleased]: https://github.com/030/n3dr/compare/6.4.0...HEAD
+[6.4.0]: https://github.com/030/n3dr/compare/6.3.0...6.4.0
 [6.3.0]: https://github.com/030/n3dr/compare/6.2.0...6.3.0
 [6.2.0]: https://github.com/030/n3dr/compare/6.1.0...6.2.0
 [6.1.0]: https://github.com/030/n3dr/compare/6.0.13...6.1.0

README.md

@@ -314,7 +314,7 @@ n3dr config \
 ### Build
 
 ```bash
-docker build -t utrecht/n3dr:6.3.0 .
+docker build -t utrecht/n3dr:6.4.0 .
 ```
 
 [![dockeri.co](https://dockeri.co/image/utrecht/n3dr)](https://hub.docker.com/r/utrecht/n3dr)
@@ -324,7 +324,7 @@ docker build -t utrecht/n3dr:6.3.0 .
 ```bash
 docker run -it \
   -v /home/${USER}/.n3dr:/root/.n3dr \
-  -v /tmp/n3dr:/tmp/n3dr utrecht/n3dr:6.3.0
+  -v /tmp/n3dr:/tmp/n3dr utrecht/n3dr:6.4.0
 ```
 
 ### Upload
@@ -333,7 +333,7 @@ docker run -it \
 docker run -it \
   --entrypoint=/bin/ash \
   -v /home/${USER}/.n3dr:/root/.n3dr \
-  -v /tmp/n3dr:/tmp/n3dr utrecht/n3dr:6.3.0
+  -v /tmp/n3dr:/tmp/n3dr utrecht/n3dr:6.4.0
 ```
 
 navigate to the repository folder, e.g. `/tmp/n3dr/download*/` and upload:
@@ -545,7 +545,7 @@ a single command.
 | gitlfs    |        |        |       |
 | go        |        |        |       |
 | helm      |        |        |       |
-| maven2    | x      |        | `+`   |
+| maven2    | x      | x      | `+`   |
 | npm       | x      | x      | `*`   |
 | nuget     | x      | x      | `$`   |
 | p2        |        |        |       |

build/package/snap/snapcraft.yaml

@@ -1,7 +1,7 @@
 ---
 name: n3dr
 base: core20
-version: 6.3.0
+version: 6.4.0
 summary: Nexus3 Disaster Recovery
 description: |
   Download all artifacts at once or migrate automatically from Nexus to Nexus.

internal/artifactsv2/upload/upload.go

@@ -7,6 +7,7 @@ import (
 	"path/filepath"
 	"reflect"
 	"regexp"
+	"runtime"
 	"strconv"
 	"strings"
 	"sync"
@@ -24,6 +25,10 @@ type Nexus3 struct {
 	*connection.Nexus3
 }
 
+type mavenParts struct {
+	classifier, ext string
+}
+
 func uploadStatus(err error) (int, error) {
 	re := regexp.MustCompile(`status (\d{3})`)
 	match := re.FindStringSubmatch(err.Error())
@@ -104,16 +109,73 @@ func checkIfLocalArtifactResidesInNexus(continuationToken, localDiskRepo, path s
 	if continuationToken == "" {
 		return false, err
 	}
-	bestaat, err := checkIfLocalArtifactResidesInNexus(continuationToken, localDiskRepo, path, client)
+	exists, err := checkIfLocalArtifactResidesInNexus(continuationToken, localDiskRepo, path, client)
 	if err != nil {
 		return false, err
 	}
-	if bestaat {
+	if exists {
 		return true, nil
 	}
 	return false, err
 }
 
+func maven(path string, skipErrors bool) (mp mavenParts, err error) {
+	regexBase := `^.*\/([\w\-\.]+)\/`
+
+	if runtime.GOOS == "windows" {
+		log.Info("N3DR is running on Windows. Correcting the regexBase...")
+		regexBase = `^.*\\([\w\-\.]+)\\`
+	}
+
+	regexVersion := `(([a-z\d\-]+)|(([a-z\d\.]+)))`
+	if rv := os.Getenv("N3DR_MAVEN_UPLOAD_REGEX_VERSION"); rv != "" {
+		regexVersion = rv
+	}
+
+	regexClassifier := `(-(.*?(\-([\w.]+))?)?)?`
+	if rc := os.Getenv("N3DR_MAVEN_UPLOAD_REGEX_CLASSIFIER"); rc != "" {
+		regexClassifier = rc
+	}
+
+	re := regexp.MustCompile(regexBase + regexVersion + regexClassifier + `\.([a-z]+)$`)
+
+	classifier := ""
+	ext := ""
+	if re.Match([]byte(path)) {
+		result := re.FindAllStringSubmatch(path, -1)
+		artifactElements := result[0]
+		artifactElementsLength := len(result[0])
+		log.Debugf("ArtifactElements: '%s'. ArtifactElementLength: '%d'", artifactElements, artifactElementsLength)
+		if artifactElementsLength != 11 {
+			err := fmt.Errorf("check whether the regex retrieves ten elements from the artifact. Current: '%s'. Note that element 3 is the artifact itself", artifactElements)
+			if skipErrors {
+				log.Errorf("skipErrors: '%v'. Error: '%v'", skipErrors, err)
+			} else {
+				return mp, err
+			}
+		}
+
+		artifact := artifactElements[3]
+		version := artifactElements[1]
+		ext = artifactElements[10]
+
+		// Check if the 'version' reported in the artifact name is different from the 'real' version
+		if artifactElements[7] != artifactElements[1] {
+			classifier = artifactElements[9]
+		}
+
+		log.Debugf("Artifact: '%v', Version: '%v', Classifier: '%v', Extension: '%v'.", artifact, version, classifier, ext)
+	} else {
+		err := fmt.Errorf("check whether regexVersion: '%s' and regexClassifier: '%s' match the artifact: '%s'", regexVersion, regexClassifier, path)
+		if skipErrors {
+			log.Errorf("skipErrors: '%v'. Error: '%v'", skipErrors, err)
+		} else {
+			return mp, err
+		}
+	}
+	return mavenParts{classifier: classifier, ext: ext}, nil
+}
+
 func UploadSingleArtifact(client *client.Nexus3, path, localDiskRepo, localDiskRepoHome, repoFormat string) error {
 	dir := strings.Replace(filepath.Dir(path), localDiskRepoHome+"/", "", -1)
 	filename := filepath.Base(path)
@@ -122,12 +184,58 @@ func UploadSingleArtifact(client *client.Nexus3, path, localDiskRepo, localDiskR
 	c := components.UploadComponentParams{}
 	switch rf := repoFormat; rf {
 	case "apt":
+		fmt.Print("^")
 		c.Repository = localDiskRepo
 		f, err := os.Open(filepath.Clean(path))
 		if err != nil {
 			return err
 		}
 		c.AptAsset = f
+	case "maven2":
+		fmt.Print("+")
+		if filepath.Ext(path) == ".pom" {
+			c.Repository = localDiskRepo
+			f, err := os.Open(filepath.Clean(path))
+			if err != nil {
+				return err
+			}
+			c.Maven2Asset1 = f
+			ext1 := "pom"
+			c.Maven2Asset1Extension = &ext1
+
+			dir = filepath.Dir(path)
+			if err := filepath.WalkDir(dir,
+				func(path string, info fs.DirEntry, err error) error {
+					if err != nil {
+						return err
+					}
+					if filepath.Ext(path) == ".jar" {
+						f2, err = os.Open(filepath.Clean(path))
+						if err != nil {
+							return err
+						}
+						mp, err := maven(path, false)
+						if err != nil {
+							return err
+						}
+						c.Maven2Asset2 = f2
+						c.Maven2Asset2Extension = &mp.ext
+						c.Maven2Asset2Classifier = &mp.classifier
+					}
+					if filepath.Ext(path) == ".zip" {
+						f3, err = os.Open(filepath.Clean(path))
+						if err != nil {
+							return err
+						}
+						c.Maven2Asset3 = f3
+						ext3 := "zip"
+						c.Maven2Asset3Extension = &ext3
+					}
+					return nil
+				}); err != nil {
+				return err
+			}
+		}
 	case "npm":
 		fmt.Print("*")
 		c.Repository = localDiskRepo
@@ -183,7 +291,7 @@ func UploadSingleArtifact(client *client.Nexus3, path, localDiskRepo, localDiskR
 
 	files := []*os.File{f, f2, f3}
 	for _, file := range files {
-		if err := boek(file); err != nil {
+		if err := closeFileObjectIfNeeded(file); err != nil {
 			return err
 		}
 	}
@@ -231,7 +339,7 @@ func (n *Nexus3) ReadLocalDirAndUploadArtifacts(localDiskRepoHome, localDiskRepo
 	return nil
 }
 
-func boek(f *os.File) error {
+func closeFileObjectIfNeeded(f *os.File) error {
 	fi, err := f.Stat()
 	if err != nil {
 		return err

test/integration-tests.sh

@@ -1,12 +1,12 @@
-#!/bin/bash -e
+#!/bin/bash -ex
 
 if [ -z "${N3DR_APT_GPG_SECRET}" ]; then
   echo "N3DR_APT_GPG_SECRET should not be empty"
   echo "Create one by running:"
   echo "docker run -v /tmp/gpg-output:/root/.gnupg -v ${PWD}/test/gpg/:/tmp/ --rm -it vladgh/gpg --batch --generate-key /tmp/generate"
   echo "docker run --rm -it -v /tmp/gpg-output:/root/.gnupg -v ${PWD}/test/gpg/:/tmp/ vladgh/gpg --output /tmp/my_rsa_key --armor --export-secret-key [email protected]"
   echo "Enter 'abc' as a password, if the prompt appears"
-  printf "export N3DR_APT_GPG_SECRET=\$(sudo cat test/gpg/my_rsa_key | tr -d '\\\n')"
+  printf "export N3DR_APT_GPG_SECRET=\$(sudo cat test/gpg/my_rsa_key | tr '\\\n' ' ' | sed -r \"s|-----[A-Z]+ PGP PRIVATE KEY BLOCK-----||g;s| |\\\\\\\\\\\n|g;s|(.*)|-----BEGIN PGP PRIVATE KEY BLOCK-----\\\1-----END PGP PRIVATE KEY BLOCK-----|g\")"
   echo
   echo "sudo rm -r /tmp/gpg-output"
   echo "rm test/gpg/my_rsa_key"