Merge pull request #204 from 030/194-regex-env-variables

[#194] Allow overwrite of regex to ensure that various artifact types…

CHANGELOG.md

@@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [6.0.6] - 2021-03-23
+
+### Fixed
+
+- Not possible to skip errors, reported by [@tunix](https://github.com/tunix)
+
 ## [6.0.5] - 2021-03-21
 
 ### Fixed
@@ -482,7 +488,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - Download all artifacts from a certain Nexus3 repository.
 
-[Unreleased]: https://github.com/030/n3dr/compare/6.0.5...HEAD
+[Unreleased]: https://github.com/030/n3dr/compare/6.0.6...HEAD
+[6.0.6]: https://github.com/030/n3dr/compare/6.0.5...6.0.6
 [6.0.5]: https://github.com/030/n3dr/compare/6.0.4...6.0.5
 [6.0.4]: https://github.com/030/n3dr/compare/6.0.3...6.0.4
 [6.0.3]: https://github.com/030/n3dr/compare/6.0.2...6.0.3

README.md

@@ -180,6 +180,18 @@ navigate to the repository folder, e.g. `/tmp/n3dr/download*/` and upload:
 n3dr upload -r releases -n <url>
 ```
 
+#### skipErrors
+
+One could use `--skipErrors` or `-s` to continue-on-error:
+
+```bash
+N3DR_MAVEN_UPLOAD_REGEX_VERSION=boo \
+N3DR_MAVEN_UPLOAD_REGEX_CLASSIFIER=foo \
+n3dr upload -n some-nexus-url \
+            -r some-repository \
+            -s
+```
+
 ## Store the password in a read-only file
 
 Define the password in `~/.n3dr/config.yml`:

cmd/n3dr/root.go

@@ -18,7 +18,7 @@ import (
 
 var (
 	apiVersion, cfgFile, n3drRepo, n3drURL, n3drPass, n3drUser, Version, zipName, downloadDirName string
-	anonymous, debug, zip, insecureSkipVerify                                                     bool
+	anonymous, debug, insecureSkipVerify, skipErrors, zip                                         bool
 )
 
 var rootCmd = &cobra.Command{
@@ -50,6 +50,7 @@ func init() {
 	rootCmd.PersistentFlags().StringVar(&downloadDirName, "directory-prefix", "", "directory to store downloaded artifacts")
 	rootCmd.PersistentFlags().StringVarP(&apiVersion, "apiVersion", "v", "v1", "nexus3 APIVersion, e.g. v1 or beta")
 	rootCmd.PersistentFlags().BoolVar(&anonymous, "anonymous", false, "Skip authentication")
+	rootCmd.PersistentFlags().BoolVarP(&skipErrors, "skipErrors", "s", false, "Skip errors")
 }
 
 func configFile() (string, error) {

cmd/n3dr/upload.go

@@ -19,7 +19,7 @@ a specific Nexus3 repository, e.g. maven-releases`,
 		if err := n.ValidateNexusURL(); err != nil {
 			log.Fatal(err)
 		}
-		if err := n.Upload(); err != nil {
+		if err := n.Upload(skipErrors); err != nil {
 			log.Fatal(err)
 		}
 	},

internal/artifacts/upload.go

@@ -66,21 +66,45 @@ func sbArtifact(sb *strings.Builder, path, ext, classifier string) error {
 	return nil
 }
 
-func artifactTypeDetector(sb *strings.Builder, path string) error {
+func artifactTypeDetector(sb *strings.Builder, path string, skipErrors bool) error {
 	var err error
 
-	re := regexp.MustCompile(`^.*\/([\w\.-]+)-([\d\.]+)(-\d)?(-[0-9a-z]{8,40})?-?([a-z]+)?\.([a-z]+)$`)
+	regexVersion := `([\d\.-]+\d)`
+	if rv := os.Getenv("N3DR_MAVEN_UPLOAD_REGEX_VERSION"); rv != "" {
+		regexVersion = rv
+	}
+
+	regexClassifier := `([a-zA-Z]+)?`
+	if rc := os.Getenv("N3DR_MAVEN_UPLOAD_REGEX_CLASSIFIER"); rc != "" {
+		regexClassifier = rc
+	}
+
+	re := regexp.MustCompile(`^.*\/([\w\.-]+)-` + regexVersion + `-?` + regexClassifier + `\.([a-z]+)$`)
 	if re.Match([]byte(path)) {
 		result := re.FindAllStringSubmatch(path, -1)
-		log.Debugf("Artifact: '%v'", result[0][1])
-		log.Debugf("Version: '%v'", result[0][2]+result[0][3]+result[0][4])
-		classifier := result[0][5]
-		ext := result[0][6]
+		artifactElements := result[0]
+		log.Debugf("ArtifactElements: '%s'", artifactElements)
+		if len(result[0]) != 5 {
+			err := fmt.Errorf("Check whether the regex retrieves four elements from the artifact. Current: '%s'. Note that element 0 is the artifact itself.", artifactElements)
+			if skipErrors {
+				log.Errorf("skipErrors: '%v'. Error: '%v'", skipErrors, err)
+			} else {
+				return err
+			}
+		}
+		artifact := artifactElements[1]
+		version := artifactElements[2]
+		classifier := artifactElements[3]
+		ext := artifactElements[4]
+		log.Infof("Artifact: '%v', Version: '%v', Classifier: '%v', Extension: '%v'.", artifact, version, classifier, ext)
 		err = sbArtifact(sb, path, ext, classifier)
 	} else {
-		log.Warningf("'%v' not an artifact", path)
-		// return nil to continue-on-error to ensure that subsequent artifacts
-		// will be uploaded
+		err := fmt.Errorf("Check whether regexVersion: '%s' and regexClassifier: '%s' match the artifact: '%s'", regexVersion, regexClassifier, path)
+		if skipErrors {
+			log.Errorf("skipErrors: '%v'. Error: '%v'", skipErrors, err)
+		} else {
+			return err
+		}
 	}
 
 	return err
@@ -101,22 +125,21 @@ func (n Nexus3) multipartUpload(sb strings.Builder) error {
 	return nil
 }
 
-func pomDirs(p string) (strings.Builder, error) {
+func pomDirs(p string, skipErrors bool) (strings.Builder, error) {
 	var sb strings.Builder
 	artifactIndex = 1
 
-	err := filepath.Walk(p, func(path string, f os.FileInfo, err error) error {
+	if err := filepath.Walk(p, func(path string, f os.FileInfo, err error) error {
 		if err != nil {
 			return err
 		}
 		if !f.IsDir() {
-			if err = artifactTypeDetector(&sb, path); err != nil {
+			if err = artifactTypeDetector(&sb, path, skipErrors); err != nil {
 				return err
 			}
 		}
 		return nil
-	})
-	if err != nil {
+	}); err != nil {
 		return sb, err
 	}
 	return sb, nil
@@ -252,37 +275,46 @@ func (n *Nexus3) readFilesAndUpload() error {
 	return nil
 }
 
-func (n *Nexus3) readMavenFilesAndUpload() error {
+func (n *Nexus3) readMavenFilesAndUpload(skipErrors bool) error {
 	if err := n.detectFoldersWithPOM(n.Repository); err != nil {
 		return err
 	}
 	for i, path := range foldersWithPOMStringSlice {
 		log.Debug(strconv.Itoa(i) + " Detecting artifacts in folder '" + path + "'")
-		sb, err := pomDirs(path)
+		sb, err := pomDirs(path, skipErrors)
 		if err != nil {
 			return err
 		}
 		if sb.String() == "" {
-			return fmt.Errorf("The sb.String() should not be empty. Verify whether the path: '%s' contains artifacts", path)
+			err := fmt.Errorf("The sb.String() should not be empty. Verify whether the path: '%s' contains artifacts", path)
+			if skipErrors {
+				log.Errorf("skipErrors: '%v'. Error: '%v'", skipErrors, err)
+			} else {
+				return err
+			}
 		}
 		log.Info(strconv.Itoa(i) + " Upload '" + sb.String() + "'")
 		if err := n.multipartUpload(sb); err != nil {
-			return err
+			if skipErrors {
+				log.Errorf("skipErrors: '%v'. Error: '%v'", skipErrors, err)
+			} else {
+				return err
+			}
 		}
 	}
 	return nil
 }
 
 // Upload posts an artifact as a multipart to a specific nexus3 repository
-func (n Nexus3) Upload() error {
+func (n Nexus3) Upload(skipErrors bool) error {
 	log.Infof("Uploading '%s'", n.ArtifactType)
 	switch n.ArtifactType {
 	case "apt":
 		if err := n.readFilesAndUpload(); err != nil {
 			return err
 		}
 	case "maven2":
-		if err := n.readMavenFilesAndUpload(); err != nil {
+		if err := n.readMavenFilesAndUpload(skipErrors); err != nil {
 			return err
 		}
 	case "npm":

internal/artifacts/upload_test.go

@@ -1,19 +1,34 @@
 package artifacts
 
 import (
+	"os"
 	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
 )
 
+func TestArtifactTypeDetectorErrors(t *testing.T) {
+	var sb strings.Builder
+
+	os.Setenv("N3DR_MAVEN_UPLOAD_REGEX_VERSION", "notAMatch")
+	err := artifactTypeDetector(&sb, "a/b/c.b.a/1.2.3/a-b-c-1.2.3-dataset.jar", false)
+	assert.EqualError(t, err, "Check whether regexVersion: 'notAMatch' and regexClassifier: '([a-zA-Z]+)?' match the artifact: 'a/b/c.b.a/1.2.3/a-b-c-1.2.3-dataset.jar'")
+
+	os.Setenv("N3DR_MAVEN_UPLOAD_REGEX_CLASSIFIER", "notAMatch2")
+	err = artifactTypeDetector(&sb, "a/b/c.b.a/1.2.3/a-b-c-1.2.3-dataset.jar", false)
+	assert.EqualError(t, err, "Check whether regexVersion: 'notAMatch' and regexClassifier: 'notAMatch2' match the artifact: 'a/b/c.b.a/1.2.3/a-b-c-1.2.3-dataset.jar'")
+}
+
 func TestArtifactTypeDetector(t *testing.T) {
 	var sb strings.Builder
 
-	artifactTypeDetector(&sb, "a/b/c.b.a/1.2.3/a-b-c-1.2.3-dataset.jar")
-	artifactTypeDetector(&sb, "d/e/f-e-d/4.5.6/d.e.f-4.5.6.pom")
-	artifactTypeDetector(&sb, "d/e/f-e-d/7.8.9-2-37zgb398/d.e.f-7.8.9-2-37zgb398.war")
-	artifactTypeDetector(&sb, "d/e/f-e-d/9.8.7-81ae5835bb36126fe8091e82t14521841d8y0133/d.e.f-9.8.7-81ae5835bb36126fe8091e82t14521841d8y0133.war")
+	os.Setenv("N3DR_MAVEN_UPLOAD_REGEX_VERSION", `([\d\.-]+\d)`)
+	os.Setenv("N3DR_MAVEN_UPLOAD_REGEX_CLASSIFIER", `([a-zA-Z]+)?`)
+
+	artifactTypeDetector(&sb, "a/b/c.b.a/1.2.3/a-b-c-1.2.3-dataset.jar", false)
+	artifactTypeDetector(&sb, "d/e/f-e-d/4.5.6/d.e.f-4.5.6.pom", false)
+	artifactTypeDetector(&sb, "d/e/f-e-d/7.8.9-2/d.e.f-7.8.9-2.war", false)
 
-	assert.Equal(t, "maven2.asset0=@a/b/c.b.a/1.2.3/a-b-c-1.2.3-dataset.jar,maven2.asset0.extension=jar,maven2.asset0.classifier=dataset,maven2.asset1=@d/e/f-e-d/4.5.6/d.e.f-4.5.6.pom,maven2.asset1.extension=pom,maven2.asset2=@d/e/f-e-d/7.8.9-2-37zgb398/d.e.f-7.8.9-2-37zgb398.war,maven2.asset2.extension=war,maven2.asset3=@d/e/f-e-d/9.8.7-81ae5835bb36126fe8091e82t14521841d8y0133/d.e.f-9.8.7-81ae5835bb36126fe8091e82t14521841d8y0133.war,maven2.asset3.extension=war,", sb.String())
+	assert.Equal(t, "maven2.asset0=@a/b/c.b.a/1.2.3/a-b-c-1.2.3-dataset.jar,maven2.asset0.extension=jar,maven2.asset0.classifier=dataset,maven2.asset1=@d/e/f-e-d/4.5.6/d.e.f-4.5.6.pom,maven2.asset1.extension=pom,maven2.asset2=@d/e/f-e-d/7.8.9-2/d.e.f-7.8.9-2.war,maven2.asset2.extension=war,", sb.String())
 }

test/integration-tests.sh

@@ -62,10 +62,10 @@ nexus(){
 }
 
 artifact(){
-  mkdir -p "maven-releases/some/group${1}/file${1}/1.0.0-2-gcac5af6"
-  echo someContent > "maven-releases/some/group${1}/file${1}/1.0.0-2-gcac5af6/f.i-l.e.${1}-1.0.0-2-gcac5af6.jar"
-  echo someContentZIP > "maven-releases/some/group${1}/file${1}/1.0.0-2-gcac5af6/file${1}-1.0.0-2-gcac5af6.zip"
-  echo -e "<project>\n<modelVersion>4.0.0</modelVersion>\n<groupId>some.group${1}</groupId>\n<artifactId>file${1}</artifactId>\n<version>1.0.0-2-gcac5af6</version>\n</project>" > "maven-releases/some/group${1}/file${1}/1.0.0-2-gcac5af6/file${1}-1.0.0-2-gcac5af6.pom"
+  mkdir -p "maven-releases/some/group${1}/file${1}/1.0.0-2"
+  echo someContent > "maven-releases/some/group${1}/file${1}/1.0.0-2/f.i-l.e.${1}-1.0.0-2.jar"
+  echo someContentZIP > "maven-releases/some/group${1}/file${1}/1.0.0-2/file${1}-1.0.0-2.zip"
+  echo -e "<project>\n<modelVersion>4.0.0</modelVersion>\n<groupId>some.group${1}</groupId>\n<artifactId>file${1}</artifactId>\n<version>1.0.0-2</version>\n</project>" > "maven-releases/some/group${1}/file${1}/1.0.0-2/file${1}-1.0.0-2.pom"
 }
 
 files(){
@@ -156,10 +156,10 @@ uploadNuget(){
 backupHelper(){
   if [ "${NEXUS_VERSION}" == "3.9.0" ]; then
     count_downloads 300
-    test_zip 164
+    test_zip 152
   else
     count_downloads 400
-    test_zip 208
+    test_zip 196
   fi
   cleanup_downloads
 }
@@ -218,10 +218,10 @@ repositories(){
   $cmd -b -z
   if [ "${NEXUS_VERSION}" == "3.9.0" ]; then
     count_downloads 300
-    test_zip 164
+    test_zip 152
   else
     count_downloads 401
-    test_zip 248
+    test_zip 232
   fi
   cleanup_downloads
 }