From b9b1fac92d5e85c36568a20e54537bd0ff5a10b1 Mon Sep 17 00:00:00 2001 From: ben Date: Tue, 23 Mar 2021 10:13:06 +0100 Subject: [PATCH] [GH-194] skip errors. --- CHANGELOG.md | 9 +++- README.md | 12 ++++++ cmd/n3dr/root.go | 3 +- cmd/n3dr/upload.go | 2 +- internal/artifacts/upload.go | 72 ++++++++++++++++++++++--------- internal/artifacts/upload_test.go | 25 ++++++++--- test/integration-tests.sh | 16 +++---- 7 files changed, 103 insertions(+), 36 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6a34ddeb..468c5c49 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [6.0.6] - 2021-03-23 + +### Fixed + +- Not possible to skip errors, reported by [@tunix](https://github.com/tunix) + ## [6.0.5] - 2021-03-21 ### Fixed @@ -482,7 +488,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Download all artifacts from a certain Nexus3 repository. -[Unreleased]: https://github.com/030/n3dr/compare/6.0.5...HEAD +[Unreleased]: https://github.com/030/n3dr/compare/6.0.6...HEAD +[6.0.6]: https://github.com/030/n3dr/compare/6.0.5...6.0.6 [6.0.5]: https://github.com/030/n3dr/compare/6.0.4...6.0.5 [6.0.4]: https://github.com/030/n3dr/compare/6.0.3...6.0.4 [6.0.3]: https://github.com/030/n3dr/compare/6.0.2...6.0.3 diff --git a/README.md b/README.md index 27e0c839..f918016c 100644 --- a/README.md +++ b/README.md @@ -180,6 +180,18 @@ navigate to the repository folder, e.g. `/tmp/n3dr/download*/` and upload: n3dr upload -r releases -n ``` +#### skipErrors + +One could use `--skipErrors` or `-s` to continue-on-error: + +```bash +N3DR_MAVEN_UPLOAD_REGEX_VERSION=boo \ +N3DR_MAVEN_UPLOAD_REGEX_CLASSIFIER=foo \ +n3dr upload -n some-nexus-url \ + -r some-repository \ + -s +``` + ## Store the password in a read-only file Define the password in `~/.n3dr/config.yml`: diff --git a/cmd/n3dr/root.go b/cmd/n3dr/root.go index 0ae17f6a..20758e04 100644 --- a/cmd/n3dr/root.go +++ b/cmd/n3dr/root.go @@ -18,7 +18,7 @@ import ( var ( apiVersion, cfgFile, n3drRepo, n3drURL, n3drPass, n3drUser, Version, zipName, downloadDirName string - anonymous, debug, zip, insecureSkipVerify bool + anonymous, debug, insecureSkipVerify, skipErrors, zip bool ) var rootCmd = &cobra.Command{ @@ -50,6 +50,7 @@ func init() { rootCmd.PersistentFlags().StringVar(&downloadDirName, "directory-prefix", "", "directory to store downloaded artifacts") rootCmd.PersistentFlags().StringVarP(&apiVersion, "apiVersion", "v", "v1", "nexus3 APIVersion, e.g. v1 or beta") rootCmd.PersistentFlags().BoolVar(&anonymous, "anonymous", false, "Skip authentication") + rootCmd.PersistentFlags().BoolVarP(&skipErrors, "skipErrors", "s", false, "Skip errors") } func configFile() (string, error) { diff --git a/cmd/n3dr/upload.go b/cmd/n3dr/upload.go index 744dd9d7..d5be881e 100644 --- a/cmd/n3dr/upload.go +++ b/cmd/n3dr/upload.go @@ -19,7 +19,7 @@ a specific Nexus3 repository, e.g. maven-releases`, if err := n.ValidateNexusURL(); err != nil { log.Fatal(err) } - if err := n.Upload(); err != nil { + if err := n.Upload(skipErrors); err != nil { log.Fatal(err) } }, diff --git a/internal/artifacts/upload.go b/internal/artifacts/upload.go index 9a4aa925..e4d4a587 100644 --- a/internal/artifacts/upload.go +++ b/internal/artifacts/upload.go @@ -66,21 +66,45 @@ func sbArtifact(sb *strings.Builder, path, ext, classifier string) error { return nil } -func artifactTypeDetector(sb *strings.Builder, path string) error { +func artifactTypeDetector(sb *strings.Builder, path string, skipErrors bool) error { var err error - re := regexp.MustCompile(`^.*\/([\w\.-]+)-([\d\.]+)(-\d)?(-[0-9a-z]{8,40})?-?([a-z]+)?\.([a-z]+)$`) + regexVersion := `([\d\.-]+\d)` + if rv := os.Getenv("N3DR_MAVEN_UPLOAD_REGEX_VERSION"); rv != "" { + regexVersion = rv + } + + regexClassifier := `([a-zA-Z]+)?` + if rc := os.Getenv("N3DR_MAVEN_UPLOAD_REGEX_CLASSIFIER"); rc != "" { + regexClassifier = rc + } + + re := regexp.MustCompile(`^.*\/([\w\.-]+)-` + regexVersion + `-?` + regexClassifier + `\.([a-z]+)$`) if re.Match([]byte(path)) { result := re.FindAllStringSubmatch(path, -1) - log.Debugf("Artifact: '%v'", result[0][1]) - log.Debugf("Version: '%v'", result[0][2]+result[0][3]+result[0][4]) - classifier := result[0][5] - ext := result[0][6] + artifactElements := result[0] + log.Debugf("ArtifactElements: '%s'", artifactElements) + if len(result[0]) != 5 { + err := fmt.Errorf("Check whether the regex retrieves four elements from the artifact. Current: '%s'. Note that element 0 is the artifact itself.", artifactElements) + if skipErrors { + log.Errorf("skipErrors: '%v'. Error: '%v'", skipErrors, err) + } else { + return err + } + } + artifact := artifactElements[1] + version := artifactElements[2] + classifier := artifactElements[3] + ext := artifactElements[4] + log.Infof("Artifact: '%v', Version: '%v', Classifier: '%v', Extension: '%v'.", artifact, version, classifier, ext) err = sbArtifact(sb, path, ext, classifier) } else { - log.Warningf("'%v' not an artifact", path) - // return nil to continue-on-error to ensure that subsequent artifacts - // will be uploaded + err := fmt.Errorf("Check whether regexVersion: '%s' and regexClassifier: '%s' match the artifact: '%s'", regexVersion, regexClassifier, path) + if skipErrors { + log.Errorf("skipErrors: '%v'. Error: '%v'", skipErrors, err) + } else { + return err + } } return err @@ -101,22 +125,21 @@ func (n Nexus3) multipartUpload(sb strings.Builder) error { return nil } -func pomDirs(p string) (strings.Builder, error) { +func pomDirs(p string, skipErrors bool) (strings.Builder, error) { var sb strings.Builder artifactIndex = 1 - err := filepath.Walk(p, func(path string, f os.FileInfo, err error) error { + if err := filepath.Walk(p, func(path string, f os.FileInfo, err error) error { if err != nil { return err } if !f.IsDir() { - if err = artifactTypeDetector(&sb, path); err != nil { + if err = artifactTypeDetector(&sb, path, skipErrors); err != nil { return err } } return nil - }) - if err != nil { + }); err != nil { return sb, err } return sb, nil @@ -252,29 +275,38 @@ func (n *Nexus3) readFilesAndUpload() error { return nil } -func (n *Nexus3) readMavenFilesAndUpload() error { +func (n *Nexus3) readMavenFilesAndUpload(skipErrors bool) error { if err := n.detectFoldersWithPOM(n.Repository); err != nil { return err } for i, path := range foldersWithPOMStringSlice { log.Debug(strconv.Itoa(i) + " Detecting artifacts in folder '" + path + "'") - sb, err := pomDirs(path) + sb, err := pomDirs(path, skipErrors) if err != nil { return err } if sb.String() == "" { - return fmt.Errorf("The sb.String() should not be empty. Verify whether the path: '%s' contains artifacts", path) + err := fmt.Errorf("The sb.String() should not be empty. Verify whether the path: '%s' contains artifacts", path) + if skipErrors { + log.Errorf("skipErrors: '%v'. Error: '%v'", skipErrors, err) + } else { + return err + } } log.Info(strconv.Itoa(i) + " Upload '" + sb.String() + "'") if err := n.multipartUpload(sb); err != nil { - return err + if skipErrors { + log.Errorf("skipErrors: '%v'. Error: '%v'", skipErrors, err) + } else { + return err + } } } return nil } // Upload posts an artifact as a multipart to a specific nexus3 repository -func (n Nexus3) Upload() error { +func (n Nexus3) Upload(skipErrors bool) error { log.Infof("Uploading '%s'", n.ArtifactType) switch n.ArtifactType { case "apt": @@ -282,7 +314,7 @@ func (n Nexus3) Upload() error { return err } case "maven2": - if err := n.readMavenFilesAndUpload(); err != nil { + if err := n.readMavenFilesAndUpload(skipErrors); err != nil { return err } case "npm": diff --git a/internal/artifacts/upload_test.go b/internal/artifacts/upload_test.go index 7f5799fb..8b7a1fc2 100644 --- a/internal/artifacts/upload_test.go +++ b/internal/artifacts/upload_test.go @@ -1,19 +1,34 @@ package artifacts import ( + "os" "strings" "testing" "github.com/stretchr/testify/assert" ) +func TestArtifactTypeDetectorErrors(t *testing.T) { + var sb strings.Builder + + os.Setenv("N3DR_MAVEN_UPLOAD_REGEX_VERSION", "notAMatch") + err := artifactTypeDetector(&sb, "a/b/c.b.a/1.2.3/a-b-c-1.2.3-dataset.jar", false) + assert.EqualError(t, err, "Check whether regexVersion: 'notAMatch' and regexClassifier: '([a-zA-Z]+)?' match the artifact: 'a/b/c.b.a/1.2.3/a-b-c-1.2.3-dataset.jar'") + + os.Setenv("N3DR_MAVEN_UPLOAD_REGEX_CLASSIFIER", "notAMatch2") + err = artifactTypeDetector(&sb, "a/b/c.b.a/1.2.3/a-b-c-1.2.3-dataset.jar", false) + assert.EqualError(t, err, "Check whether regexVersion: 'notAMatch' and regexClassifier: 'notAMatch2' match the artifact: 'a/b/c.b.a/1.2.3/a-b-c-1.2.3-dataset.jar'") +} + func TestArtifactTypeDetector(t *testing.T) { var sb strings.Builder - artifactTypeDetector(&sb, "a/b/c.b.a/1.2.3/a-b-c-1.2.3-dataset.jar") - artifactTypeDetector(&sb, "d/e/f-e-d/4.5.6/d.e.f-4.5.6.pom") - artifactTypeDetector(&sb, "d/e/f-e-d/7.8.9-2-37zgb398/d.e.f-7.8.9-2-37zgb398.war") - artifactTypeDetector(&sb, "d/e/f-e-d/9.8.7-81ae5835bb36126fe8091e82t14521841d8y0133/d.e.f-9.8.7-81ae5835bb36126fe8091e82t14521841d8y0133.war") + os.Setenv("N3DR_MAVEN_UPLOAD_REGEX_VERSION", `([\d\.-]+\d)`) + os.Setenv("N3DR_MAVEN_UPLOAD_REGEX_CLASSIFIER", `([a-zA-Z]+)?`) + + artifactTypeDetector(&sb, "a/b/c.b.a/1.2.3/a-b-c-1.2.3-dataset.jar", false) + artifactTypeDetector(&sb, "d/e/f-e-d/4.5.6/d.e.f-4.5.6.pom", false) + artifactTypeDetector(&sb, "d/e/f-e-d/7.8.9-2/d.e.f-7.8.9-2.war", false) - assert.Equal(t, "maven2.asset0=@a/b/c.b.a/1.2.3/a-b-c-1.2.3-dataset.jar,maven2.asset0.extension=jar,maven2.asset0.classifier=dataset,maven2.asset1=@d/e/f-e-d/4.5.6/d.e.f-4.5.6.pom,maven2.asset1.extension=pom,maven2.asset2=@d/e/f-e-d/7.8.9-2-37zgb398/d.e.f-7.8.9-2-37zgb398.war,maven2.asset2.extension=war,maven2.asset3=@d/e/f-e-d/9.8.7-81ae5835bb36126fe8091e82t14521841d8y0133/d.e.f-9.8.7-81ae5835bb36126fe8091e82t14521841d8y0133.war,maven2.asset3.extension=war,", sb.String()) + assert.Equal(t, "maven2.asset0=@a/b/c.b.a/1.2.3/a-b-c-1.2.3-dataset.jar,maven2.asset0.extension=jar,maven2.asset0.classifier=dataset,maven2.asset1=@d/e/f-e-d/4.5.6/d.e.f-4.5.6.pom,maven2.asset1.extension=pom,maven2.asset2=@d/e/f-e-d/7.8.9-2/d.e.f-7.8.9-2.war,maven2.asset2.extension=war,", sb.String()) } diff --git a/test/integration-tests.sh b/test/integration-tests.sh index 6489202c..b9a36fba 100755 --- a/test/integration-tests.sh +++ b/test/integration-tests.sh @@ -62,10 +62,10 @@ nexus(){ } artifact(){ - mkdir -p "maven-releases/some/group${1}/file${1}/1.0.0-2-gcac5af6" - echo someContent > "maven-releases/some/group${1}/file${1}/1.0.0-2-gcac5af6/f.i-l.e.${1}-1.0.0-2-gcac5af6.jar" - echo someContentZIP > "maven-releases/some/group${1}/file${1}/1.0.0-2-gcac5af6/file${1}-1.0.0-2-gcac5af6.zip" - echo -e "\n4.0.0\nsome.group${1}\nfile${1}\n1.0.0-2-gcac5af6\n" > "maven-releases/some/group${1}/file${1}/1.0.0-2-gcac5af6/file${1}-1.0.0-2-gcac5af6.pom" + mkdir -p "maven-releases/some/group${1}/file${1}/1.0.0-2" + echo someContent > "maven-releases/some/group${1}/file${1}/1.0.0-2/f.i-l.e.${1}-1.0.0-2.jar" + echo someContentZIP > "maven-releases/some/group${1}/file${1}/1.0.0-2/file${1}-1.0.0-2.zip" + echo -e "\n4.0.0\nsome.group${1}\nfile${1}\n1.0.0-2\n" > "maven-releases/some/group${1}/file${1}/1.0.0-2/file${1}-1.0.0-2.pom" } files(){ @@ -156,10 +156,10 @@ uploadNuget(){ backupHelper(){ if [ "${NEXUS_VERSION}" == "3.9.0" ]; then count_downloads 300 - test_zip 164 + test_zip 152 else count_downloads 400 - test_zip 208 + test_zip 196 fi cleanup_downloads } @@ -218,10 +218,10 @@ repositories(){ $cmd -b -z if [ "${NEXUS_VERSION}" == "3.9.0" ]; then count_downloads 300 - test_zip 164 + test_zip 152 else count_downloads 401 - test_zip 248 + test_zip 232 fi cleanup_downloads }