From 416ac3b83ecdc6fe8d054b1c40fb7baf5b6294fd Mon Sep 17 00:00:00 2001 From: ben Date: Sun, 2 Jan 2022 00:14:19 +0100 Subject: [PATCH] [GH-228] Upload maven artifacts using repositoriesV2. --- CHANGELOG.md | 9 +- README.md | 8 +- build/package/snap/snapcraft.yaml | 2 +- internal/artifactsv2/upload/upload.go | 116 +++++++++++++++++++++++++- test/integration-tests.sh | 4 +- 5 files changed, 127 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 84212245..3b0253e8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [6.4.0] - 2021-01-02 + +### Added + +- Upload of Maven2 artifacts using `repositoriesV2`. + ## [6.3.0] - 2021-12-31 ### Added @@ -591,7 +597,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Download all artifacts from a certain Nexus3 repository. -[Unreleased]: https://github.com/030/n3dr/compare/6.3.0...HEAD +[Unreleased]: https://github.com/030/n3dr/compare/6.4.0...HEAD +[6.4.0]: https://github.com/030/n3dr/compare/6.3.0...6.4.0 [6.3.0]: https://github.com/030/n3dr/compare/6.2.0...6.3.0 [6.2.0]: https://github.com/030/n3dr/compare/6.1.0...6.2.0 [6.1.0]: https://github.com/030/n3dr/compare/6.0.13...6.1.0 diff --git a/README.md b/README.md index 1e41ae94..0b806a29 100644 --- a/README.md +++ b/README.md @@ -314,7 +314,7 @@ n3dr config \ ### Build ```bash -docker build -t utrecht/n3dr:6.3.0 . +docker build -t utrecht/n3dr:6.4.0 . ``` [![dockeri.co](https://dockeri.co/image/utrecht/n3dr)](https://hub.docker.com/r/utrecht/n3dr) @@ -324,7 +324,7 @@ docker build -t utrecht/n3dr:6.3.0 . ```bash docker run -it \ -v /home/${USER}/.n3dr:/root/.n3dr \ - -v /tmp/n3dr:/tmp/n3dr utrecht/n3dr:6.3.0 + -v /tmp/n3dr:/tmp/n3dr utrecht/n3dr:6.4.0 ``` ### Upload @@ -333,7 +333,7 @@ docker run -it \ docker run -it \ --entrypoint=/bin/ash \ -v /home/${USER}/.n3dr:/root/.n3dr \ - -v /tmp/n3dr:/tmp/n3dr utrecht/n3dr:6.3.0 + -v /tmp/n3dr:/tmp/n3dr utrecht/n3dr:6.4.0 ``` navigate to the repository folder, e.g. `/tmp/n3dr/download*/` and upload: @@ -545,7 +545,7 @@ a single command. | gitlfs | | | | | go | | | | | helm | | | | -| maven2 | x | | `+` | +| maven2 | x | x | `+` | | npm | x | x | `*` | | nuget | x | x | `$` | | p2 | | | | diff --git a/build/package/snap/snapcraft.yaml b/build/package/snap/snapcraft.yaml index 95840863..91b90aa6 100644 --- a/build/package/snap/snapcraft.yaml +++ b/build/package/snap/snapcraft.yaml @@ -1,7 +1,7 @@ --- name: n3dr base: core20 -version: 6.3.0 +version: 6.4.0 summary: Nexus3 Disaster Recovery description: | Download all artifacts at once or migrate automatically from Nexus to Nexus. diff --git a/internal/artifactsv2/upload/upload.go b/internal/artifactsv2/upload/upload.go index b72bda33..3e0cb092 100644 --- a/internal/artifactsv2/upload/upload.go +++ b/internal/artifactsv2/upload/upload.go @@ -7,6 +7,7 @@ import ( "path/filepath" "reflect" "regexp" + "runtime" "strconv" "strings" "sync" @@ -24,6 +25,10 @@ type Nexus3 struct { *connection.Nexus3 } +type mavenParts struct { + classifier, ext string +} + func uploadStatus(err error) (int, error) { re := regexp.MustCompile(`status (\d{3})`) match := re.FindStringSubmatch(err.Error()) @@ -104,16 +109,73 @@ func checkIfLocalArtifactResidesInNexus(continuationToken, localDiskRepo, path s if continuationToken == "" { return false, err } - bestaat, err := checkIfLocalArtifactResidesInNexus(continuationToken, localDiskRepo, path, client) + exists, err := checkIfLocalArtifactResidesInNexus(continuationToken, localDiskRepo, path, client) if err != nil { return false, err } - if bestaat { + if exists { return true, nil } return false, err } +func maven(path string, skipErrors bool) (mp mavenParts, err error) { + regexBase := `^.*\/([\w\-\.]+)\/` + + if runtime.GOOS == "windows" { + log.Info("N3DR is running on Windows. Correcting the regexBase...") + regexBase = `^.*\\([\w\-\.]+)\\` + } + + regexVersion := `(([a-z\d\-]+)|(([a-z\d\.]+)))` + if rv := os.Getenv("N3DR_MAVEN_UPLOAD_REGEX_VERSION"); rv != "" { + regexVersion = rv + } + + regexClassifier := `(-(.*?(\-([\w.]+))?)?)?` + if rc := os.Getenv("N3DR_MAVEN_UPLOAD_REGEX_CLASSIFIER"); rc != "" { + regexClassifier = rc + } + + re := regexp.MustCompile(regexBase + regexVersion + regexClassifier + `\.([a-z]+)$`) + + classifier := "" + ext := "" + if re.Match([]byte(path)) { + result := re.FindAllStringSubmatch(path, -1) + artifactElements := result[0] + artifactElementsLength := len(result[0]) + log.Debugf("ArtifactElements: '%s'. ArtifactElementLength: '%d'", artifactElements, artifactElementsLength) + if artifactElementsLength != 11 { + err := fmt.Errorf("check whether the regex retrieves ten elements from the artifact. Current: '%s'. Note that element 3 is the artifact itself", artifactElements) + if skipErrors { + log.Errorf("skipErrors: '%v'. Error: '%v'", skipErrors, err) + } else { + return mp, err + } + } + + artifact := artifactElements[3] + version := artifactElements[1] + ext = artifactElements[10] + + // Check if the 'version' reported in the artifact name is different from the 'real' version + if artifactElements[7] != artifactElements[1] { + classifier = artifactElements[9] + } + + log.Debugf("Artifact: '%v', Version: '%v', Classifier: '%v', Extension: '%v'.", artifact, version, classifier, ext) + } else { + err := fmt.Errorf("check whether regexVersion: '%s' and regexClassifier: '%s' match the artifact: '%s'", regexVersion, regexClassifier, path) + if skipErrors { + log.Errorf("skipErrors: '%v'. Error: '%v'", skipErrors, err) + } else { + return mp, err + } + } + return mavenParts{classifier: classifier, ext: ext}, nil +} + func UploadSingleArtifact(client *client.Nexus3, path, localDiskRepo, localDiskRepoHome, repoFormat string) error { dir := strings.Replace(filepath.Dir(path), localDiskRepoHome+"/", "", -1) filename := filepath.Base(path) @@ -122,12 +184,58 @@ func UploadSingleArtifact(client *client.Nexus3, path, localDiskRepo, localDiskR c := components.UploadComponentParams{} switch rf := repoFormat; rf { case "apt": + fmt.Print("^") c.Repository = localDiskRepo f, err := os.Open(filepath.Clean(path)) if err != nil { return err } c.AptAsset = f + case "maven2": + fmt.Print("+") + if filepath.Ext(path) == ".pom" { + c.Repository = localDiskRepo + f, err := os.Open(filepath.Clean(path)) + if err != nil { + return err + } + c.Maven2Asset1 = f + ext1 := "pom" + c.Maven2Asset1Extension = &ext1 + + dir = filepath.Dir(path) + if err := filepath.WalkDir(dir, + func(path string, info fs.DirEntry, err error) error { + if err != nil { + return err + } + if filepath.Ext(path) == ".jar" { + f2, err = os.Open(filepath.Clean(path)) + if err != nil { + return err + } + mp, err := maven(path, false) + if err != nil { + return err + } + c.Maven2Asset2 = f2 + c.Maven2Asset2Extension = &mp.ext + c.Maven2Asset2Classifier = &mp.classifier + } + if filepath.Ext(path) == ".zip" { + f3, err = os.Open(filepath.Clean(path)) + if err != nil { + return err + } + c.Maven2Asset3 = f3 + ext3 := "zip" + c.Maven2Asset3Extension = &ext3 + } + return nil + }); err != nil { + return err + } + } case "npm": fmt.Print("*") c.Repository = localDiskRepo @@ -183,7 +291,7 @@ func UploadSingleArtifact(client *client.Nexus3, path, localDiskRepo, localDiskR files := []*os.File{f, f2, f3} for _, file := range files { - if err := boek(file); err != nil { + if err := closeFileObjectIfNeeded(file); err != nil { return err } } @@ -231,7 +339,7 @@ func (n *Nexus3) ReadLocalDirAndUploadArtifacts(localDiskRepoHome, localDiskRepo return nil } -func boek(f *os.File) error { +func closeFileObjectIfNeeded(f *os.File) error { fi, err := f.Stat() if err != nil { return err diff --git a/test/integration-tests.sh b/test/integration-tests.sh index ed57d392..009aa654 100755 --- a/test/integration-tests.sh +++ b/test/integration-tests.sh @@ -1,4 +1,4 @@ -#!/bin/bash -e +#!/bin/bash -ex if [ -z "${N3DR_APT_GPG_SECRET}" ]; then echo "N3DR_APT_GPG_SECRET should not be empty" @@ -6,7 +6,7 @@ if [ -z "${N3DR_APT_GPG_SECRET}" ]; then echo "docker run -v /tmp/gpg-output:/root/.gnupg -v ${PWD}/test/gpg/:/tmp/ --rm -it vladgh/gpg --batch --generate-key /tmp/generate" echo "docker run --rm -it -v /tmp/gpg-output:/root/.gnupg -v ${PWD}/test/gpg/:/tmp/ vladgh/gpg --output /tmp/my_rsa_key --armor --export-secret-key joe@foo.bar" echo "Enter 'abc' as a password, if the prompt appears" - printf "export N3DR_APT_GPG_SECRET=\$(sudo cat test/gpg/my_rsa_key | tr -d '\\\n')" + printf "export N3DR_APT_GPG_SECRET=\$(sudo cat test/gpg/my_rsa_key | tr '\\\n' ' ' | sed -r \"s|-----[A-Z]+ PGP PRIVATE KEY BLOCK-----||g;s| |\\\\\\\\\\\n|g;s|(.*)|-----BEGIN PGP PRIVATE KEY BLOCK-----\\\1-----END PGP PRIVATE KEY BLOCK-----|g\")" echo echo "sudo rm -r /tmp/gpg-output" echo "rm test/gpg/my_rsa_key"