From c3325c53636e3e6f4014dbd3f7a6696bb416091f Mon Sep 17 00:00:00 2001 From: ben Date: Sun, 29 Mar 2020 04:09:08 +0200 Subject: [PATCH] [GH-113] Download of specific artifacts. --- CHANGELOG.md | 23 ++++++++++++++++++++-- README.md | 32 ++++++++++++++++++++++++------ cli/backup.go | 23 +++++++++++++++------- cli/backup_test.go | 2 +- cli/repositories.go | 4 ++-- cli/upload.go | 2 +- cmd/backup.go | 5 ++++- cmd/repositories.go | 3 ++- integration-tests.sh | 46 +++++++++++++++++++++++++++++++++++--------- 9 files changed, 110 insertions(+), 30 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 20242b06..96e788e6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,24 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] -### Changed +None + +## [3.5.0] - 2020-03-29 +### Added +- Download of specific artifacts using a regex +- Instructions added to README how to use this new feature + +### Improved +- Section about how to clone an old Nexus3 repository + +### Fixed +- Three code smells that were reported by SonarCloud + +## [3.4.0] - 2020-03-25 +### Added +- Upload of zip artifacts + +## Changed - Create files for testing rather than storing them in git ## [3.3.3] - 2019-12-12 @@ -142,7 +159,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Download all artifacts from a certain Nexus3 repository. -[Unreleased]: https://github.com/030/n3dr/compare/3.3.3...HEAD +[Unreleased]: https://github.com/030/n3dr/compare/3.5.0...HEAD +[3.5.0]: https://github.com/030/n3dr/compare/3.4.0...3.5.0 +[3.4.0]: https://github.com/030/n3dr/compare/3.3.3...3.4.0 [3.3.3]: https://github.com/030/n3dr/compare/3.3.2...3.3.3 [3.3.2]: https://github.com/030/n3dr/compare/3.3.1...3.3.2 [3.3.1]: https://github.com/030/n3dr/compare/3.3.0...3.3.1 diff --git a/README.md b/README.md index 4b9e7cfe..ec70b2e4 100644 --- a/README.md +++ b/README.md @@ -121,6 +121,23 @@ Note: a new folder will be created for every repository: * download/maven-public * download/maven-releases +## Backup only certain artifacts + +It is possible to only download artifacts that match a regular expression. If +one would like to download all artifacts from 'some/group42' then one could do +that as follows: + +``` +./n3dr-linux backup -u admin -n http://localhost:8081 -r maven-releases -x 'some/group42' +``` + +If one would like to deploy is while download from all repositories then use +the `-x` option as well: + +``` +./n3dr-linux repositories -u admin -n http://localhost:8081 -b -x 'some/group42' +``` + ## Add all downloaded archives to a ZIP archive In order to add all archives to a zip archive, one has to use the --zip or -z flag. @@ -134,16 +151,19 @@ running the following command: ./n3dr-linux upload -u admin -n http://localhost:8081 -r maven-public ``` -## "Clone" the content of a repository in a different Nexus 3 server in a different repository +## "Clone" a Nexus3 repository -These are the basic steps to "clone" and eventually rename the content of a -repository from one nexus3 server to another one: +Suppose that one has created a new Nexus3 repository, e.g. NexusNEW and that +one would like to copy the content of the old repository, e.g. NexusOLD, then +these basic steps could be issued to "clone" NexusOLD: ``` -n3dr backup -u -n -r +n3dr backup -u -n \ +-r cd download -mv -n3dr upload -u -n -r +mv +n3dr upload -u -n \ +-r ``` ## Rationale for N3DR diff --git a/cli/backup.go b/cli/backup.go index af9eb2f3..c6c79b52 100644 --- a/cli/backup.go +++ b/cli/backup.go @@ -100,6 +100,7 @@ func (n Nexus3) continuationTokenRecursion(t string) ([]string, error) { } func createArtifact(d string, f string, content string) error { + log.Debug("Create artifact: '" + d + "/" + f + "'") err := os.MkdirAll(d, os.ModePerm) if err != nil { return err @@ -177,6 +178,7 @@ func (n Nexus3) downloadURLs() ([]interface{}, error) { jq := gojsonq.New().JSONString(json) downloadURLsInterface := jq.From("items").Pluck("downloadUrl") + log.Debug("DownloadURLs: " + fmt.Sprintf("%v", downloadURLsInterface)) downloadURLsInterfaceArray := downloadURLsInterface.([]interface{}) downloadURLsInterfaceArrayAll = append(downloadURLsInterfaceArrayAll, downloadURLsInterfaceArray...) @@ -189,7 +191,7 @@ func (n Nexus3) downloadURLs() ([]interface{}, error) { } // StoreArtifactsOnDisk downloads all artifacts from nexus and saves them on disk -func (n Nexus3) StoreArtifactsOnDisk() error { +func (n Nexus3) StoreArtifactsOnDisk(regex string) error { urls, err := n.downloadURLs() if err != nil { return err @@ -200,13 +202,20 @@ func (n Nexus3) StoreArtifactsOnDisk() error { log.Info("Backing up artifacts '" + n.Repository + "'") bar := pb.StartNew(len(urls)) for _, downloadURL := range urls { - u := fmt.Sprint(downloadURL) + url := fmt.Sprint(downloadURL) - // Exclude download of md5 and sha1 files as these are unavailable - // unless the metadata.xml is opened first - if !(filepath.Ext(u) == ".md5" || filepath.Ext(u) == ".sha1") { - if err := n.downloadArtifact(fmt.Sprint(downloadURL)); err != nil { - return err + log.Debug("Only download artifacts that match the regex: '" + regex + "'") + r, err := regexp.Compile(regex) + if err != nil { + return err + } + if r.MatchString(url) { + // Exclude download of md5 and sha1 files as these are unavailable + // unless the metadata.xml is opened first + if !(filepath.Ext(url) == ".md5" || filepath.Ext(url) == ".sha1") { + if err := n.downloadArtifact(fmt.Sprint(downloadURL)); err != nil { + return err + } } } diff --git a/cli/backup_test.go b/cli/backup_test.go index 4e765ad5..4c5a60c1 100644 --- a/cli/backup_test.go +++ b/cli/backup_test.go @@ -49,7 +49,7 @@ func TestDownloadURLs(t *testing.T) { } func TestStoreArtifactsOnDisk(t *testing.T) { - if err := n.StoreArtifactsOnDisk(); err != nil { + if err := n.StoreArtifactsOnDisk(".*"); err != nil { log.Fatal(err) } diff --git a/cli/repositories.go b/cli/repositories.go index 0b453b64..2de87aa6 100644 --- a/cli/repositories.go +++ b/cli/repositories.go @@ -43,7 +43,7 @@ func (n Nexus3) CountRepositories() error { } // Downloads retrieves artifacts from all repositories -func (n Nexus3) Downloads() error { +func (n Nexus3) Downloads(regex string) error { repos, err := n.repositoriesSlice() if err != nil { return err @@ -51,7 +51,7 @@ func (n Nexus3) Downloads() error { for _, name := range repos { n := Nexus3{URL: n.URL, User: n.User, Pass: n.Pass, Repository: name.(string), APIVersion: n.APIVersion, ZIP: n.ZIP} - if err := n.StoreArtifactsOnDisk(); err != nil { + if err := n.StoreArtifactsOnDisk(regex); err != nil { return err } } diff --git a/cli/upload.go b/cli/upload.go index 516d24c6..51a80d71 100644 --- a/cli/upload.go +++ b/cli/upload.go @@ -99,7 +99,7 @@ func (n Nexus3) Upload() error { return err } for i, path := range foldersWithPOMStringSlice { - log.Info(strconv.Itoa(i) + " Detecting artifacts in folder '" + path + "'") + log.Debug(strconv.Itoa(i) + " Detecting artifacts in folder '" + path + "'") sb, err := pomDirs(path) if err != nil { return err diff --git a/cmd/backup.go b/cmd/backup.go index f47593fa..552b5c4f 100644 --- a/cmd/backup.go +++ b/cmd/backup.go @@ -22,6 +22,8 @@ import ( "github.com/spf13/viper" ) +var regex string + // backupCmd represents the backup command var backupCmd = &cobra.Command{ Use: "backup", @@ -36,7 +38,7 @@ reside in a certain Nexus3 repository`, }, Run: func(cmd *cobra.Command, args []string) { n := cli.Nexus3{URL: n3drURL, User: n3drUser, Pass: viper.GetString("n3drPass"), Repository: n3drRepo, APIVersion: apiVersion, ZIP: zip} - if err := n.StoreArtifactsOnDisk(); err != nil { + if err := n.StoreArtifactsOnDisk(regex); err != nil { log.Fatal(err) } @@ -55,4 +57,5 @@ func init() { } rootCmd.AddCommand(backupCmd) + backupCmd.Flags().StringVarP(®ex, "regex", "x", ".*", "only download artifacts that match a regular expression, e.g. 'some/group42'") } diff --git a/cmd/repositories.go b/cmd/repositories.go index 52b3f993..39a27aa9 100644 --- a/cmd/repositories.go +++ b/cmd/repositories.go @@ -61,7 +61,7 @@ download artifacts from all repositories`, } } if backup { - err := n.Downloads() + err := n.Downloads(regex) if err != nil { log.Fatal(err) } @@ -75,4 +75,5 @@ func init() { repositoriesCmd.Flags().BoolVarP(&names, "names", "a", false, "print all repository names") repositoriesCmd.Flags().BoolVarP(&count, "count", "c", false, "count the number of repositories") repositoriesCmd.Flags().BoolVarP(&backup, "backup", "b", false, "backup artifacts from all repositories") + repositoriesCmd.Flags().StringVarP(®ex, "regex", "x", ".*", "only download artifacts that match a regular expression, e.g. 'some/group42'") } diff --git a/integration-tests.sh b/integration-tests.sh index 2dec7a00..b3199a6b 100755 --- a/integration-tests.sh +++ b/integration-tests.sh @@ -35,7 +35,7 @@ readiness(){ until docker logs nexus | grep 'Started Sonatype Nexus OSS' do echo "Nexus unavailable" - sleep 10 + sleep 3 done } @@ -52,10 +52,10 @@ password(){ } artifact(){ - mkdir -p "maven-releases/file${1}/file${1}/1.0.0" - echo someContent > "maven-releases/file${1}/file${1}/1.0.0/file${1}-1.0.0.jar" - echo someContentZIP > "maven-releases/file${1}/file${1}/1.0.0/file${1}-1.0.0.zip" - echo -e "\n4.0.0\nfile${1}\nfile${1}\n1.0.0\n" > "maven-releases/file${1}/file${1}/1.0.0/file${1}-1.0.0.pom" + mkdir -p "maven-releases/some/group${1}/file${1}/1.0.0" + echo someContent > "maven-releases/some/group${1}/file${1}/1.0.0/file${1}-1.0.0.jar" + echo someContentZIP > "maven-releases/some/group${1}/file${1}/1.0.0/file${1}-1.0.0.zip" + echo -e "\n4.0.0\nsome.group${1}\nfile${1}\n1.0.0\n" > "maven-releases/some/group${1}/file${1}/1.0.0/file${1}-1.0.0.pom" } files(){ @@ -75,15 +75,42 @@ backup(){ if [ "${NEXUS_VERSION}" == "3.9.0" ]; then count_downloads 300 - test_zip 132 + test_zip 140 else count_downloads 400 - test_zip 168 + test_zip 180 fi cleanup_downloads } +regex(){ + echo "Testing backup regex..." + $TOOL backup -n http://localhost:9999 -u admin -p $PASSWORD -r maven-releases -v "${NEXUS_API_VERSION}" -x 'some/group42' + $TOOL backup -n http://localhost:9999 -u admin -p $PASSWORD -r maven-releases -v "${NEXUS_API_VERSION}" -x 'some/group42' -z + if [ "${NEXUS_VERSION}" == "3.9.0" ]; then + count_downloads 3 + test_zip 4 + else + count_downloads 4 + test_zip 4 + fi + cleanup_downloads + + + echo -e "\nTesting repositories regex..." + $TOOL repositories -n http://localhost:9999 -u admin -p $PASSWORD -v "${NEXUS_API_VERSION}" -b -x 'some/group42' + $TOOL repositories -n http://localhost:9999 -u admin -p $PASSWORD -v "${NEXUS_API_VERSION}" -b -x 'some/group42' -z + if [ "${NEXUS_VERSION}" == "3.9.0" ]; then + count_downloads 6 + test_zip 4 + else + count_downloads 8 + test_zip 8 + fi + cleanup_downloads +} + repositories(){ local cmd="$TOOL repositories -n http://localhost:9999 -u admin -p $PASSWORD -v ${NEXUS_API_VERSION}" @@ -95,10 +122,10 @@ repositories(){ if [ "${NEXUS_VERSION}" == "3.9.0" ]; then count_downloads 600 - test_zip 256 + test_zip 272 else count_downloads 800 - test_zip 336 + test_zip 356 fi cleanup_downloads @@ -141,6 +168,7 @@ main(){ upload backup repositories + regex bats --tap tests.bats }