Merge pull request #240 from 030/238-upload-yum

[GH-238] Upload yum artifacts.

README.md

@@ -314,7 +314,7 @@ n3dr config \
 ### Build
 
 ```bash
-docker build -t utrecht/n3dr:6.4.3 .
+docker build -t utrecht/n3dr:6.5.0 .
 ```
 
 [![dockeri.co](https://dockeri.co/image/utrecht/n3dr)](https://hub.docker.com/r/utrecht/n3dr)
@@ -324,7 +324,7 @@ docker build -t utrecht/n3dr:6.4.3 .
 ```bash
 docker run -it \
   -v /home/${USER}/.n3dr:/root/.n3dr \
-  -v /tmp/n3dr:/tmp/n3dr utrecht/n3dr:6.4.3
+  -v /tmp/n3dr:/tmp/n3dr utrecht/n3dr:6.5.0
 ```
 
 ### Upload
@@ -333,7 +333,7 @@ docker run -it \
 docker run -it \
   --entrypoint=/bin/ash \
   -v /home/${USER}/.n3dr:/root/.n3dr \
-  -v /tmp/n3dr:/tmp/n3dr utrecht/n3dr:6.4.3
+  -v /tmp/n3dr:/tmp/n3dr utrecht/n3dr:6.5.0
 ```
 
 navigate to the repository folder, e.g. `/tmp/n3dr/download*/` and upload:
@@ -545,7 +545,7 @@ a single command.
 | r         |        |        |       |
 | raw       | x      | x      | `%`   |
 | rubygems  |        |        |       |
-| yum       |        |        |       |
+| yum       | x      | x      | `#`   |
 | unknown   | x      | x      | `?`   |
 
 #### backup
@@ -636,8 +636,8 @@ go test internal/artifacts/common.go internal/artifacts/common_test.go
 #### Packer
 
 ```bash
-packer init packer/windows2016.json.pkr.hcl
-PACKER_LOG=1 packer build packer/windows2016.json.pkr.hcl
+packer init build/packer/windows2016.json.pkr.hcl
+PACKER_LOG=1 packer build build/packer/windows2016.json.pkr.hcl
 ```
 
 #### Vagrant

build/package/snap/snapcraft.yaml

@@ -1,7 +1,7 @@
 ---
 name: n3dr
 base: core20
-version: 6.4.3
+version: 6.5.0
 summary: Nexus3 Disaster Recovery
 description: |
   Download all artifacts at once or migrate automatically from Nexus to Nexus.

build/packer/windows2016.json.pkr.hcl

@@ -25,7 +25,7 @@ variable "iso_url" {
 source "virtualbox-iso" "autogenerated_1" {
   communicator         = "winrm"
   disk_size            = 61440
-  floppy_files         = ["packer/Autounattend.xml", "packer/scripts/winrmConfig.bat"]
+  floppy_files         = ["build/packer/Autounattend.xml", "build/packer/scripts/winrmConfig.bat"]
   guest_additions_mode = "attach"
   guest_os_type        = "Windows2016_64"
   headless             = "${var.headless}"
@@ -47,7 +47,7 @@ build {
   provisioner "windows-update" {}
 
   provisioner "powershell" {
-    scripts          = ["packer/scripts/PowershellScript.ps1"]
+    scripts          = ["build/packer/scripts/PowershellScript.ps1"]
   }
 
   post-processor "vagrant" {

cmd/n3dr/configRepository.go

@@ -60,10 +60,16 @@ var configRepositoryCmd = &cobra.Command{
 				}
 			}
 		case "yum":
-			if configRepoRecipe == "proxy" {
+			if configRepoRecipe == "hosted" {
+				if err := r.CreateYumHosted(configRepoName); err != nil {
+					log.Fatal(err)
+				}
+			} else if configRepoRecipe == "proxy" {
 				if err := r.CreateYumProxied(configRepoName); err != nil {
 					log.Fatal(err)
 				}
+			} else {
+				log.Fatalf("configRepoRecipe: '%s' not supported in conjunction with configRepoType: '%s'", configRepoRecipe, configRepoType)
 			}
 		default:
 			log.Fatalf("configRepoType should not be empty, but: 'apt' or 'raw' and not: '%s'. Did you populate the --configRepoType parameter?", configRepoType)

docs/CHANGELOG.md

@@ -7,10 +7,28 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [6.5.0] - 2021-01-16
+
+### Added
+
+- create hosted yum repositories.
+- example how to create a hosted yum repository.
+- upload RPMs into yum repository.
+
 ### Changed
 
 - master renamed to main branch.
 
+### Fixed
+
+- return error if upload of an artifact fails to ensure that one is aware if
+  some file cannot be uploaded and that files are not uploaded if they already
+  exist in a Nexus3 repository.
+
+### Removed
+
+- some duplicated code by using `FilesToBeSkipped` and `PrintType` functions.
+
 ## [6.4.3] - 2021-01-10
 
 ### Added
@@ -636,7 +654,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - Download all artifacts from a certain Nexus3 repository.
 
-[Unreleased]: https://github.com/030/n3dr/compare/6.4.3...HEAD
+[Unreleased]: https://github.com/030/n3dr/compare/6.5.0...HEAD
+[6.5.0]: https://github.com/030/n3dr/compare/6.4.3...6.5.0
 [6.4.3]: https://github.com/030/n3dr/compare/6.4.2...6.4.3
 [6.4.2]: https://github.com/030/n3dr/compare/6.4.1...6.4.2
 [6.4.1]: https://github.com/030/n3dr/compare/6.4.0...6.4.1

examples/README.md

@@ -2,5 +2,5 @@
 
 ## repositoriesV2
 
-Use the [basePathPrefix](./repositoriesV2/BASE_PATH_PREFIX.md) subcommand if Nexus3 is running
-behind a proxy.
+Use the [basePathPrefix](./repositoriesV2/BASE_PATH_PREFIX.md) subcommand if
+Nexus3 is running behind a proxy.

examples/configRepository/README.md

@@ -0,0 +1,15 @@
+# configRepository
+
+## Create a repository
+
+### Hosted yum
+
+```bash
+n3dr configRepository \
+  --configRepoName some-repo \
+  --configRepoType yum \
+  -p <admin-pass> \
+  -u <admin-user> \
+  -n=<FQDN-without-http://-or-https>:<port-if-applicable> \
+  --https=false
+```

internal/artifactsv2/download.go

@@ -6,7 +6,6 @@ import (
 	"net/http"
 	"os"
 	"path/filepath"
-	"regexp"
 	"sync"
 	"time"
 
@@ -120,12 +119,11 @@ func (n *Nexus3) downloadIfChecksumMismatchLocalFile(continuationToken, repo str
 
 				log.Debugf("downloadURL: '%s', format: '%s', checksum: '%s'", asset.DownloadURL, asset.Format, checksum)
 				assetPath := asset.Path
-				filesToBeSkipped, err := regexp.MatchString(`^\.(sha(1|256|512)|md5)$`, filepath.Ext(assetPath))
+				filesToBeSkipped, err := artifacts.FilesToBeSkipped(assetPath)
 				if err != nil {
 					errs = append(errs, err)
 					return
 				}
-				log.Debugf("file: %s, filesToBeSkipped: %v", assetPath, filesToBeSkipped)
 				if !filesToBeSkipped {
 					file := filepath.Join(n.DownloadDirName, repo, assetPath)
 					downloadedFileChecksum, checksumLocalFileErrs := artifacts.ChecksumLocalFile(file, shaType)

internal/artifactsv2/upload/upload.go

@@ -182,17 +182,16 @@ func UploadSingleArtifact(client *client.Nexus3, path, localDiskRepo, localDiskR
 
 	var f, f2, f3 *os.File
 	c := components.UploadComponentParams{}
+	artifacts.PrintType(repoFormat)
 	switch rf := repoFormat; rf {
 	case "apt":
-		fmt.Print("^")
 		c.Repository = localDiskRepo
 		f, err := os.Open(filepath.Clean(path))
 		if err != nil {
 			return err
 		}
 		c.AptAsset = f
 	case "maven2":
-		fmt.Print("+")
 		if filepath.Ext(path) == ".pom" {
 			c.Repository = localDiskRepo
 			f, err := os.Open(filepath.Clean(path))
@@ -237,23 +236,20 @@ func UploadSingleArtifact(client *client.Nexus3, path, localDiskRepo, localDiskR
 			}
 		}
 	case "npm":
-		fmt.Print("*")
 		c.Repository = localDiskRepo
 		f, err := os.Open(filepath.Clean(path))
 		if err != nil {
 			return err
 		}
 		c.NpmAsset = f
 	case "nuget":
-		fmt.Print("$")
 		c.Repository = localDiskRepo
 		f, err := os.Open(filepath.Clean(path))
 		if err != nil {
 			return err
 		}
 		c.NugetAsset = f
 	case "raw":
-		fmt.Print("%")
 		c.Repository = localDiskRepo
 		f, err := os.Open(filepath.Clean(path))
 		if err != nil {
@@ -262,8 +258,15 @@ func UploadSingleArtifact(client *client.Nexus3, path, localDiskRepo, localDiskR
 		c.RawAsset1 = f
 		c.RawDirectory = &dir
 		c.RawAsset1Filename = &filename
+	case "yum":
+		c.Repository = localDiskRepo
+		f, err := os.Open(filepath.Clean(path))
+		if err != nil {
+			return err
+		}
+		c.YumAsset = f
+		c.YumAssetFilename = &filename
 	default:
-		fmt.Print("?")
 		return nil
 	}
 
@@ -278,11 +281,7 @@ func UploadSingleArtifact(client *client.Nexus3, path, localDiskRepo, localDiskR
 			return uploadStatusErr
 		}
 		if statusCode == 204 {
-			log.Infof("artifact: '%v' has been uploaded", path)
-			return nil
-		}
-		if statusCode == 400 {
-			log.Debugf("artifact: '%v' has already been uploaded, perhaps 'redeploy' is disabled?", path)
+			log.Debugf("artifact: '%v' has been uploaded", path)
 			return nil
 		}
 
@@ -308,7 +307,11 @@ func (n *Nexus3) ReadLocalDirAndUploadArtifacts(localDiskRepoHome, localDiskRepo
 				return err
 			}
 
-			if !info.IsDir() && !(filepath.Ext(path) == ".sha512") && !(filepath.Ext(path) == ".sha256") && !(filepath.Ext(path) == ".sha1") && !(filepath.Ext(path) == ".md5") {
+			filesToBeSkipped, err := artifacts.FilesToBeSkipped(filepath.Ext(path))
+			if err != nil {
+				return err
+			}
+			if !info.IsDir() && !filesToBeSkipped {
 				wg.Add(1)
 				go func() {
 					defer wg.Done()

internal/config/repository/repository.go

@@ -126,6 +126,39 @@ func (r *Repository) CreateRawHosted(name string) error {
 	return nil
 }
 
+func (r *Repository) CreateYumHosted(name string) error {
+	log.Infof("Creating yum hosted repository: '%s'...", name)
+	client := r.Nexus3.Client()
+	if name == "" {
+		return fmt.Errorf("repo name should not be empty")
+	}
+
+	online := true
+	strictContentTypeValidation := true
+	writePolicy := "allow_once"
+	mhsa := models.HostedStorageAttributes{BlobStoreName: "default", StrictContentTypeValidation: &strictContentTypeValidation, WritePolicy: &writePolicy}
+
+	var repoDataDepth int32 = 0
+	yum := models.YumAttributes{DeployPolicy: models.YumAttributesDeployPolicySTRICT, RepodataDepth: &repoDataDepth}
+	mr := models.YumHostedRepositoryAPIRequest{Name: &name, Online: &online, Storage: &mhsa, Yum: &yum}
+	createYumHosted := repository_management.CreateRepository21Params{Body: &mr}
+	createYumHosted.WithTimeout(time.Second * 30)
+	if _, err := client.RepositoryManagement.CreateRepository21(&createYumHosted); err != nil {
+		repositoryCreated, errRegex := regexp.MatchString("status 400", err.Error())
+		if errRegex != nil {
+			return err
+		}
+		if repositoryCreated {
+			log.Infof("repository: '%s' has already been created", name)
+			return nil
+		}
+
+		return fmt.Errorf("could not create repository: '%v', err: '%v'", name, err)
+	}
+	log.Infof("created the following repository: '%v'", name)
+	return nil
+}
+
 func (r *Repository) Delete(name string) error {
 	log.Infof("Deleting repository: '%s'...", name)
 	client := r.Nexus3.Client()

internal/pkg/artifactsv2/artifacts/file.go

@@ -8,6 +8,7 @@ import (
 	"io"
 	"os"
 	"path/filepath"
+	"regexp"
 
 	"github.com/030/n3dr/internal/goswagger/models"
 	log "github.com/sirupsen/logrus"
@@ -53,6 +54,8 @@ func PrintType(assetFormat string) {
 		fmt.Print("$")
 	case "raw":
 		fmt.Print("%")
+	case "yum":
+		fmt.Print("#")
 	default:
 		fmt.Print("?")
 		log.Debugf("Unknown type: '%s'", af)
@@ -72,3 +75,12 @@ func Checksum(asset *models.AssetXO) (string, string) {
 	}
 	return shaType, checksum
 }
+
+func FilesToBeSkipped(path string) (bool, error) {
+	filesToBeSkipped, err := regexp.MatchString(`^\.(sha(1|256|512)|md5)$`, filepath.Ext(path))
+	if err != nil {
+		return false, err
+	}
+	log.Debugf("file: %s, filesToBeSkipped: %v", path, filesToBeSkipped)
+	return filesToBeSkipped, nil
+}