From 571b146821f87a45c59ffcc92413950bcd89faaf Mon Sep 17 00:00:00 2001 From: ben Date: Sun, 16 Jan 2022 01:13:35 +0100 Subject: [PATCH] [GH-238] Upload yum artifacts. --- README.md | 12 ++-- build/package/snap/snapcraft.yaml | 2 +- build/packer/windows2016.json.pkr.hcl | 4 +- cmd/n3dr/configRepository.go | 8 ++- docs/CHANGELOG.md | 21 ++++++- examples/README.md | 4 +- examples/configRepository/README.md | 15 +++++ internal/artifactsv2/download.go | 4 +- internal/artifactsv2/upload/upload.go | 27 +++++---- internal/config/repository/repository.go | 33 +++++++++++ internal/pkg/artifactsv2/artifacts/file.go | 12 ++++ test/integration-tests.sh | 68 ++++++++++++++++++---- 12 files changed, 171 insertions(+), 39 deletions(-) create mode 100644 examples/configRepository/README.md diff --git a/README.md b/README.md index 1930f8e7..966a29ce 100644 --- a/README.md +++ b/README.md @@ -314,7 +314,7 @@ n3dr config \ ### Build ```bash -docker build -t utrecht/n3dr:6.4.3 . +docker build -t utrecht/n3dr:6.5.0 . ``` [![dockeri.co](https://dockeri.co/image/utrecht/n3dr)](https://hub.docker.com/r/utrecht/n3dr) @@ -324,7 +324,7 @@ docker build -t utrecht/n3dr:6.4.3 . ```bash docker run -it \ -v /home/${USER}/.n3dr:/root/.n3dr \ - -v /tmp/n3dr:/tmp/n3dr utrecht/n3dr:6.4.3 + -v /tmp/n3dr:/tmp/n3dr utrecht/n3dr:6.5.0 ``` ### Upload @@ -333,7 +333,7 @@ docker run -it \ docker run -it \ --entrypoint=/bin/ash \ -v /home/${USER}/.n3dr:/root/.n3dr \ - -v /tmp/n3dr:/tmp/n3dr utrecht/n3dr:6.4.3 + -v /tmp/n3dr:/tmp/n3dr utrecht/n3dr:6.5.0 ``` navigate to the repository folder, e.g. `/tmp/n3dr/download*/` and upload: @@ -545,7 +545,7 @@ a single command. | r | | | | | raw | x | x | `%` | | rubygems | | | | -| yum | | | | +| yum | x | x | `#` | | unknown | x | x | `?` | #### backup @@ -636,8 +636,8 @@ go test internal/artifacts/common.go internal/artifacts/common_test.go #### Packer ```bash -packer init packer/windows2016.json.pkr.hcl -PACKER_LOG=1 packer build packer/windows2016.json.pkr.hcl +packer init build/packer/windows2016.json.pkr.hcl +PACKER_LOG=1 packer build build/packer/windows2016.json.pkr.hcl ``` #### Vagrant diff --git a/build/package/snap/snapcraft.yaml b/build/package/snap/snapcraft.yaml index 6fe6fe02..0845f2a9 100644 --- a/build/package/snap/snapcraft.yaml +++ b/build/package/snap/snapcraft.yaml @@ -1,7 +1,7 @@ --- name: n3dr base: core20 -version: 6.4.3 +version: 6.5.0 summary: Nexus3 Disaster Recovery description: | Download all artifacts at once or migrate automatically from Nexus to Nexus. diff --git a/build/packer/windows2016.json.pkr.hcl b/build/packer/windows2016.json.pkr.hcl index da6be9fd..caf09010 100644 --- a/build/packer/windows2016.json.pkr.hcl +++ b/build/packer/windows2016.json.pkr.hcl @@ -25,7 +25,7 @@ variable "iso_url" { source "virtualbox-iso" "autogenerated_1" { communicator = "winrm" disk_size = 61440 - floppy_files = ["packer/Autounattend.xml", "packer/scripts/winrmConfig.bat"] + floppy_files = ["build/packer/Autounattend.xml", "build/packer/scripts/winrmConfig.bat"] guest_additions_mode = "attach" guest_os_type = "Windows2016_64" headless = "${var.headless}" @@ -47,7 +47,7 @@ build { provisioner "windows-update" {} provisioner "powershell" { - scripts = ["packer/scripts/PowershellScript.ps1"] + scripts = ["build/packer/scripts/PowershellScript.ps1"] } post-processor "vagrant" { diff --git a/cmd/n3dr/configRepository.go b/cmd/n3dr/configRepository.go index aac63ba4..31f541e6 100644 --- a/cmd/n3dr/configRepository.go +++ b/cmd/n3dr/configRepository.go @@ -60,10 +60,16 @@ var configRepositoryCmd = &cobra.Command{ } } case "yum": - if configRepoRecipe == "proxy" { + if configRepoRecipe == "hosted" { + if err := r.CreateYumHosted(configRepoName); err != nil { + log.Fatal(err) + } + } else if configRepoRecipe == "proxy" { if err := r.CreateYumProxied(configRepoName); err != nil { log.Fatal(err) } + } else { + log.Fatalf("configRepoRecipe: '%s' not supported in conjunction with configRepoType: '%s'", configRepoRecipe, configRepoType) } default: log.Fatalf("configRepoType should not be empty, but: 'apt' or 'raw' and not: '%s'. Did you populate the --configRepoType parameter?", configRepoType) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 8be119c8..5a1c6cdb 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -7,10 +7,28 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [6.5.0] - 2021-01-16 + +### Added + +- create hosted yum repositories. +- example how to create a hosted yum repository. +- upload RPMs into yum repository. + ### Changed - master renamed to main branch. +### Fixed + +- return error if upload of an artifact fails to ensure that one is aware if + some file cannot be uploaded and that files are not uploaded if they already + exist in a Nexus3 repository. + +### Removed + +- some duplicated code by using `FilesToBeSkipped` and `PrintType` functions. + ## [6.4.3] - 2021-01-10 ### Added @@ -636,7 +654,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Download all artifacts from a certain Nexus3 repository. -[Unreleased]: https://github.com/030/n3dr/compare/6.4.3...HEAD +[Unreleased]: https://github.com/030/n3dr/compare/6.5.0...HEAD +[6.5.0]: https://github.com/030/n3dr/compare/6.4.3...6.5.0 [6.4.3]: https://github.com/030/n3dr/compare/6.4.2...6.4.3 [6.4.2]: https://github.com/030/n3dr/compare/6.4.1...6.4.2 [6.4.1]: https://github.com/030/n3dr/compare/6.4.0...6.4.1 diff --git a/examples/README.md b/examples/README.md index 6d9ccbe1..85d8e7df 100644 --- a/examples/README.md +++ b/examples/README.md @@ -2,5 +2,5 @@ ## repositoriesV2 -Use the [basePathPrefix](./repositoriesV2/BASE_PATH_PREFIX.md) subcommand if Nexus3 is running -behind a proxy. +Use the [basePathPrefix](./repositoriesV2/BASE_PATH_PREFIX.md) subcommand if +Nexus3 is running behind a proxy. diff --git a/examples/configRepository/README.md b/examples/configRepository/README.md new file mode 100644 index 00000000..6c7b901a --- /dev/null +++ b/examples/configRepository/README.md @@ -0,0 +1,15 @@ +# configRepository + +## Create a repository + +### Hosted yum + +```bash +n3dr configRepository \ + --configRepoName some-repo \ + --configRepoType yum \ + -p \ + -u \ + -n=: \ + --https=false +``` diff --git a/internal/artifactsv2/download.go b/internal/artifactsv2/download.go index ebfd4899..b3c5757c 100644 --- a/internal/artifactsv2/download.go +++ b/internal/artifactsv2/download.go @@ -6,7 +6,6 @@ import ( "net/http" "os" "path/filepath" - "regexp" "sync" "time" @@ -120,12 +119,11 @@ func (n *Nexus3) downloadIfChecksumMismatchLocalFile(continuationToken, repo str log.Debugf("downloadURL: '%s', format: '%s', checksum: '%s'", asset.DownloadURL, asset.Format, checksum) assetPath := asset.Path - filesToBeSkipped, err := regexp.MatchString(`^\.(sha(1|256|512)|md5)$`, filepath.Ext(assetPath)) + filesToBeSkipped, err := artifacts.FilesToBeSkipped(assetPath) if err != nil { errs = append(errs, err) return } - log.Debugf("file: %s, filesToBeSkipped: %v", assetPath, filesToBeSkipped) if !filesToBeSkipped { file := filepath.Join(n.DownloadDirName, repo, assetPath) downloadedFileChecksum, checksumLocalFileErrs := artifacts.ChecksumLocalFile(file, shaType) diff --git a/internal/artifactsv2/upload/upload.go b/internal/artifactsv2/upload/upload.go index 60a48b53..0e9b183a 100644 --- a/internal/artifactsv2/upload/upload.go +++ b/internal/artifactsv2/upload/upload.go @@ -182,9 +182,9 @@ func UploadSingleArtifact(client *client.Nexus3, path, localDiskRepo, localDiskR var f, f2, f3 *os.File c := components.UploadComponentParams{} + artifacts.PrintType(repoFormat) switch rf := repoFormat; rf { case "apt": - fmt.Print("^") c.Repository = localDiskRepo f, err := os.Open(filepath.Clean(path)) if err != nil { @@ -192,7 +192,6 @@ func UploadSingleArtifact(client *client.Nexus3, path, localDiskRepo, localDiskR } c.AptAsset = f case "maven2": - fmt.Print("+") if filepath.Ext(path) == ".pom" { c.Repository = localDiskRepo f, err := os.Open(filepath.Clean(path)) @@ -237,7 +236,6 @@ func UploadSingleArtifact(client *client.Nexus3, path, localDiskRepo, localDiskR } } case "npm": - fmt.Print("*") c.Repository = localDiskRepo f, err := os.Open(filepath.Clean(path)) if err != nil { @@ -245,7 +243,6 @@ func UploadSingleArtifact(client *client.Nexus3, path, localDiskRepo, localDiskR } c.NpmAsset = f case "nuget": - fmt.Print("$") c.Repository = localDiskRepo f, err := os.Open(filepath.Clean(path)) if err != nil { @@ -253,7 +250,6 @@ func UploadSingleArtifact(client *client.Nexus3, path, localDiskRepo, localDiskR } c.NugetAsset = f case "raw": - fmt.Print("%") c.Repository = localDiskRepo f, err := os.Open(filepath.Clean(path)) if err != nil { @@ -262,8 +258,15 @@ func UploadSingleArtifact(client *client.Nexus3, path, localDiskRepo, localDiskR c.RawAsset1 = f c.RawDirectory = &dir c.RawAsset1Filename = &filename + case "yum": + c.Repository = localDiskRepo + f, err := os.Open(filepath.Clean(path)) + if err != nil { + return err + } + c.YumAsset = f + c.YumAssetFilename = &filename default: - fmt.Print("?") return nil } @@ -278,11 +281,7 @@ func UploadSingleArtifact(client *client.Nexus3, path, localDiskRepo, localDiskR return uploadStatusErr } if statusCode == 204 { - log.Infof("artifact: '%v' has been uploaded", path) - return nil - } - if statusCode == 400 { - log.Debugf("artifact: '%v' has already been uploaded, perhaps 'redeploy' is disabled?", path) + log.Debugf("artifact: '%v' has been uploaded", path) return nil } @@ -308,7 +307,11 @@ func (n *Nexus3) ReadLocalDirAndUploadArtifacts(localDiskRepoHome, localDiskRepo return err } - if !info.IsDir() && !(filepath.Ext(path) == ".sha512") && !(filepath.Ext(path) == ".sha256") && !(filepath.Ext(path) == ".sha1") && !(filepath.Ext(path) == ".md5") { + filesToBeSkipped, err := artifacts.FilesToBeSkipped(filepath.Ext(path)) + if err != nil { + return err + } + if !info.IsDir() && !filesToBeSkipped { wg.Add(1) go func() { defer wg.Done() diff --git a/internal/config/repository/repository.go b/internal/config/repository/repository.go index f5b7991d..49fab871 100644 --- a/internal/config/repository/repository.go +++ b/internal/config/repository/repository.go @@ -126,6 +126,39 @@ func (r *Repository) CreateRawHosted(name string) error { return nil } +func (r *Repository) CreateYumHosted(name string) error { + log.Infof("Creating yum hosted repository: '%s'...", name) + client := r.Nexus3.Client() + if name == "" { + return fmt.Errorf("repo name should not be empty") + } + + online := true + strictContentTypeValidation := true + writePolicy := "allow_once" + mhsa := models.HostedStorageAttributes{BlobStoreName: "default", StrictContentTypeValidation: &strictContentTypeValidation, WritePolicy: &writePolicy} + + var repoDataDepth int32 = 0 + yum := models.YumAttributes{DeployPolicy: models.YumAttributesDeployPolicySTRICT, RepodataDepth: &repoDataDepth} + mr := models.YumHostedRepositoryAPIRequest{Name: &name, Online: &online, Storage: &mhsa, Yum: &yum} + createYumHosted := repository_management.CreateRepository21Params{Body: &mr} + createYumHosted.WithTimeout(time.Second * 30) + if _, err := client.RepositoryManagement.CreateRepository21(&createYumHosted); err != nil { + repositoryCreated, errRegex := regexp.MatchString("status 400", err.Error()) + if errRegex != nil { + return err + } + if repositoryCreated { + log.Infof("repository: '%s' has already been created", name) + return nil + } + + return fmt.Errorf("could not create repository: '%v', err: '%v'", name, err) + } + log.Infof("created the following repository: '%v'", name) + return nil +} + func (r *Repository) Delete(name string) error { log.Infof("Deleting repository: '%s'...", name) client := r.Nexus3.Client() diff --git a/internal/pkg/artifactsv2/artifacts/file.go b/internal/pkg/artifactsv2/artifacts/file.go index 45521269..bc917498 100644 --- a/internal/pkg/artifactsv2/artifacts/file.go +++ b/internal/pkg/artifactsv2/artifacts/file.go @@ -8,6 +8,7 @@ import ( "io" "os" "path/filepath" + "regexp" "github.com/030/n3dr/internal/goswagger/models" log "github.com/sirupsen/logrus" @@ -53,6 +54,8 @@ func PrintType(assetFormat string) { fmt.Print("$") case "raw": fmt.Print("%") + case "yum": + fmt.Print("#") default: fmt.Print("?") log.Debugf("Unknown type: '%s'", af) @@ -72,3 +75,12 @@ func Checksum(asset *models.AssetXO) (string, string) { } return shaType, checksum } + +func FilesToBeSkipped(path string) (bool, error) { + filesToBeSkipped, err := regexp.MatchString(`^\.(sha(1|256|512)|md5)$`, filepath.Ext(path)) + if err != nil { + return false, err + } + log.Debugf("file: %s, filesToBeSkipped: %v", path, filesToBeSkipped) + return filesToBeSkipped, nil +} diff --git a/test/integration-tests.sh b/test/integration-tests.sh index 97fdf946..61be0a3f 100755 --- a/test/integration-tests.sh +++ b/test/integration-tests.sh @@ -32,11 +32,13 @@ fi readonly DOWNLOAD_LOCATION=/tmp/n3dr readonly DOWNLOAD_LOCATION_PASS=${DOWNLOAD_LOCATION}-pass +readonly DOWNLOAD_LOCATION_RPROXY=${DOWNLOAD_LOCATION}-rproxy readonly DOWNLOAD_LOCATION_SYNC=${DOWNLOAD_LOCATION}-sync readonly DOWNLOAD_LOCATION_SYNC_A=${DOWNLOAD_LOCATION_SYNC}-a readonly DOWNLOAD_LOCATION_SYNC_B=${DOWNLOAD_LOCATION_SYNC}-b readonly DOWNLOAD_LOCATION_SYNC_C=${DOWNLOAD_LOCATION_SYNC}-c -readonly DOWNLOAD_LOCATION_SYNC_SIZE=23M +readonly DOWNLOAD_LOCATION_SYNC_SIZE=151452 +readonly HOSTED_REPO_YUM=REPO_NAME_HOSTED_YUM readonly PORT_NEXUS_A=9999 readonly PORT_NEXUS_B=9998 readonly PORT_NEXUS_C=9997 @@ -103,8 +105,7 @@ files() { upload() { echo "#134 archetype-catalog download issue" - echo "URL:" - echo "${URL_NEXUS_A}/repository/maven-releases/archetype-catalog.xml" + echo "URL: '${URL_NEXUS_A}/repository/maven-releases/archetype-catalog.xml'" echo "does not seem to contain a Maven artifact" curl -f ${URL_NEXUS_A}/repository/maven-releases/archetype-catalog.xml @@ -161,6 +162,17 @@ createHostedNPM() { -d "{\"name\":\"REPO_NAME_HOSTED_NPM\",\"online\":true,\"storage\":{\"blobStoreName\":\"default\",\"strictContentTypeValidation\":true,\"writePolicy\":\"ALLOW_ONCE\"}}" } +createHostedYum() { + echo "Creating hosted yum repository..." + ./"${N3DR_DELIVERABLE}" configRepository \ + --configRepoName ${HOSTED_REPO_YUM} \ + --configRepoType yum \ + -p "${1}" \ + -n "${2}" \ + -u admin \ + --https=false +} + uploadNPM() { if [ "${NEXUS_API_VERSION}" != "beta" ]; then createHostedNPM "${PASSWORD_NEXUS_A}" "${URL_NEXUS_A}" @@ -197,6 +209,38 @@ uploadNuget() { fi } +uploadYumArtifact() { + curl -X 'POST' \ + ${URL_NEXUS_A}/service/rest/v1/components?repository=${HOSTED_REPO_YUM} \ + -s \ + -f \ + -u "admin:${PASSWORD_NEXUS_A}" \ + -H 'accept: application/json' \ + -H 'Content-Type: multipart/form-data' \ + -F "yum.asset=@${HOSTED_REPO_YUM}/${1};type=application/x-rpm" \ + -F "yum.asset.filename=${1}" +} + +uploadYum() { + if [ "${NEXUS_API_VERSION}" != "beta" ]; then + createHostedYum "${PASSWORD_NEXUS_A}" "${URL_NEXUS_A_V2}" + + mkdir ${HOSTED_REPO_YUM} + cd ${HOSTED_REPO_YUM} + curl -sL https://yum.puppet.com/puppet7-release-el-8.noarch.rpm \ + -o puppet.rpm + curl -sL https://packages.chef.io/files/stable/chef-workstation/21.10.640/el/8/chef-workstation-21.10.640-1.el8.x86_64.rpm \ + -o chef.rpm + cd .. + + echo "Upload yum artifacts..." + for a in chef puppet; do uploadYumArtifact "${a}.rpm"; done + echo + else + echo "yum upload not supported in beta API" + fi +} + backupHelper() { if [ "${NEXUS_VERSION}" == "3.9.0" ]; then count_downloads 300 @@ -250,7 +294,7 @@ repositories() { $cmd -a | grep maven-releases echo "> Counting number of repositories..." - expected_number=7 + expected_number=8 if [ "${NEXUS_API_VERSION}" == "beta" ]; then expected_number=5 fi @@ -259,7 +303,7 @@ repositories() { [ "${actual_number}" == "${expected_number}" ] echo "> Testing zip functionality..." - testZipSizeDir=/tmp/n3dr/testZipSize/ + testZipSizeDir=${DOWNLOAD_LOCATION}/testZipSize/ $cmd -b -z \ --directory-prefix ${testZipSizeDir} \ --directory-prefix-zip ${testZipSizeDir} @@ -307,9 +351,11 @@ cleanup_downloads() { rm -rf nuget-hosted rm -rf REPO_NAME_HOSTED_APT rm -rf REPO_NAME_HOSTED_NPM + rm -rf ${HOSTED_REPO_YUM} rm -rf maven-releases rm -rf "${DOWNLOAD_LOCATION}" rm -rf "${DOWNLOAD_LOCATION_PASS}" + rm -rf "${DOWNLOAD_LOCATION_RPROXY}" rm -rf "${DOWNLOAD_LOCATION_SYNC}" rm -rf "${DOWNLOAD_LOCATION_SYNC_A}" rm -rf "${DOWNLOAD_LOCATION_SYNC_B}" @@ -324,10 +370,6 @@ version() { echo } -cac() { - echo "Configuration as code" -} - test_sync() { ./"${N3DR_DELIVERABLE}" repositoriesV2 \ --backup \ @@ -336,7 +378,7 @@ test_sync() { -n "${3}" \ -u admin \ --https=false - size=$(du -sh "${1}") + size=$(du -s "${1}") echo "Backup size '${1}': '${size}'" echo "${size}" | grep "${DOWNLOAD_LOCATION_SYNC_SIZE}" } @@ -348,6 +390,8 @@ sync() { createHostedAPT "${PASSWORD_NEXUS_C}" "${URL_NEXUS_C}" createHostedNPM "${PASSWORD_NEXUS_B}" "${URL_NEXUS_B}" createHostedNPM "${PASSWORD_NEXUS_C}" "${URL_NEXUS_C}" + createHostedYum "${PASSWORD_NEXUS_B}" "${URL_NEXUS_B_V2}" + createHostedYum "${PASSWORD_NEXUS_C}" "${URL_NEXUS_C_V2}" ./"${N3DR_DELIVERABLE}" sync \ --otherNexus3Passwords="${PASSWORD_NEXUS_B}","${PASSWORD_NEXUS_C}" \ @@ -370,12 +414,13 @@ rproxy() { if [ "${NEXUS_API_VERSION}" != "beta" ]; then echo "Testing rproxy in front of a nexus server..." ip_nexus_a=$(docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' nexus-a) - mkdir -p /tmp/n3dr-rproxy + mkdir -p ${DOWNLOAD_LOCATION_RPROXY} sed -i "s|WILL_BE_REPLACED|${ip_nexus_a}|" ../../test/rproxy-nginx-nexus3.conf docker run -d --rm --name rproxy-nginx-nexus3 -p 9990:80 -v "${PWD}"/../../test/rproxy-nginx-nexus3.conf:/etc/nginx/nginx.conf nginx:1.21.5-alpine sleep 10 ./"${N3DR_DELIVERABLE}" repositoriesV2 \ --count \ + --directory-prefix "${DOWNLOAD_LOCATION_RPROXY}" \ -p "${PASSWORD_NEXUS_A}" \ -n localhost:9990 \ -u admin \ @@ -399,6 +444,7 @@ main() { uploadDeb uploadNPM uploadNuget + uploadYum repositories regex zipName