From 0b0f6878311fbda8325e22d6fc2dda3e6cd180cf Mon Sep 17 00:00:00 2001 From: ben Date: Mon, 11 Oct 2021 17:21:59 +0200 Subject: [PATCH] [GH-208] Use Packer and Vagrant in order to test N3DR on Windows. --- .gitignore | 5 +- CHANGELOG.md | 7 +- README.md | 42 ++++++- Vagrantfile | 42 +++++++ build/package/snap/snapcraft.yaml | 2 +- internal/artifacts/upload.go | 12 +- packer/Autounattend.xml | 163 ++++++++++++++++++++++++++++ packer/scripts/PowershellScript.ps1 | 83 ++++++++++++++ packer/scripts/winrmConfig.bat | 13 +++ packer/windows2016.json.pkr.hcl | 56 ++++++++++ vagrant/scripts/n3dr.ps1 | 11 ++ vagrant/scripts/nexus3.sh | 40 +++++++ 12 files changed, 467 insertions(+), 9 deletions(-) create mode 100644 Vagrantfile create mode 100644 packer/Autounattend.xml create mode 100644 packer/scripts/PowershellScript.ps1 create mode 100644 packer/scripts/winrmConfig.bat create mode 100644 packer/windows2016.json.pkr.hcl create mode 100644 vagrant/scripts/n3dr.ps1 create mode 100644 vagrant/scripts/nexus3.sh diff --git a/.gitignore b/.gitignore index 12a4cd73..314d1606 100644 --- a/.gitignore +++ b/.gitignore @@ -17,4 +17,7 @@ vendor .vscode *.zip coverage.txt -*.snap.xdelta3 \ No newline at end of file +*.snap.xdelta3 +.vagrant +virtualbox* +*.exe diff --git a/CHANGELOG.md b/CHANGELOG.md index 47c743b8..e30302e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,9 +7,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [6.0.13] - 2021-10-11 + ### Fixed - Resolve gosec G304, G307, G401 and G501 issues. +- Upload fails from Windows, reported by + [TheMaddinFromTqg](https://github.com/TheMaddinFromTqg). ## [6.0.12] - 2021-10-09 @@ -548,7 +552,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Download all artifacts from a certain Nexus3 repository. -[Unreleased]: https://github.com/030/n3dr/compare/6.0.12...HEAD +[Unreleased]: https://github.com/030/n3dr/compare/6.0.13...HEAD +[6.0.13]: https://github.com/030/n3dr/compare/6.0.12...6.0.13 [6.0.12]: https://github.com/030/n3dr/compare/6.0.11...6.0.12 [6.0.11]: https://github.com/030/n3dr/compare/6.0.10...6.0.11 [6.0.10]: https://github.com/030/n3dr/compare/6.0.9...6.0.10 diff --git a/README.md b/README.md index bce61c1a..f0c317d2 100644 --- a/README.md +++ b/README.md @@ -167,7 +167,7 @@ option. ### Build ```bash -docker build -t utrecht/n3dr:6.0.12 . +docker build -t utrecht/n3dr:6.0.13 . ``` [![dockeri.co](https://dockeri.co/image/utrecht/n3dr)](https://hub.docker.com/r/utrecht/n3dr) @@ -177,7 +177,7 @@ docker build -t utrecht/n3dr:6.0.12 . ```bash docker run -it \ -v /home/${USER}/.n3dr:/root/.n3dr \ - -v /tmp/n3dr:/tmp/n3dr utrecht/n3dr:6.0.12 + -v /tmp/n3dr:/tmp/n3dr utrecht/n3dr:6.0.13 ``` ### Upload @@ -186,7 +186,7 @@ docker run -it \ docker run -it \ --entrypoint=/bin/ash \ -v /home/${USER}/.n3dr:/root/.n3dr \ - -v /tmp/n3dr:/tmp/n3dr utrecht/n3dr:6.0.12 + -v /tmp/n3dr:/tmp/n3dr utrecht/n3dr:6.0.13 ``` navigate to the repository folder, e.g. `/tmp/n3dr/download*/` and upload: @@ -396,3 +396,39 @@ a single command. ```bash go test internal/artifacts/common.go internal/artifacts/common_test.go ``` + +### Integration testing on Windows + +#### Packer + +```bash +packer init packer/windows2016.json.pkr.hcl +PACKER_LOG=1 packer build packer/windows2016.json.pkr.hcl +``` + +#### Vagrant + +```bash +vagrant box add virtualbox_windows2016.box --name win2016/n3dr +vagrant box list +vagrant plugin install vagrant-reload vagrant-windows-update +export VAGRANT_N3DR_NETWORK_ADAPTER=$(vboxmanage list bridgedifs |\ + grep Name: | head -1 | awk '{ print $2 }') +VAGRANT_NEXUS3_IP=192.168.0.42 VAGRANT_N3DR_IP=192.168.0.43 vagrant up +vagrant provision nexus3 +vagrant destroy -f +vagrant ssh nexus3 +``` + +Login as `vagrant` with pass `vagrant` and issue: + +```bash +cd C:\vagrant +.\cmd\n3dr\n3dr.exe download -r maven-releases -n http://192.168.0.42:9999 \ + -u admin -p some-password +.\cmd\n3dr\n3dr.exe upload -r maven-releases -n http://192.168.0.42:9999 \ + -u admin -p some-password +``` + +To check whether it is possible to upload artifacts to Nexus3 from Windows +using N3DR. diff --git a/Vagrantfile b/Vagrantfile new file mode 100644 index 00000000..264c3f7d --- /dev/null +++ b/Vagrantfile @@ -0,0 +1,42 @@ +Vagrant.require_version ">= 2.2.18" + +Vagrant.configure("2") do |config| + config.vm.provider "virtualbox" do |v| + v.gui = true + v.customize ['setextradata', :id, 'GUI/ScaleFactor', '2.0'] + v.customize ["setextradata", "global", "GUI/SuppressMessages", "all" ] + v.customize ["modifyvm", :id, "--clipboard", "bidirectional"] + v.customize ["modifyvm", :id, "--memory", "2048"] + v.customize ["modifyvm", :id, "--cpus", "2"] + end + + config.vm.define "n3dr" do |n3dr| + n3dr.vm.guest = :windows + n3dr.vm.communicator = "winrm" + n3dr.winrm.username = "vagrant" + n3dr.winrm.password = "vagrant" + n3dr.vm.box = "win2016/n3dr" + n3dr.vm.hostname = "n3dr" + n3dr.vm.network "public_network", + ip: ENV['VAGRANT_N3DR_IP'], + bridge: ENV['VAGRANT_N3DR_NETWORK_ADAPTER'] + n3dr.vm.provision "shell", + path: "vagrant/scripts/n3dr.ps1" + n3dr.vm.provision "windows-update", filters: [ + "exclude:$_.Title -like '*Preview*'", + "include:$_.Title -like '*Cumulative Update for *'", + "include:$_.AutoSelectOnWebSites"] + end + + config.vm.define "nexus3" do |nexus3| + nexus3.vm.box = "ubuntu/focal64" + nexus3.vm.hostname = "nexus3" + nexus3.vm.network "public_network", + ip: ENV['VAGRANT_NEXUS3_IP'], + bridge: ENV['VAGRANT_N3DR_NETWORK_ADAPTER'] + nexus3.vm.provision "shell" do |s| + s.path = "vagrant/scripts/nexus3.sh" + s.env = { "N3DR_APT_GPG_SECRET" => ENV['N3DR_APT_GPG_SECRET'] } + end + end +end diff --git a/build/package/snap/snapcraft.yaml b/build/package/snap/snapcraft.yaml index a1a2893f..50388b5e 100644 --- a/build/package/snap/snapcraft.yaml +++ b/build/package/snap/snapcraft.yaml @@ -1,7 +1,7 @@ --- name: n3dr base: core20 -version: 6.0.12 +version: 6.0.13 summary: Nexus3 Disaster Recovery description: | Download all artifacts at once or migrate automatically from Nexus to Nexus. diff --git a/internal/artifacts/upload.go b/internal/artifacts/upload.go index f0dc361e..aebae3ca 100644 --- a/internal/artifacts/upload.go +++ b/internal/artifacts/upload.go @@ -11,6 +11,7 @@ import ( "path" "path/filepath" "regexp" + "runtime" "strconv" "strings" @@ -68,8 +69,13 @@ func sbArtifact(sb *strings.Builder, path, ext, classifier string) error { return nil } -func artifactTypeDetector(sb *strings.Builder, path string, skipErrors bool) error { - var err error +func artifactTypeDetector(sb *strings.Builder, path string, skipErrors bool) (err error) { + regexBase := `^.*\/([\w\-\.]+)\/` + + if runtime.GOOS == "windows" { + log.Info("N3DR is running on Windows. Correcting the regexBase...") + regexBase = `^.*\\([\w\-\.]+)\\` + } regexVersion := `(([a-z\d\-]+)|(([a-z\d\.]+)))` if rv := os.Getenv("N3DR_MAVEN_UPLOAD_REGEX_VERSION"); rv != "" { @@ -81,7 +87,7 @@ func artifactTypeDetector(sb *strings.Builder, path string, skipErrors bool) err regexClassifier = rc } - re := regexp.MustCompile(`^.*\/([\w\-\.]+)\/` + regexVersion + regexClassifier + `\.([a-z]+)$`) + re := regexp.MustCompile(regexBase + regexVersion + regexClassifier + `\.([a-z]+)$`) classifier := "" if re.Match([]byte(path)) { diff --git a/packer/Autounattend.xml b/packer/Autounattend.xml new file mode 100644 index 00000000..17f8e0b1 --- /dev/null +++ b/packer/Autounattend.xml @@ -0,0 +1,163 @@ + + + + + vagrant-win2016 + Microsoft + + W. Europe Standard Time + + + true + + + true + + + false + false + + + + + true + Google + Google + http://www.google.com/search?q={searchTerms} + + + true + true + about:blank + + + false + + + 0 + + + + + true + Remote Desktop + all + + + + + true + + + + + + + + 0 + 1 + + + + /IMAGE/NAME + Windows Server 2016 SERVERSTANDARD + + + OnError + false + + + + true + Vagrant Administrator + Vagrant Inc. + + + + + + 1 + Primary + true + + + + + true + false + NTFS + + C + 1 + 1 + + + 0 + true + + OnError + + + + en-US + en-US + en-US + en-US + + + + + + + + + dgBhAGcAcgBhAG4AdABQAGEAcwBzAHcAbwByAGQA + false</PlainText> + </Password> + <Description>Vagrant User</Description> + <DisplayName>vagrant</DisplayName> + <Group>Administrators</Group> + <Name>vagrant</Name> + </LocalAccount> + </LocalAccounts> + </UserAccounts> + <AutoLogon> + <Password> + <Value>dgBhAGcAcgBhAG4AdABQAGEAcwBzAHcAbwByAGQA</Value> + <PlainText>false</PlainText> + </Password> + <Enabled>true</Enabled> + <Username>vagrant</Username> + </AutoLogon> + <OOBE> + <HideEULAPage>true</HideEULAPage> + <HideLocalAccountScreen>true</HideLocalAccountScreen> + <HideOEMRegistrationScreen>true</HideOEMRegistrationScreen> + <HideOnlineAccountScreens>true</HideOnlineAccountScreens> + <HideWirelessSetupInOOBE>true</HideWirelessSetupInOOBE> + <NetworkLocation>Home</NetworkLocation> + <!-- + ProtectYourPC: + 1 Specifies the recommended level of protection for your computer. + 2 Specifies that only updates are installed. + 3 Specifies that automatic protection is disabled. + --> + <ProtectYourPC>3</ProtectYourPC> + </OOBE> + <FirstLogonCommands> + <SynchronousCommand wcm:action="add"> + <CommandLine>cmd.exe /c a:\winrmConfig.bat</CommandLine> + <Description>Configure WinRM</Description> + <Order>3</Order> + <RequiresUserInput>true</RequiresUserInput> + </SynchronousCommand> + </FirstLogonCommands> + </component> + </settings> + <settings pass="offlineServicing"> + <component name="Microsoft-Windows-LUA-Settings" processorArchitecture="amd64" publicKeyToken="31bf3856ad364e35" language="neutral" versionScope="nonSxS" xmlns:wcm="http://schemas.microsoft.com/WMIConfig/2002/State" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"> + <EnableLUA>false</EnableLUA> + </component> + </settings> + <cpi:offlineImage cpi:source="wim:d:/sources/install.wim#Windows Server 2016 SERVERSTANDARD" xmlns:cpi="urn:schemas-microsoft-com:cpi"/> +</unattend> \ No newline at end of file diff --git a/packer/scripts/PowershellScript.ps1 b/packer/scripts/PowershellScript.ps1 new file mode 100644 index 00000000..4f715423 --- /dev/null +++ b/packer/scripts/PowershellScript.ps1 @@ -0,0 +1,83 @@ +function InstallNuget { + [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 + + # https://answers.microsoft.com/en-us/windows/forum/all/trying-to-install-program-using-powershell-and/4c3ac2b2-ebd4-4b2a-a673-e283827da143 + Set-ItemProperty -Path 'HKLM:\SOFTWARE\Wow6432Node\Microsoft\.NetFramework\v4.0.30319' -Name 'SchUseStrongCrypto' -Value '1' -Type DWord + Set-ItemProperty -Path 'HKLM:\SOFTWARE\Microsoft\.NetFramework\v4.0.30319' -Name 'SchUseStrongCrypto' -Value '1' -Type DWord + + Install-PackageProvider -Name Nuget -Force +} + +function PrivateNetwork { + Install-Module WindowsBox.Network -Force + Set-NetworkToPrivate +} + +function DisableAutoLogon { + Install-Module WindowsBox.AutoLogon -Force + Disable-AutoLogon +} + +function InstallChocolatey { + Write-Output "Installing Chocolatey..." + Set-ExecutionPolicy Bypass -Scope Process -Force; + [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072; + Invoke-Expression ((New-Object System.Net.WebClient).DownloadString('https://community.chocolatey.org/install.ps1')) + Write-Output "Chocolatey installation has been completed" +} + +function InstallChocolateyPackages { + choco install -y golang +} + +function SetVagrantAccount { + Install-Module WindowsBox.VagrantAccount -Force + Set-VagrantAccount +} + +function InstallVMGuestTools { + Install-Module WindowsBox.VMGuestTools -Force + Install-VMGuestTools +} + +function OptimizeDiskUsage { + Install-Module WindowsBox.Compact -Force + Optimize-DiskUsage +} + +function DisableHibernate { + Install-Module WindowsBox.Hibernation -Force + Disable-Hibernation +} + +function EnableRDP { + Install-Module WindowsBox.RDP -Force + Enable-RDP +} + +function DisableUAC { + Install-Module WindowsBox.UAC -Force + Disable-UAC +} + +function ExplorerConfig { + Install-Module WindowsBox.Explorer -Force + Set-ExplorerConfiguration +} + +function Main { + InstallNuget + PrivateNetwork + DisableAutoLogon + DisableUAC + EnableRDP + ExplorerConfig + DisableHibernate + SetVagrantAccount + InstallVMGuestTools + InstallChocolatey + InstallChocolateyPackages + OptimizeDiskUsage +} + +Main diff --git a/packer/scripts/winrmConfig.bat b/packer/scripts/winrmConfig.bat new file mode 100644 index 00000000..7fe32a92 --- /dev/null +++ b/packer/scripts/winrmConfig.bat @@ -0,0 +1,13 @@ +rem basic config for winrm +cmd.exe /c winrm quickconfig -q + +rem allow unencrypted traffic, and configure auth to use basic username/password auth +cmd.exe /c winrm set winrm/config/service @{AllowUnencrypted="true"} +cmd.exe /c winrm set winrm/config/service/auth @{Basic="true"} + +rem update firewall rules to open the right port and to allow remote administration +cmd.exe /c netsh advfirewall firewall set rule group="remote administration" new enable=yes + +rem restart winrm +cmd.exe /c net stop winrm +cmd.exe /c net start winrm diff --git a/packer/windows2016.json.pkr.hcl b/packer/windows2016.json.pkr.hcl new file mode 100644 index 00000000..da6be9fd --- /dev/null +++ b/packer/windows2016.json.pkr.hcl @@ -0,0 +1,56 @@ +packer { + required_plugins { + windows-update = { + version = "0.14.0" + source = "github.com/rgl/windows-update" + } + } +} + +variable "headless" { + type = string + default = "false" +} + +variable "iso_checksum" { + type = string + default = "1ce702a578a3cb1ac3d14873980838590f06d5b7101c5daaccbac9d73f1fb50f" +} + +variable "iso_url" { + type = string + default = "https://software-download.microsoft.com/download/pr/Windows_Server_2016_Datacenter_EVAL_en-us_14393_refresh.ISO" +} + +source "virtualbox-iso" "autogenerated_1" { + communicator = "winrm" + disk_size = 61440 + floppy_files = ["packer/Autounattend.xml", "packer/scripts/winrmConfig.bat"] + guest_additions_mode = "attach" + guest_os_type = "Windows2016_64" + headless = "${var.headless}" + iso_checksum = "sha256:${var.iso_checksum}" + iso_url = "${var.iso_url}" + post_shutdown_delay = "30s" + shutdown_command = "shutdown /s /t 10 /f /d p:4:1 /c \"Packer Shutdown\"" + vboxmanage = [["modifyvm", "{{ .Name }}", "--memory", "2048"], ["modifyvm", "{{ .Name }}", "--cpus", "2"], ["modifyvm", "{{ .Name }}", "--vram", "32"]] + winrm_password = "vagrant" + winrm_timeout = "1h" + winrm_username = "vagrant" +} + +build { + sources = ["source.virtualbox-iso.autogenerated_1"] + + provisioner "windows-update" {} + provisioner "windows-restart" {} + provisioner "windows-update" {} + + provisioner "powershell" { + scripts = ["packer/scripts/PowershellScript.ps1"] + } + + post-processor "vagrant" { + output = "virtualbox_windows2016.box" + } +} diff --git a/vagrant/scripts/n3dr.ps1 b/vagrant/scripts/n3dr.ps1 new file mode 100644 index 00000000..68edd3b1 --- /dev/null +++ b/vagrant/scripts/n3dr.ps1 @@ -0,0 +1,11 @@ +function Build { + Write-Output "Building N3DR..." + Set-Location C:\vagrant\cmd\n3dr + go build +} + +function Main { + Build +} + +Main diff --git a/vagrant/scripts/nexus3.sh b/vagrant/scripts/nexus3.sh new file mode 100644 index 00000000..6eb0394d --- /dev/null +++ b/vagrant/scripts/nexus3.sh @@ -0,0 +1,40 @@ +#!/bin/bash -e + +docker() { + echo "Installing docker..." + apt-get update + apt-get install -y \ + apt-transport-https \ + ca-certificates \ + curl \ + gnupg \ + lsb-release + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor --yes -o /usr/share/keyrings/docker-archive-keyring.gpg + echo \ + "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu \ + $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list >/dev/null + apt-get update + apt-get install -y docker-ce docker-ce-cli containerd.io + sudo systemctl restart docker + sudo docker run hello-world + sudo usermod -aG docker vagrant + whoami +} + +n3dr() { + cd /vagrant + sed -i "s|^\(trap clean EXIT\)|#\1|" test/integration-tests.sh + sed -i "s| upload|echo \$PASSWORD \&\& exit 0 #upload|" test/integration-tests.sh + sed -i "s| build|#build|" test/integration-tests.sh + ./test/integration-tests.sh + cd - + nexus3ip=$(ip a | grep 192 | awk '{ print $2 }' | sed -e "s|\/24||g") + echo "Navigate to ${nexus3ip}:9999 and login with 'admin' and the provided password" +} + +main() { + docker + n3dr +} + +main