From 10147b7a8e3355626e367b3c31f50d48ef36bc38 Mon Sep 17 00:00:00 2001 From: Randy Fay Date: Sat, 8 Jul 2017 10:04:53 -0600 Subject: [PATCH] Windows building and (attempt at) appveyor support (#37) * Adjustments to build and test to make them work on Windows with git bash * Experimental appveyor.yml for windows testing * Put back accidentally committed vendor changes * Use windows pwd syntax instead of linux * Attempt to clean up after govendor fetch * Improve the README to mention windows building * Try out using docker-ce * Experiment: with docker-ce try simple mount * Add docs on how to build using powershell * Add chocolatey and explain about its usage in readme * Add comment about why we can't currently use appveyor successfully --- .appveyor.yml | 50 ++++++++++++ README.md | 32 ++++++++ gitignore.example | 1 + makefile_components/README.md | 1 + makefile_components/base_build_go.mak | 77 ++++++++++--------- .../base_build_python-docker.mak | 2 + makefile_components/base_test_go.mak | 10 +-- tests/Makefile | 11 +-- tests/pkg/clean/build_tools_test.go | 23 +++--- tests/pkg/version/version.go | 2 +- 10 files changed, 151 insertions(+), 58 deletions(-) create mode 100644 .appveyor.yml diff --git a/.appveyor.yml b/.appveyor.yml new file mode 100644 index 0000000..bac9218 --- /dev/null +++ b/.appveyor.yml @@ -0,0 +1,50 @@ +# At this time the appveyor.yml doesn't help us much because we use +# Linux docker containers extensively, and appveyor is unable to use them. +# So even building (not to mention testing) isn't possible. +version: "{build}" +image: Visual Studio 2017 + +# Source Config +clone_folder: c:\gopath\src\github.com\drud\build-tools + +# Build host + +environment: + GOPATH: c:\gopath + GOVERSION: 1.8.3 + BASH: "C:/Program Files/git/bin/bash" + +init: + - git config --global core.autocrlf input + +# Build + +install: + - ps: Stop-Service docker + - ps: Remove-Item -Force -Recurse $env:ProgramFiles\docker + - ps: Invoke-WebRequest -Uri "https://download.docker.com/win/static/test/x86_64/docker-17.06.0-ce.zip" -OutFile "docker.zip" + - ps: Expand-Archive -Path "docker.zip" -DestinationPath $env:ProgramFiles -Force + - ps: Remove-Item docker.zip + - ps: Start-Service docker + - ps: Invoke-WebRequest "https://github.com/docker/compose/releases/download/1.14.0/docker-compose-Windows-x86_64.exe" -UseBasicParsing -OutFile $Env:ProgramFiles\docker\docker-compose.exe + - docker run -it -v C:/gopath:/junk busybox ls + + - docker version + - docker-compose version + # Install the specific Go version. + - rmdir c:\go /s /q + - curl -fsS -o golang.msi https://storage.googleapis.com/golang/go%GOVERSION%.windows-amd64.msi + - msiexec /i golang.msi /q + - go version + - choco install -y git make + - make --version + +build_script: + - cd tests + - '"%BASH%" -c "make windows"' + +deploy: false + +test_script: + - cd tests + - '"%BASH%" -c "make test"' diff --git a/README.md b/README.md index 8777bda..fcb453b 100644 --- a/README.md +++ b/README.md @@ -52,6 +52,38 @@ make VERSION=0.3.0 push make clean ``` +On Windows, using the tools described below, use the command: + +``` +"C:\Program Files\git\bin\bash" -c "make" +"C:\Program Files\git\bin\bash" -c "make test" +"C:\Program Files\git\bin\bash" -c "make test TESTARGS='-run TestSomething'" +"C:\Program Files\git\bin\bash" -c "make gofmt" +``` + +If you're using Powershell instead of cmd, just prepend an `&` on the command, as in: + +``` +&"C:\Program Files\git\bin\bash" -c "make" +``` + +(Note that if you're working with the code, you can just run git bash and do make (and anything else you want) from inside it.) + +## Installed requirements + +You'll need: +* docker-ce (will work with move versions and platforms) +* gnu make +* golang + +On windows the building is somewhat more difficult due to the build being bash/linux/make-oriented, but support is provided. You need: +* [chocolatey](https://chocolatey.org/install) installed +* make for Windows 3.81 (Recommended package [choco install make](https://chocolatey.org/packages/make) on chocolatey.org) +* git for windows (Recommended package [choco install git](https://chocolatey.org/packages/git.install)) +* docker for windows. + +(You can certainly install the base gnu make package, and the traditional git for windows package should work fine. Chocolatey installs are recommended here because there are many, many ways to get mixes of unix-style components that absolutely don't work. Microsoft's lovely bash-for-windows is a great tool, but it's an actual Ubuntu environment so isn't a good place for testing Windows builds.) + ## Golang compiler component golang projects and static analysis functions like gofmt are built in a container from drud/golang-build-container (from https://github.com/drud/golang-build-container). The version of the container is specified in build-tools. diff --git a/gitignore.example b/gitignore.example index 9fc2f05..947a853 100644 --- a/gitignore.example +++ b/gitignore.example @@ -12,3 +12,4 @@ /.dockerfile /VERSION.txt /.docker_image + diff --git a/makefile_components/README.md b/makefile_components/README.md index bc0561a..136ba74 100644 --- a/makefile_components/README.md +++ b/makefile_components/README.md @@ -3,3 +3,4 @@ Please do not change these files. They're intended to be replaceable globally in all repositories as our needs changed. If one of these sections does not meet your needs, consider copying its contents into ../Makefile and commenting out the include and adding a comment about what you did and why. + diff --git a/makefile_components/base_build_go.mak b/makefile_components/base_build_go.mak index 9974d8c..2b2ad4a 100644 --- a/makefile_components/base_build_go.mak +++ b/makefile_components/base_build_go.mak @@ -8,10 +8,12 @@ .PHONY: all build test push clean container-clean bin-clean version static govendor gofmt govet golint GOTMP=.gotmp -SHELL := /bin/bash +SHELL = /bin/bash GOFILES = $(shell find $(SRC_DIRS) -name "*.go") +BUILD_OS = $(shell go env GOHOSTOS) + BUILD_IMAGE ?= drud/golang-build-container:v0.4.2 BUILD_BASE_DIR ?= $$PWD @@ -23,7 +25,7 @@ GOMETALINTER_ARGS ?= --vendored-linters --disable=gocyclo --disable=gotype --dis COMMIT := $(shell git describe --tags --always --dirty) -BUILDINFO = $(shell echo Built $$(date) $$USER@$$(hostname) $(BUILD_IMAGE) ) +BUILDINFO = $(shell echo Built $$(date) $$(whoami)@$$(hostname) $(BUILD_IMAGE) ) VERSION_VARIABLES += VERSION COMMIT BUILDINFO @@ -31,25 +33,30 @@ VERSION_LDFLAGS := $(foreach v,$(VERSION_VARIABLES),-X "$(PKG)/pkg/version.$(v)= LDFLAGS := -extldflags -static $(VERSION_LDFLAGS) +PWD=$(shell pwd) +ifeq ($(BUILD_OS),windows) + TMPPWD=$(shell cmd /C echo %cd%) + PWD=$(shell echo "$(TMPPWD)" | awk '{gsub("\\\\", "/"); print}' ) +endif + build: linux darwin linux darwin windows: $(GOFILES) @echo "building $@ from $(SRC_AND_UNDER)" - @rm -f VERSION.txt - @mkdir -p bin/$@ $(GOTMP)/{std/$@,bin,src/$(PKG)} - + @$(shell rm -f VERSION.txt) + @$(shell mkdir -p bin/$@ $(GOTMP)/{std/$@,bin,src/$(PKG)}) @docker run -t --rm -u $(shell id -u):$(shell id -g) \ - -v $$(pwd)/$(GOTMP):/go \ - -v $$(pwd):/go/src/$(PKG) \ - -v $$(pwd)/bin/$@:/go/bin \ - -v $$(pwd)/bin/$@:/go/bin/$@ \ - -v $$(pwd)/$(GOTMP)/std/$@:/usr/local/go/pkg/$@_amd64_static \ + -v $(PWD)/$(GOTMP):/go \ + -v $(PWD):/go/src/$(PKG) \ + -v $(PWD)/bin/$@:/go/bin \ + -v $(PWD)/bin/$@:/go/bin/$@ \ + -v $(PWD)/$(GOTMP)/std/$@:/usr/local/go/pkg/$@_amd64_static \ -e CGO_ENABLED=0 \ -e GOOS=$@ \ -w /go/src/$(PKG) \ $(BUILD_IMAGE) \ go install -installsuffix static -ldflags ' $(LDFLAGS) ' $(SRC_AND_UNDER) - @touch $@ + @$(shell touch $@) @echo $(VERSION) >VERSION.txt static: govendor gofmt govet lint @@ -57,8 +64,8 @@ static: govendor gofmt govet lint govendor: @echo -n "Using govendor to check for missing dependencies and unused dependencies: " @docker run -t --rm -u $(shell id -u):$(shell id -g) \ - -v $$(pwd)/$(GOTMP):/go \ - -v $$(pwd):/go/src/$(PKG) \ + -v $(PWD)/$(GOTMP):/go \ + -v $(PWD):/go/src/$(PKG) \ -w /go/src/$(PKG) \ $(BUILD_IMAGE) \ bash -c 'OUT=$$(govendor list +missing +unused); if [ -n "$$OUT" ]; then echo "$$OUT"; exit 1; fi' @@ -66,8 +73,8 @@ govendor: gofmt: @echo "Checking gofmt: " @docker run -t --rm -u $(shell id -u):$(shell id -g) \ - -v $$(pwd)/$(GOTMP):/go \ - -v $$(pwd):/go/src/$(PKG) \ + -v $(PWD)/$(GOTMP):/go \ + -v $(PWD):/go/src/$(PKG) \ -w /go/src/$(PKG) \ $(BUILD_IMAGE) \ bash -c 'export OUT=$$(gofmt -l $(SRC_DIRS)) && if [ -n "$$OUT" ]; then echo "These files need gofmt -w: $$OUT"; exit 1; fi' @@ -75,8 +82,8 @@ gofmt: govet: @echo "Checking go vet: " @docker run -t --rm -u $(shell id -u):$(shell id -g) \ - -v $$(pwd)/$(GOTMP):/go \ - -v $$(pwd):/go/src/$(PKG) \ + -v $(PWD)/$(GOTMP):/go \ + -v $(PWD):/go/src/$(PKG) \ -w /go/src/$(PKG) \ $(BUILD_IMAGE) \ bash -c 'go vet $(SRC_AND_UNDER)' @@ -84,8 +91,8 @@ govet: golint: @echo "Checking golint: " @docker run -t --rm -u $(shell id -u):$(shell id -g) \ - -v $$(pwd)/$(GOTMP):/go \ - -v $$(pwd):/go/src/$(PKG) \ + -v $(PWD)/$(GOTMP):/go \ + -v $(PWD):/go/src/$(PKG) \ -w /go/src/$(PKG) \ $(BUILD_IMAGE) \ bash -c 'export OUT=$$(golint $(SRC_AND_UNDER)) && if [ -n "$$OUT" ]; then echo "Golint problems discovered: $$OUT"; exit 1; fi' @@ -93,8 +100,8 @@ golint: errcheck: @echo "Checking errcheck: " @docker run -t --rm -u $(shell id -u):$(shell id -g) \ - -v $$(pwd)/$(GOTMP):/go \ - -v $$(pwd):/go/src/$(PKG) \ + -v $(PWD)/$(GOTMP):/go \ + -v $(PWD):/go/src/$(PKG) \ -w /go/src/$(PKG) \ $(BUILD_IMAGE) \ errcheck $(SRC_AND_UNDER) @@ -102,8 +109,8 @@ errcheck: staticcheck: @echo "Checking staticcheck: " @docker run -t --rm -u $(shell id -u):$(shell id -g) \ - -v $$(pwd)/$(GOTMP):/go \ - -v $$(pwd):/go/src/$(PKG) \ + -v $(PWD)/$(GOTMP):/go \ + -v $(PWD):/go/src/$(PKG) \ -w /go/src/$(PKG) \ $(BUILD_IMAGE) \ staticcheck $(SRC_AND_UNDER) @@ -111,8 +118,8 @@ staticcheck: unused: @echo "Checking unused variables and functions: " @docker run -t --rm -u $(shell id -u):$(shell id -g) \ - -v $$(pwd)/$(GOTMP):/go \ - -v $$(pwd):/go/src/$(PKG) \ + -v $(PWD)/$(GOTMP):/go \ + -v $(PWD):/go/src/$(PKG) \ -w /go/src/$(PKG) \ $(BUILD_IMAGE) \ unused $(SRC_AND_UNDER) @@ -120,8 +127,8 @@ unused: codecoroner: @echo "Checking codecoroner for unused functions: " @docker run -t --rm -u $(shell id -u):$(shell id -g) \ - -v $$(pwd)/$(GOTMP):/go \ - -v $$(pwd):/go/src/$(PKG) \ + -v $(PWD)/$(GOTMP):/go \ + -v $(PWD):/go/src/$(PKG) \ -w /go/src/$(PKG) \ $(BUILD_IMAGE) \ bash -c 'OUT=$$(codecoroner -tests -ignore vendor funcs $(SRC_AND_UNDER)); if [ -n "$$OUT" ]; then echo "$$OUT"; exit 1; fi' \ @@ -130,8 +137,8 @@ codecoroner: varcheck: @echo "Checking unused globals and struct members: " @docker run -t --rm -u $(shell id -u):$(shell id -g) \ - -v $$(pwd)/$(GOTMP):/go \ - -v $$(pwd):/go/src/$(PKG) \ + -v $(PWD)/$(GOTMP):/go \ + -v $(PWD):/go/src/$(PKG) \ -w /go/src/$(PKG) \ $(BUILD_IMAGE) \ varcheck $(SRC_AND_UNDER) && structcheck $(SRC_AND_UNDER) @@ -139,8 +146,8 @@ varcheck: misspell: @echo "Checking for misspellings: " @docker run -t --rm -u $(shell id -u):$(shell id -g) \ - -v $$(pwd)/$(GOTMP):/go \ - -v $$(pwd):/go/src/$(PKG) \ + -v $(PWD)/$(GOTMP):/go \ + -v $(PWD):/go/src/$(PKG) \ -w /go/src/$(PKG) \ $(BUILD_IMAGE) \ misspell $(SRC_DIRS) @@ -148,8 +155,8 @@ misspell: gometalinter: @echo "gometalinter: " @docker run -t --rm -u $(shell id -u):$(shell id -g) \ - -v $$(pwd)/$(GOTMP):/go \ - -v $$(pwd):/go/src/$(PKG) \ + -v $(PWD)/$(GOTMP):/go \ + -v $(PWD):/go/src/$(PKG) \ -w /go/src/$(PKG) \ $(BUILD_IMAGE) \ gometalinter $(GOMETALINTER_ARGS) $(SRC_AND_UNDER) @@ -160,10 +167,10 @@ version: clean: container-clean bin-clean container-clean: - rm -rf .container-* .dockerfile* .push-* linux darwin windows container VERSION.txt .docker_image + $(shell rm -rf .container-* .dockerfile* .push-* linux darwin windows container VERSION.txt .docker_image) bin-clean: - rm -rf $(GOTMP) bin .tmp + $(shell rm -rf $(GOTMP) bin .tmp) # print-ANYVAR prints the expanded variable print-%: ; @echo $* = $($*) diff --git a/makefile_components/base_build_python-docker.mak b/makefile_components/base_build_python-docker.mak index 6daf563..35b9e39 100644 --- a/makefile_components/base_build_python-docker.mak +++ b/makefile_components/base_build_python-docker.mak @@ -9,6 +9,8 @@ SHELL := /bin/bash +PWD := $(shell pwd) + BUILD_IMAGE ?= drud/golang-build-container:v0.2.0 all: VERSION.txt build diff --git a/makefile_components/base_test_go.mak b/makefile_components/base_test_go.mak index 627ec28..bb2e6d9 100644 --- a/makefile_components/base_test_go.mak +++ b/makefile_components/base_test_go.mak @@ -4,17 +4,17 @@ ##### contents into ../Makefile and commenting out the include and adding a ##### comment about what you did and why. -TESTOS = $(shell uname -s | tr '[:upper:]' '[:lower:]') +TESTOS = $(BUILD_OS) test: build @mkdir -p bin/linux @mkdir -p $(GOTMP)/{src/$(PKG),pkg,bin,std/linux} @echo "Testing $(SRC_AND_UNDER) with TESTARGS=$(TESTARGS)" @docker run -t --rm -u $(shell id -u):$(shell id -g) \ - -v $$(pwd)/$(GOTMP):/go \ - -v $$(pwd):/go/src/$(PKG) \ - -v $$(pwd)/bin/linux:/go/bin \ - -v $$(pwd)/$(GOTMP)/std/linux:/usr/local/go/pkg/linux_amd64_static \ + -v $(PWD)/$(GOTMP):/go \ + -v $(PWD):/go/src/$(PKG) \ + -v $(PWD)/bin/linux:/go/bin \ + -v $(PWD)/$(GOTMP)/std/linux:/usr/local/go/pkg/linux_amd64_static \ -e CGO_ENABLED=0 \ -w /go/src/$(PKG) \ $(BUILD_IMAGE) \ diff --git a/tests/Makefile b/tests/Makefile index 604073c..75a54ad 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -69,11 +69,12 @@ test_precompile: test COMMAND := govendor list container_cmd: @docker run \ - -t \ - -u $(shell id -u):$(shell id -g) \ - -v $$(pwd)/$(GOTMP):/go \ - -v $$(pwd):/go/src/$(PKG) \ + -t --rm -u $(shell id -u):$(shell id -g) \ + -v $(PWD)/$(GOTMP):/go \ + -v $(PWD):/go/src/$(PKG) \ + -v $(PWD)/bin/linux:/go/bin \ + -v $(PWD)/$(GOTMP)/std/linux:/usr/local/go/pkg/linux_amd64_static \ -w /go/src/$(PKG) \ $(BUILD_IMAGE) \ - bash -c "$(COMMAND)" + $(COMMAND) diff --git a/tests/pkg/clean/build_tools_test.go b/tests/pkg/clean/build_tools_test.go index 37a8928..56a4a7a 100644 --- a/tests/pkg/clean/build_tools_test.go +++ b/tests/pkg/clean/build_tools_test.go @@ -4,10 +4,10 @@ import ( "os/exec" "testing" - "fmt" "log" "os" - "strings" + + "runtime" "github.com/drud/build-tools/tests/pkg/version" "github.com/stretchr/testify/assert" @@ -23,12 +23,8 @@ func init() { if err != nil { log.Fatalln("Failed to chdir to ../..", err) } - // Operating system - Darwin or Linux - v, err := exec.Command("uname", "-s").Output() - if err != nil { - log.Fatalln("Failed to run uname command:", string(v)) - } - osname = strings.TrimSpace(string(v)) + // Operating system - Darwin or Linux or Windows + osname = runtime.GOOS } // Runs a number of standard make targets and test for basic sanity of result @@ -38,13 +34,12 @@ func TestBuild(t *testing.T) { // Map OS name to output location binlocs := map[string]string{ - "Darwin": "bin/darwin/darwin_amd64/build_tools_dummy", - "Linux": "bin/linux/build_tools_dummy", - "Windows": "bin/windows/windows_amd64/build_tools_dummy.exe", + "darwin": "bin/darwin/darwin_amd64/build_tools_dummy", + "linux": "bin/linux/build_tools_dummy", + "windows": "bin/windows/windows_amd64/build_tools_dummy.exe", } dir, _ := os.Getwd() - fmt.Println("Current Directory:", dir) v, err := exec.Command("which", "make").Output() a.NoError(err) @@ -148,6 +143,10 @@ func TestGovendor(t *testing.T) { _, err = exec.Command("make", "COMMAND=govendor fetch "+neededPackage, "container_cmd").Output() assert.NoError(err) + // Try to clean up the mess we may have made in the vendor directory + _, _ = exec.Command("git", "checkout vendor").Output() + _, _ = exec.Command("git", "clean -fd vendor").Output() + } // Test golint on clean and unclean code diff --git a/tests/pkg/version/version.go b/tests/pkg/version/version.go index bf71347..8815fd9 100644 --- a/tests/pkg/version/version.go +++ b/tests/pkg/version/version.go @@ -4,4 +4,4 @@ package version var VERSION = "" var COMMIT = "COMMIT should be overridden" -var BUILDINFO = "BUILDINFO should have new info" \ No newline at end of file +var BUILDINFO = "BUILDINFO should have new info"