Use release-0.13 branch of test-infra (#799)

(Merging manually to skip WI test)

Gopkg.toml

@@ -40,6 +40,11 @@ required = [
   name = "knative.dev/serving"
   branch = "release-0.13"
 
+# This is a preemptive override.
+[[override]]
+  name = "knative.dev/test-infra"
+  branch = "release-0.13"
+
 [[override]]
   name = "go.uber.org/zap"
   revision = "67bc79d13d155c02fd008f721863ff8cc5f30659"

vendor/knative.dev/test-infra/scripts/e2e-tests.sh

@@ -93,7 +93,7 @@ function dump_cluster_state() {
   echo "***    Start of information dump    ***"
   echo "***************************************"
 
-  local output="${ARTIFACTS}/k8s.dump.txt"
+  local output="${ARTIFACTS}/k8s.dump-$(basename ${E2E_SCRIPT}).txt"
   echo ">>> The dump is located at ${output}"
 
   for crd in $(kubectl api-resources --verbs=list -o name | sort); do

vendor/knative.dev/test-infra/scripts/presubmit-tests.sh

@@ -28,7 +28,7 @@ readonly PRESUBMIT_TEST_FAIL_FAST=${PRESUBMIT_TEST_FAIL_FAST:-0}
 readonly NO_PRESUBMIT_FILES=(\.png \.gitignore \.gitattributes ^OWNERS ^OWNERS_ALIASES ^AUTHORS)
 
 # Flag if this is a presubmit run or not.
-[[ IS_PROW && -n "${PULL_PULL_SHA}" ]] && IS_PRESUBMIT=1 || IS_PRESUBMIT=0
+(( IS_PROW )) && [[ -n "${PULL_PULL_SHA}" ]] && IS_PRESUBMIT=1 || IS_PRESUBMIT=0
 readonly IS_PRESUBMIT
 
 # List of changed files on presubmit, LF separated.
@@ -177,11 +177,12 @@ function default_build_test_runner() {
     # Consider an error message everything that's not a package name.
     errors_go1="$(grep -v '^\(github\.com\|knative\.dev\)/' "${report}" | sort | uniq)"
   fi
-  # Get all build tags in go code (ignore /vendor)
+  # Get all build tags in go code (ignore /vendor and /hack)
   local tags="$(grep -r '// +build' . \
-      | grep -v '^./vendor/' | cut -f3 -d' ' | sort | uniq | tr '\n' ' ')"
+      | grep -v '^./vendor/' | grep -v '^./hack/' | cut -f3 -d' ' | sort | uniq | tr '\n' ' ')"
   local tagged_pkgs="$(grep -r '// +build' . \
-    | grep -v '^./vendor/' | grep ":// +build " | cut -f1 -d: | xargs dirname | sort | uniq | tr '\n' ' ')"
+    | grep -v '^./vendor/' | grep -v '^./hack/' | grep ":// +build " | cut -f1 -d: | xargs dirname \
+    | sort | uniq | tr '\n' ' ')"
   for pkg in ${tagged_pkgs}; do
     # `go test -c` lets us compile the tests but do not run them.
     if ! capture_output "${report}" go test -c -tags="${tags}" ${pkg} ; then
@@ -362,7 +363,7 @@ function main() {
 
   local failed=0
 
-  if [[ ${#TESTS_TO_RUN[@]} > 0 ]]; then
+  if [[ ${#TESTS_TO_RUN[@]} -gt 0 ]]; then
     if (( RUN_BUILD_TESTS || RUN_UNIT_TESTS || RUN_INTEGRATION_TESTS )); then
       abort "--run-test must be used alone"
     fi

vendor/knative.dev/test-infra/tools/dep-collector/gobuild.go

@@ -0,0 +1,88 @@
+/*
+Copyright 2020 The Knative Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	gb "go/build"
+	"os/exec"
+	"path/filepath"
+	"strings"
+)
+
+// https://golang.org/pkg/cmd/go/internal/modinfo/#ModulePublic
+type modInfo struct {
+	Path string
+	Dir  string
+}
+
+type gobuild struct {
+	mod *modInfo
+}
+
+// moduleInfo returns the module path and module root directory for a project
+// using go modules, otherwise returns nil.
+// If there is something wrong in getting the module info, it will return an error.
+//
+// Related: https://github.com/golang/go/issues/26504
+func moduleInfo() (*modInfo, error) {
+	// If `go list -m` returns an error, the project is not using Go modules.
+	c := exec.Command("go", "list", "-m")
+	_, err := c.Output()
+	if err != nil {
+		return nil, nil
+	}
+
+	lc := exec.Command("go", "list", "-mod=readonly", "-m", "-json")
+	output, err := lc.Output()
+	if err != nil {
+		return nil, fmt.Errorf("failed getting module info: %v", err)
+	}
+	var info modInfo
+	if err := json.Unmarshal(output, &info); err != nil {
+		return nil, fmt.Errorf("failed parsing module info %q: %v", output, err)
+	}
+	return &info, nil
+}
+
+// importPackage wraps go/build.Import to handle go modules.
+//
+// Note that we will fall back to GOPATH if the project isn't using go modules.
+func (g *gobuild) importPackage(s string) (*gb.Package, error) {
+	if g.mod == nil {
+		return gb.Import(s, WorkingDir, gb.ImportComment)
+	}
+
+	// If we're inside a go modules project, try to use the module's directory
+	// as our source root to import:
+	// * paths that match module path prefix (they should be in this project)
+	// * relative paths (they should also be in this project)
+	gp, err := gb.Import(s, g.mod.Dir, gb.ImportComment)
+	return gp, err
+}
+
+func (g *gobuild) qualifyLocalImport(ip string) (string, error) {
+	if g.mod == nil {
+		gopathsrc := filepath.Join(gb.Default.GOPATH, "src")
+		if !strings.HasPrefix(WorkingDir, gopathsrc) {
+			return "", fmt.Errorf("working directory must be on ${GOPATH}/src = %s", gopathsrc)
+		}
+		return filepath.Join(strings.TrimPrefix(WorkingDir, gopathsrc+string(filepath.Separator)), ip), nil
+	}
+	return filepath.Join(g.mod.Path, ip), nil
+}

vendor/knative.dev/test-infra/tools/dep-collector/imports.go

@@ -19,74 +19,80 @@ package main
 import (
 	"fmt"
 	gb "go/build"
-	"path/filepath"
 	"sort"
 	"strings"
 )
 
-func CollectTransitiveImports(binaries []string) ([]string, error) {
+type ImportInfo struct {
+	ImportPath string
+	Dir        string
+}
+
+func CollectTransitiveImports(binaries []string) ([]ImportInfo, error) {
 	// Perform a simple DFS to collect the binaries' transitive dependencies.
-	visited := make(map[string]struct{})
+	visited := make(map[string]ImportInfo)
+	mi, err := moduleInfo()
+	if err != nil {
+		return nil, fmt.Errorf("failed getting Go module info: %v", err)
+	}
+	g := &gobuild{mi}
 	for _, importpath := range binaries {
 		if gb.IsLocalImport(importpath) {
-			ip, err := qualifyLocalImport(importpath)
+			ip, err := g.qualifyLocalImport(importpath)
 			if err != nil {
 				return nil, err
 			}
 			importpath = ip
 		}
 
-		pkg, err := gb.Import(importpath, WorkingDir, gb.ImportComment)
+		pkg, err := g.importPackage(importpath)
 		if err != nil {
 			return nil, err
 		}
-		if err := visit(pkg, visited); err != nil {
+		if err := visit(g, pkg, visited); err != nil {
 			return nil, err
 		}
 	}
 
 	// Sort the dependencies deterministically.
 	var list sort.StringSlice
-	for ip := range visited {
-		if !strings.Contains(ip, "/vendor/") {
+	for dir := range visited {
+		if !strings.Contains(dir, "/vendor/") {
 			// Skip files outside of vendor
 			continue
 		}
-		list = append(list, ip)
+		list = append(list, dir)
 	}
 	list.Sort()
 
-	return list, nil
-}
-
-func qualifyLocalImport(ip string) (string, error) {
-	gopathsrc := filepath.Join(gb.Default.GOPATH, "src")
-	if !strings.HasPrefix(WorkingDir, gopathsrc) {
-		return "", fmt.Errorf("working directory must be on ${GOPATH}/src = %s", gopathsrc)
+	iiList := make([]ImportInfo, len(list))
+	for i := range iiList {
+		iiList[i] = visited[list[i]]
 	}
-	return filepath.Join(strings.TrimPrefix(WorkingDir, gopathsrc+string(filepath.Separator)), ip), nil
+
+	return iiList, nil
 }
 
-func visit(pkg *gb.Package, visited map[string]struct{}) error {
-	if _, ok := visited[pkg.ImportPath]; ok {
+func visit(g *gobuild, pkg *gb.Package, visited map[string]ImportInfo) error {
+	if _, ok := visited[pkg.Dir]; ok {
 		return nil
 	}
-	visited[pkg.ImportPath] = struct{}{}
+	visited[pkg.Dir] = ImportInfo{Dir: pkg.Dir, ImportPath: pkg.ImportPath}
 
 	for _, ip := range pkg.Imports {
 		if ip == "C" {
 			// skip cgo
 			continue
 		}
-		subpkg, err := gb.Import(ip, WorkingDir, gb.ImportComment)
+		subpkg, err := g.importPackage(ip)
 		if err != nil {
 			return fmt.Errorf("%v\n -> %v", pkg.ImportPath, err)
 		}
 		if !strings.HasPrefix(subpkg.Dir, WorkingDir) {
 			// Skip import paths outside of our workspace (std library)
 			continue
 		}
-		if err := visit(subpkg, visited); err != nil {
+		if err := visit(g, subpkg, visited); err != nil {
 			return fmt.Errorf("%v (%v)\n -> %v", pkg.ImportPath, pkg.Dir, err)
 		}
 	}

vendor/knative.dev/test-infra/tools/dep-collector/licenses.go

@@ -18,7 +18,6 @@ package main
 
 import (
 	"fmt"
-	gb "go/build"
 	"io/ioutil"
 	"os"
 	"path/filepath"
@@ -72,7 +71,7 @@ func (lt *LicenseFile) Check(classifier *licenseclassifier.License) error {
 	}
 	ms := classifier.MultipleMatch(body, false)
 	for _, m := range ms {
-		return fmt.Errorf("Found matching forbidden license in %q: %v", lt.EnclosingImportPath, m.Name)
+		return fmt.Errorf("found matching forbidden license in %q: %v", lt.EnclosingImportPath, m.Name)
 	}
 	return nil
 }
@@ -108,17 +107,13 @@ func (lt *LicenseFile) CSVRow(classifier *licenseclassifier.License) (string, er
 	}, ","), nil
 }
 
-func findLicense(ip string) (*LicenseFile, error) {
-	pkg, err := gb.Import(ip, WorkingDir, gb.ImportComment)
-	if err != nil {
-		return nil, err
-	}
-	dir := pkg.Dir
-
+func findLicense(ii ImportInfo) (*LicenseFile, error) {
+	dir := ii.Dir
+	ip := ii.ImportPath
 	for {
 		// When we reach the root of our workspace, stop searching.
 		if dir == WorkingDir {
-			return nil, fmt.Errorf("unable to find license for %q", pkg.ImportPath)
+			return nil, fmt.Errorf("unable to find license for %q", ip)
 		}
 
 		for _, name := range LicenseNames {
@@ -178,11 +173,12 @@ func (lc LicenseCollection) Check(classifier *licenseclassifier.License) error {
 	return fmt.Errorf("Errors validating licenses:\n%v", strings.Join(errors, "\n"))
 }
 
-func CollectLicenses(imports []string) (LicenseCollection, error) {
+// CollectLicenses collects a list of licenses for the given imports.
+func CollectLicenses(importInfos []ImportInfo) (LicenseCollection, error) {
 	// for each of the import paths, search for a license file.
 	licenseFiles := make(map[string]*LicenseFile)
-	for _, ip := range imports {
-		lf, err := findLicense(ip)
+	for _, info := range importInfos {
+		lf, err := findLicense(info)
 		if err != nil {
 			return nil, err
 		}