From 87a14b0df7e79574e5057473b7c23b03e2cd2a88 Mon Sep 17 00:00:00 2001 From: Alexandre Mahdhaoui Date: Sun, 14 Apr 2024 01:21:28 +0200 Subject: [PATCH] =?UTF-8?q?=F0=9F=8C=B1=20verify=20go=20modules=20are=20in?= =?UTF-8?q?=20sync=20with=20upstream=20k/k?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit addresses issues were go modules aren't in sync with upstream k/k by adding these changes: - add `tools/cmd/gomodcheck/main.go` to: - Parse and compares k/k dependencies to controller-runtime's ones. - If any version diffs is found, returns a payload describing the diffs and exit 1. - The user may exclude packages by passing them as arguments. - extend the `verify-modules` make target with `gomodcheck`. --- Makefile | 9 +- hack/.gomodcheck.yaml | 14 +++ hack/tools/cmd/gomodcheck/main.go | 203 ++++++++++++++++++++++++++++++ 3 files changed, 225 insertions(+), 1 deletion(-) create mode 100644 hack/.gomodcheck.yaml create mode 100644 hack/tools/cmd/gomodcheck/main.go diff --git a/Makefile b/Makefile index 438613b3eb..e1a6941433 100644 --- a/Makefile +++ b/Makefile @@ -92,6 +92,12 @@ GOLANGCI_LINT_PKG := github.com/golangci/golangci-lint/cmd/golangci-lint $(GOLANGCI_LINT): # Build golangci-lint from tools folder. GOBIN=$(TOOLS_BIN_DIR) $(GO_INSTALL) $(GOLANGCI_LINT_PKG) $(GOLANGCI_LINT_BIN) $(GOLANGCI_LINT_VER) +GO_MOD_CHECK_DIR := $(abspath ./hack/tools/cmd/gomodcheck) +GO_MOD_CHECK := $(abspath $(TOOLS_BIN_DIR)/gomodcheck) +GO_MOD_CHECK_IGNORE := $(abspath ./hack/.gomodcheck.yaml) +$(GO_MOD_CHECK): # Build gomodcheck + go build -C $(GO_MOD_CHECK_DIR) -o $(GO_MOD_CHECK) + ## -------------------------------------- ## Linting ## -------------------------------------- @@ -130,11 +136,12 @@ clean-bin: ## Remove all generated binaries. rm -rf hack/tools/bin .PHONY: verify-modules -verify-modules: modules ## Verify go modules are up to date +verify-modules: modules $(GO_MOD_CHECK) ## Verify go modules are up to date @if !(git diff --quiet HEAD -- go.sum go.mod $(TOOLS_DIR)/go.mod $(TOOLS_DIR)/go.sum $(ENVTEST_DIR)/go.mod $(ENVTEST_DIR)/go.sum $(SCRATCH_ENV_DIR)/go.sum); then \ git diff; \ echo "go module files are out of date, please run 'make modules'"; exit 1; \ fi + $(GO_MOD_CHECK) $(GO_MOD_CHECK_IGNORE) APIDIFF_OLD_COMMIT ?= $(shell git rev-parse origin/main) diff --git a/hack/.gomodcheck.yaml b/hack/.gomodcheck.yaml new file mode 100644 index 0000000000..8dbe45fac0 --- /dev/null +++ b/hack/.gomodcheck.yaml @@ -0,0 +1,14 @@ +upstreamRefs: + - k8s.io/api + - k8s.io/apiextensions-apiserver + - k8s.io/apimachinery + - k8s.io/apiserver + - k8s.io/client-go + - k8s.io/component-base + - k8s.io/klog/v2 + # k8s.io/utils -> conflicts with k/k deps + +excludedModules: + # --- test dependencies: + - github.com/onsi/ginkgo/v2 + - github.com/onsi/gomega diff --git a/hack/tools/cmd/gomodcheck/main.go b/hack/tools/cmd/gomodcheck/main.go new file mode 100644 index 0000000000..9c32243976 --- /dev/null +++ b/hack/tools/cmd/gomodcheck/main.go @@ -0,0 +1,203 @@ +package main + +import ( + "encoding/json" + "fmt" + "os" + "os/exec" + "regexp" + "strings" + + "go.uber.org/zap" + "sigs.k8s.io/yaml" +) + +const ( + modFile = "./go.mod" +) + +type config struct { + UpstreamRefs []string `yaml:"upstreamRefs"` + ExcludedModules []string `yaml:"excludedModules"` +} + +type upstream struct { + Ref string `json:"ref"` + Version string `json:"version"` +} + +// representation of an out of sync module +type oosMod struct { + Name string `json:"name"` + Version string `json:"version"` + Upstreams []upstream `json:"upstreams"` +} + +func main() { + l, _ := zap.NewProduction() + logger := l.Sugar() + + if len(os.Args) < 2 { + fmt.Printf("USAGE: %s [PATH_TO_CONFIG_FILE]\n", os.Args[0]) + os.Exit(1) + } + + // --- 1. parse config + b, err := os.ReadFile(os.Args[1]) + if err != nil { + logger.Fatal(err.Error()) + } + + cfg := new(config) + if err := yaml.Unmarshal(b, cfg); err != nil { + logger.Fatal(err.Error()) + } + + excludedMods := make(map[string]any) + for _, mod := range cfg.ExcludedModules { + excludedMods[mod] = nil + } + + // --- 2. project mods + deps, err := parseModFile() + if err != nil { + logger.Fatal(err.Error()) + } + + // --- 3. upstream mods (holding upstream refs) + upstreamModGraph, err := getUpstreamModGraph(cfg.UpstreamRefs) + if err != nil { + logger.Fatal(err.Error()) + } + + oosMods := make([]oosMod, 0) + + // --- 4. validate + // for each module in our project, + // if it matches an upstream module, + // then for each upstream module, + // if project module version doesn't match upstream version, + // then we add the version and the ref to the list of out of sync modules. + for mod, version := range deps { + if _, ok := excludedMods[mod]; ok { + logger.Infof("skipped excluded module: %s", mod) + continue + } + + if versionToRef, ok := upstreamModGraph[mod]; ok { + upstreams := make([]upstream, 0) + + for upstreamVersion, upstreamRef := range versionToRef { + if version != upstreamVersion { + upstreams = append(upstreams, upstream{ + Ref: upstreamRef, + Version: upstreamVersion, + }) + } + } + + if len(upstreams) > 0 { + oosMods = append(oosMods, oosMod{ + Name: mod, + Version: version, + Upstreams: upstreams, + }) + } + } + } + + if len(oosMods) == 0 { + fmt.Println("Success! 🎉") + os.Exit(0) + } + + b, err = json.MarshalIndent(map[string]any{"outOfSyncModules": oosMods}, "", " ") + if err != nil { + panic(err) + } + + fmt.Println(string(b)) + os.Exit(1) +} + +var ( + cleanMods = regexp.MustCompile(`\t| *//.*`) + modDelimStart = regexp.MustCompile(`^require.*`) + modDelimEnd = ")" +) + +func parseModFile() (map[string]string, error) { + b, err := os.ReadFile(modFile) + if err != nil { + return nil, err + } + + in := string(cleanMods.ReplaceAll(b, []byte(""))) + out := make(map[string]string) + + start := false + for _, s := range strings.Split(in, "\n") { + switch { + case modDelimStart.MatchString(s) && !start: + start = true + case s == modDelimEnd: + return out, nil + case start: + kv := strings.SplitN(s, " ", 2) + if len(kv) < 2 { + return nil, fmt.Errorf("unexpected format for module: %q", s) + } + + out[kv[0]] = kv[1] + } + } + + return out, nil +} + +func getUpstreamModGraph(upstreamRefs []string) (map[string]map[string]string, error) { + b, err := exec.Command("go", "mod", "graph").Output() + if err != nil { + return nil, err + } + + graph := string(b) + o1Refs := make(map[string]bool) + for _, upstreamRef := range upstreamRefs { + o1Refs[upstreamRef] = false + } + + modToVersionToUpstreamRef := make(map[string]map[string]string) + + for _, line := range strings.Split(graph, "\n") { + upstreamRef := strings.SplitN(line, "@", 2)[0] + if _, ok := o1Refs[upstreamRef]; ok { + o1Refs[upstreamRef] = true + kv := strings.SplitN(strings.SplitN(line, " ", 2)[1], "@", 2) + name := kv[0] + version := kv[1] + + if m, ok := modToVersionToUpstreamRef[kv[0]]; ok { + m[version] = upstreamRef + } else { + versionToRef := map[string]string{version: upstreamRef} + modToVersionToUpstreamRef[name] = versionToRef + } + } + } + + notFound := "" + for ref, found := range o1Refs { + if !found { + notFound = fmt.Sprintf("%s%s, ", notFound, ref) + } + } + + if notFound != "" { + return nil, fmt.Errorf("cannot verify modules;"+ + "the following specified upstream module cannot be found in go.mod: [ %s ]", + strings.TrimSuffix(notFound, ", ")) + } + + return modToVersionToUpstreamRef, nil +}