feat(mockgen): record code coverage for test scripts

Prior to this change, the test scripts (under ./cmd/mockgen) would run
the command using go-run, which builds the mockgen binary and then runs
it.  This meant that no code coverage was collected for these tests.

This change introduces a custom TestMain function that builds the binary
with the -cover option (new in go 1.20) which produces coverage data in
a binary format.  The go-test command can also produce data in the same
format when given the -test.gocoverdir option.  When the ./cmd/mockgen
tests find this option, coverage for the script tests is enabled.  This
allows us to then combine the coverage from the tests across all
packages.  At this time, the go-test command doesn't appear to be able
to do this automatically and the coverage reported is inaccurate.  To
generate the combined coverage use the covdata tool after running
go-test:

    mkdir "$PWD/covdata"
    go test -cover ./... -test.gocoverdir="$PWD/covdata"
    go tool covdata percent -i="$PWD/covdata"

The same tool can also convert to the old text format:

    go tool covdata textfmt -o cover.out -i="$PWD/covdata"

See https://go.dev/doc/build-cover

.github/workflows/test.yml

@@ -20,11 +20,16 @@ jobs:
         run: go mod tidy && git diff --exit-code
 
       - name: Test
-        run: go test -v ./...
+        run: |
+          mkdir -p "${{ runner.temp }}/covdata"
+          go test -v -cover ./...  -test.gocoverdir="${{ runner.temp }}/covdata"
+          go tool covdata percent -i="${{ runner.temp }}/covdata"
+          go tool covdata textfmt -o "${{ runner.temp }}/coverage.out" -i="${{ runner.temp }}/covdata"
 
       - name: Coverage report
         uses: ncruces/go-coverage-report@v0
         with:
+          coverage-file: "${{ runner.temp }}/coverage.out"
           report: true
           chart: true
           amend: true

cmd/mockgen/main_test.go

@@ -1,30 +1,78 @@
 package main_test
 
 import (
+	"bytes"
 	"context"
+	"flag"
+	"fmt"
 	"os"
+	"os/exec"
 	"path/filepath"
+	"strings"
 	"testing"
 
 	"rsc.io/script"
 	"rsc.io/script/scripttest"
 )
 
-func TestMain(t *testing.T) {
+var env []string
+
+func TestMockgen(t *testing.T) {
+	ctx, eng := context.Background(), script.NewEngine()
+	scripttest.Test(t, ctx, eng, env, "testdata/*.txt")
+}
+
+func TestMain(m *testing.M) {
+	// Temporary directory for mockgen binary.
+	bindir, err := os.MkdirTemp("", "mockgen")
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+	defer os.RemoveAll(bindir)
+
+	// The Module Under Test (MUT) directory.
 	mutdir, err := os.Getwd()
 	if err != nil {
-		t.Fatal(err)
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
 	}
 	mutdir = filepath.Join(mutdir, "..", "..")
-	scripttest.Test(
-		t,
-		context.Background(),
-		script.NewEngine(),
-		[]string{
-			"PATH=" + os.Getenv("PATH"),
-			"HOME=" + os.Getenv("HOME"),
-			"MUT=" + mutdir,
-		},
-		"testdata/*.txt",
-	)
+
+	// When running with -test.gocoverdir, forward this setting to each
+	// test script.
+	fset := flag.NewFlagSet("mockgen", flag.ContinueOnError)
+	fset.SetOutput(&bytes.Buffer{}) // ignore errors
+	covdir := fset.String("test.gocoverdir", "", "write coverage intermediate files to this directory")
+	err = fset.Parse(os.Args[1:])
+	for err != nil {
+		err = fset.Parse(fset.Args())
+	}
+
+	env = []string{
+		"PATH=" + strings.Join([]string{bindir, os.Getenv("PATH")}, ":"),
+		"HOME=" + os.Getenv("HOME"),
+		"MUT=" + mutdir,
+		"BINDIR=" + bindir,
+	}
+	if *covdir != "" {
+		env = append(env, "GOCOVERDIR="+*covdir)
+	}
+
+	run(env, "testdata/setup.sh")
+	m.Run()
+	if *covdir != "" {
+		run(env, "testdata/report.sh")
+	}
+}
+
+func run(env []string, file string) {
+	cmd := exec.Command("bash", file)
+	cmd.Env = env
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	if err := cmd.Run(); err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
 }

cmd/mockgen/testdata/commands.txt

@@ -1,6 +1,6 @@
 exec go mod edit -replace github.com/Versent/go-mock=$MUT
 exec go mod tidy
-exec go run github.com/Versent/go-mock/cmd/mockgen commands
+exec mockgen commands
 
 cmp stdout stdout.golden
 cmp stderr stderr.golden

cmd/mockgen/testdata/fallback.txt

@@ -1,6 +1,6 @@
 exec go mod edit -replace github.com/Versent/go-mock=$MUT
 exec go mod tidy
-exec go run github.com/Versent/go-mock/cmd/mockgen
+exec mockgen
 
 cmp    stdout stdout.golden
 cmpenv stderr stderr.golden

cmd/mockgen/testdata/fallback_all_args.txt

@@ -1,6 +1,6 @@
 exec go mod edit -replace github.com/Versent/go-mock=$MUT
 exec go mod tidy
-exec go run github.com/Versent/go-mock/cmd/mockgen -header HEADER -tags extra .
+exec mockgen -header HEADER -tags extra .
 
 cmp    stdout stdout.golden
 cmpenv stderr stderr.golden

cmd/mockgen/testdata/fallback_flag_undefined.txt

@@ -1,6 +1,6 @@
 exec go mod edit -replace github.com/Versent/go-mock=$MUT
 exec go mod tidy
-! exec go run github.com/Versent/go-mock/cmd/mockgen -undefined
+! exec mockgen -undefined
 
 cmp stdout stdout.golden
 cmp stderr stderr.golden
@@ -18,7 +18,6 @@ gen [-header file] [-tags buildtags] [package ...]
     	path to file to insert as a header in mock_gen.go
   -tags string
     	append build tags to the default mockstub
-exit status 2
 -- go.mod --
 module test
 

cmd/mockgen/testdata/fallback_header.txt

@@ -1,6 +1,6 @@
 exec go mod edit -replace github.com/Versent/go-mock=$MUT
 exec go mod tidy
-exec go run github.com/Versent/go-mock/cmd/mockgen -header HEADER
+exec mockgen -header HEADER
 
 cmp    stdout stdout.golden
 cmpenv stderr stderr.golden

cmd/mockgen/testdata/fallback_packages_missing.txt

@@ -1,6 +1,6 @@
 exec go mod edit -replace github.com/Versent/go-mock=$MUT
 exec go mod tidy
-! exec go run github.com/Versent/go-mock/cmd/mockgen ./missing
+! exec mockgen ./missing
 
 cmp    stdout stdout.golden
 cmpenv stderr stderr.golden
@@ -9,7 +9,6 @@ cmpenv stderr stderr.golden
 -- stderr.golden --
 mockgen: -: stat $WORK/missing: directory not found
 mockgen: generate failed
-exit status 1
 -- go.mod --
 module test
 

cmd/mockgen/testdata/fallback_tags.txt

@@ -1,6 +1,6 @@
 exec go mod edit -replace github.com/Versent/go-mock=$MUT
 exec go mod tidy
-exec go run github.com/Versent/go-mock/cmd/mockgen -tags extra
+exec mockgen -tags extra
 
 cmp    stdout stdout.golden
 cmpenv stderr stderr.golden

cmd/mockgen/testdata/flags.txt

@@ -1,6 +1,6 @@
 exec go mod edit -replace github.com/Versent/go-mock=$MUT
 exec go mod tidy
-exec go run github.com/Versent/go-mock/cmd/mockgen flags
+exec mockgen flags
 
 cmp stdout stdout.golden
 cmp stderr stderr.golden

cmd/mockgen/testdata/flags_gen.txt

@@ -1,6 +1,6 @@
 exec go mod edit -replace github.com/Versent/go-mock=$MUT
 exec go mod tidy
-exec go run github.com/Versent/go-mock/cmd/mockgen flags gen
+exec mockgen flags gen
 
 cmp stdout stdout.golden
 cmp stderr stderr.golden

cmd/mockgen/testdata/gen.txt

@@ -1,6 +1,6 @@
 exec go mod edit -replace github.com/Versent/go-mock=$MUT
 exec go mod tidy
-exec go run github.com/Versent/go-mock/cmd/mockgen gen
+exec mockgen gen
 
 cmp    stdout stdout.golden
 cmpenv stderr stderr.golden

cmd/mockgen/testdata/gen_all_args.txt

@@ -1,6 +1,6 @@
 exec go mod edit -replace github.com/Versent/go-mock=$MUT
 exec go mod tidy
-exec go run github.com/Versent/go-mock/cmd/mockgen gen -header HEADER -tags extra .
+exec mockgen gen -header HEADER -tags extra .
 
 cmp    stdout stdout.golden
 cmpenv stderr stderr.golden

cmd/mockgen/testdata/gen_flag_undefined.txt

@@ -1,6 +1,6 @@
 exec go mod edit -replace github.com/Versent/go-mock=$MUT
 exec go mod tidy
-! exec go run github.com/Versent/go-mock/cmd/mockgen gen -undefined
+! exec mockgen gen -undefined
 
 cmp stdout stdout.golden
 cmp stderr stderr.golden
@@ -18,7 +18,6 @@ gen [-header file] [-tags buildtags] [package ...]
     	path to file to insert as a header in mock_gen.go
   -tags string
     	append build tags to the default mockstub
-exit status 2
 -- go.mod --
 module test
 

cmd/mockgen/testdata/gen_header.txt

@@ -1,6 +1,6 @@
 exec go mod edit -replace github.com/Versent/go-mock=$MUT
 exec go mod tidy
-exec go run github.com/Versent/go-mock/cmd/mockgen gen -header HEADER
+exec mockgen gen -header HEADER
 
 cmp    stdout stdout.golden
 cmpenv stderr stderr.golden

cmd/mockgen/testdata/gen_package.txt

@@ -1,6 +1,6 @@
 exec go mod edit -replace github.com/Versent/go-mock=$MUT
 exec go mod tidy
-exec go run github.com/Versent/go-mock/cmd/mockgen gen .
+exec mockgen gen .
 
 cmp    stdout stdout.golden
 cmpenv stderr stderr.golden

cmd/mockgen/testdata/gen_packages_missing.txt

@@ -1,6 +1,6 @@
 exec go mod edit -replace github.com/Versent/go-mock=$MUT
 exec go mod tidy
-! exec go run github.com/Versent/go-mock/cmd/mockgen gen ./missing
+! exec mockgen gen ./missing
 
 cmp    stdout stdout.golden
 cmpenv stderr stderr.golden
@@ -9,7 +9,6 @@ cmpenv stderr stderr.golden
 -- stderr.golden --
 mockgen: -: stat $WORK/missing: directory not found
 mockgen: generate failed
-exit status 1
 -- go.mod --
 module test
 

cmd/mockgen/testdata/gen_tags.txt

@@ -1,6 +1,6 @@
 exec go mod edit -replace github.com/Versent/go-mock=$MUT
 exec go mod tidy
-exec go run github.com/Versent/go-mock/cmd/mockgen gen -tags extra
+exec mockgen gen -tags extra
 
 cmp    stdout stdout.golden
 cmpenv stderr stderr.golden

cmd/mockgen/testdata/help.txt

@@ -1,6 +1,6 @@
 exec go mod edit -replace github.com/Versent/go-mock=$MUT
 exec go mod tidy
-exec go run github.com/Versent/go-mock/cmd/mockgen help
+exec mockgen help
 
 cmp stdout stdout.golden
 cmp stderr stderr.golden

cmd/mockgen/testdata/report.sh

@@ -0,0 +1 @@
+go tool covdata percent -i=$GOCOVERDIR

cmd/mockgen/testdata/setup.sh

@@ -0,0 +1 @@
+exec go build ${GOCOVERDIR:+-cover} -o $BINDIR/mockgen .

internal/cmd/mockgen/gen.go

@@ -62,15 +62,14 @@ func (cmd *GenCmd) Execute(ctx context.Context, f *flag.FlagSet, args ...any) su
 		mock.WithHeaderFile(cmd.headerFile),
 		mock.WithArgs(args...),
 		mock.WithWDFallback(),
+		mock.WithPrefixFileName(cmd.prefixFileName),
+		mock.WithTags(cmd.tags),
 	)(&opts)
 	if err != nil {
 		cmd.log.Println(err)
 		return subcommands.ExitFailure
 	}
 
-	opts.PrefixOutputFile = cmd.prefixFileName
-	opts.Tags = cmd.tags
-
 	outs, errs := mock.Generate(ctx, packages(f), opts)
 	if len(errs) > 0 {
 		logErrors(cmd.log, errs...)