Add server request body assertions to acceptance tests

acceptance/acceptance_test.go

@@ -19,6 +19,7 @@ import (
 	"github.com/databricks/cli/internal/testutil"
 	"github.com/databricks/cli/libs/env"
 	"github.com/databricks/cli/libs/testdiff"
+	"github.com/databricks/cli/libs/testserver"
 	"github.com/databricks/databricks-sdk-go"
 	"github.com/stretchr/testify/require"
 )
@@ -30,11 +31,16 @@ var KeepTmp bool
 // example: var SingleTest = "bundle/variables/empty"
 var SingleTest = ""
 
+// TODO: Add a custom diff function based on the method and path.
+
 // If enabled, instead of compiling and running CLI externally, we'll start in-process server that accepts and runs
 // CLI commands. The $CLI in test scripts is a helper that just forwards command-line arguments to this server (see bin/callserver.py).
 // Also disables parallelism in tests.
 var InprocessMode bool
 
+// TODO: Acceptance tests are taking long to run. Is it possible to shorten
+// that time?
+
 func init() {
 	flag.BoolVar(&InprocessMode, "inprocess", SingleTest != "", "Run CLI in the same process as test (for debugging)")
 	flag.BoolVar(&KeepTmp, "keeptmp", false, "Do not delete TMP directory after run")
@@ -104,10 +110,10 @@ func testAccept(t *testing.T, InprocessMode bool, singleTest string) int {
 	cloudEnv := os.Getenv("CLOUD_ENV")
 
 	if cloudEnv == "" {
-		server := StartServer(t)
-		AddHandlers(server)
+		defaultServer := StartServer(t)
+		AddHandlers(defaultServer)
 		// Redirect API access to local server:
-		t.Setenv("DATABRICKS_HOST", server.URL)
+		t.Setenv("DATABRICKS_HOST", defaultServer.URL)
 		t.Setenv("DATABRICKS_TOKEN", "dapi1234")
 
 		homeDir := t.TempDir()
@@ -124,6 +130,8 @@ func testAccept(t *testing.T, InprocessMode bool, singleTest string) int {
 	testdiff.PrepareReplacementsUser(t, &repls, *user)
 	testdiff.PrepareReplacementsWorkspaceClient(t, &repls, workspaceClient)
 	testdiff.PrepareReplacementsUUID(t, &repls)
+	testdiff.PrepareReplacementVersions(t, &repls)
+	testdiff.PrepareReplacementOperatingSystem(t, &repls)
 
 	testDirs := getTests(t)
 	require.NotEmpty(t, testDirs)
@@ -138,7 +146,7 @@ func testAccept(t *testing.T, InprocessMode bool, singleTest string) int {
 	for _, dir := range testDirs {
 		testName := strings.ReplaceAll(dir, "\\", "/")
 		t.Run(testName, func(t *testing.T) {
-			if !InprocessMode {
+			if !InprocessMode && !hasCustomServer(t, dir) {
 				t.Parallel()
 			}
 
@@ -149,6 +157,10 @@ func testAccept(t *testing.T, InprocessMode bool, singleTest string) int {
 	return len(testDirs)
 }
 
+func hasCustomServer(t *testing.T, dir string) bool {
+	return testutil.DetectFile(t, filepath.Join(dir, testserver.ConfigFileName))
+}
+
 func getTests(t *testing.T) []string {
 	testDirs := make([]string, 0, 128)
 
@@ -200,6 +212,17 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont
 	err = CopyDir(dir, tmpDir, inputs, outputs)
 	require.NoError(t, err)
 
+	// If there is a server.json file in the test directory, start a custom server.
+	// Redirect all API calls to this server.
+	var server *testserver.Server
+	if hasCustomServer(t, dir) {
+		server = testserver.NewFromConfig(t, dir)
+		t.Setenv("DATABRICKS_HOST", server.URL)
+		t.Cleanup(func() {
+			server.Close()
+		})
+	}
+
 	args := []string{"bash", "-euo", "pipefail", EntryPointScript}
 	cmd := exec.Command(args[0], args[1:]...)
 	if coverDir != "" {
@@ -224,6 +247,12 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont
 	formatOutput(out, err)
 	require.NoError(t, out.Close())
 
+	// Write the requests made to the server to disk if a custom server is defined.
+	if server != nil {
+		outRequestsPath := filepath.Join(tmpDir, "out.requests.json")
+		server.WriteRequestsToDisk(outRequestsPath)
+	}
+
 	printedRepls := false
 
 	// Compare expected outputs

acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/databricks.yml

@@ -3,6 +3,7 @@
 bundle:
   name: my_jobs_as_code
   uuid: <UUID>
+  databricks_cli_version: ">= 0.238.0"
 
 experimental:
   python:

acceptance/cmd_server_test.go

@@ -8,10 +8,11 @@ import (
 	"testing"
 
 	"github.com/databricks/cli/internal/testcli"
+	"github.com/databricks/cli/libs/testserver"
 	"github.com/stretchr/testify/require"
 )
 
-func StartCmdServer(t *testing.T) *TestServer {
+func StartCmdServer(t *testing.T) *testserver.Server {
 	server := StartServer(t)
 	server.Handle("/", func(r *http.Request) (any, error) {
 		q := r.URL.Query()

acceptance/server_test.go

@@ -1,73 +1,25 @@
 package acceptance_test
 
 import (
-	"encoding/json"
 	"net/http"
-	"net/http/httptest"
 	"testing"
 
+	"github.com/databricks/cli/libs/testserver"
 	"github.com/databricks/databricks-sdk-go/service/catalog"
 	"github.com/databricks/databricks-sdk-go/service/compute"
 	"github.com/databricks/databricks-sdk-go/service/iam"
 	"github.com/databricks/databricks-sdk-go/service/workspace"
 )
 
-type TestServer struct {
-	*httptest.Server
-	Mux *http.ServeMux
-}
-
-type HandlerFunc func(r *http.Request) (any, error)
-
-func NewTestServer() *TestServer {
-	mux := http.NewServeMux()
-	server := httptest.NewServer(mux)
-
-	return &TestServer{
-		Server: server,
-		Mux:    mux,
-	}
-}
-
-func (s *TestServer) Handle(pattern string, handler HandlerFunc) {
-	s.Mux.HandleFunc(pattern, func(w http.ResponseWriter, r *http.Request) {
-		resp, err := handler(r)
-		if err != nil {
-			http.Error(w, err.Error(), http.StatusInternalServerError)
-			return
-		}
-
-		w.Header().Set("Content-Type", "application/json")
-
-		var respBytes []byte
-
-		respString, ok := resp.(string)
-		if ok {
-			respBytes = []byte(respString)
-		} else {
-			respBytes, err = json.MarshalIndent(resp, "", "    ")
-			if err != nil {
-				http.Error(w, err.Error(), http.StatusInternalServerError)
-				return
-			}
-		}
-
-		if _, err := w.Write(respBytes); err != nil {
-			http.Error(w, err.Error(), http.StatusInternalServerError)
-			return
-		}
-	})
-}
-
-func StartServer(t *testing.T) *TestServer {
-	server := NewTestServer()
+func StartServer(t *testing.T) *testserver.Server {
+	server := testserver.New(t)
 	t.Cleanup(func() {
 		server.Close()
 	})
 	return server
 }
 
-func AddHandlers(server *TestServer) {
+func AddHandlers(server *testserver.Server) {
 	server.Handle("GET /api/2.0/policies/clusters/list", func(r *http.Request) (any, error) {
 		return compute.ListPoliciesResponse{
 			Policies: []compute.Policy{

acceptance/workspace/jobs/create/out.requests.json

@@ -0,0 +1,14 @@
+[
+    {
+        "method": "POST",
+        "path": "/api/2.1/jobs/create",
+        "body": {
+            "name": "abc"
+        },
+        "headers": {
+            "Authorization": "Bearer $DATABRICKS_TOKEN",
+            "Content-Type": "application/json",
+            "User-Agent": "cli/$CLI_VERSION+bd231012d146 databricks-sdk-go/$GO_SDK_VERSION go/$GO_VERSION os/$OPERATING_SYSTEM cmd/jobs_create cmd-exec-id/<UUID> auth/pat"
+        }
+    }
+]

acceptance/workspace/jobs/create/output.txt

@@ -0,0 +1,5 @@
+
+>>> $CLI jobs create --json {"name":"abc"}
+{
+  "job_id":1111
+}

acceptance/workspace/jobs/create/script

@@ -0,0 +1 @@
+trace $CLI jobs create --json '{"name":"abc"}'

acceptance/workspace/jobs/create/server.json

@@ -0,0 +1,11 @@
+[
+  {
+    "method": "POST",
+    "path": "/api/2.1/jobs/create",
+    "response": {
+      "body": {
+        "job_id": 1111
+      }
+    }
+  }
+]

internal/testutil/file.go

@@ -62,6 +62,18 @@ func StatFile(t TestingT, path string) os.FileInfo {
 	return fi
 }
 
+func DetectFile(t TestingT, path string) bool {
+	_, err := os.Stat(path)
+	if err == nil {
+		return true
+	}
+	if os.IsNotExist(err) {
+		return false
+	}
+	t.Fatalf("unexpected error: %v", err)
+	return false
+}
+
 // AssertFileContents asserts that the file at path has the expected content.
 func AssertFileContents(t TestingT, path, expected string) bool {
 	actual := ReadFile(t, path)

libs/testdiff/golden.go

@@ -14,7 +14,7 @@ import (
 var OverwriteMode = false
 
 func init() {
-	flag.BoolVar(&OverwriteMode, "update", false, "Overwrite golden files")
+	flag.BoolVar(&OverwriteMode, "update", true, "Overwrite golden files")
 }
 
 func ReadFile(t testutil.TestingT, ctx context.Context, filename string) string {

libs/testdiff/replacement.go

@@ -9,10 +9,13 @@ import (
 	"slices"
 	"strings"
 
+	"github.com/databricks/cli/internal/build"
 	"github.com/databricks/cli/internal/testutil"
 	"github.com/databricks/cli/libs/iamutil"
 	"github.com/databricks/databricks-sdk-go"
 	"github.com/databricks/databricks-sdk-go/service/iam"
+	"github.com/databricks/databricks-sdk-go/version"
+	"golang.org/x/mod/semver"
 )
 
 const (
@@ -211,3 +214,42 @@ func PrepareReplacementsTemporaryDirectory(t testutil.TestingT, r *ReplacementsC
 	t.Helper()
 	r.append(privatePathRegex, "/tmp/.../$3")
 }
+
+// TODO: Record server io for all acceptance tests.
+
+func PrepareReplacementVersions(t testutil.TestingT, r *ReplacementsContext) {
+	t.Helper()
+	r.Set(version.Version, "$GO_SDK_VERSION")
+
+	// TODO: Fix build metadata here? Otherwise lets set the build version manually
+	// in the tests.
+	// (0|[1-9]\d*)
+	// \.
+	// (0|[1-9]\d*)
+	// \.
+	// (0|[1-9]\d*)
+	// (?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?
+	// (?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?
+
+	// buildInfo := build.GetInfo()
+	// test build versions can contain build metadata in their semver. Add a regex to match that.
+	// TODO: This does not work. Fix it.
+	// cliVersionRegex := regexp.MustCompile(build.DefaultSemver)
+	// r.append(cliVersionRegex, "$CLI_VERSION")
+
+	r.Set(build.DefaultSemver, "$CLI_VERSION")
+
+	r.Set(goVersion(), "$GO_VERSION")
+}
+
+func PrepareReplacementOperatingSystem(t testutil.TestingT, r *ReplacementsContext) {
+	t.Helper()
+	r.Set(runtime.GOOS, "$OPERATING_SYSTEM")
+}
+
+func goVersion() string {
+	gv := runtime.Version()
+	ssv := strings.ReplaceAll(gv, "go", "v")
+	sv := semver.Canonical(ssv)
+	return strings.TrimPrefix(sv, "v")
+}