Merge cockroachdb#124405

124405: mixedversion: introduce deployment modes r=herkolategan,DarrylWong,srosenberg a=renatolabs

This commit adds the notion of a "deployment mode" to mixedversion tests. Currently, we only define a `system-only` deployment mode, corresponding to the deployment performed on all mixedversion tests to date: cockroach is started without any tenants (virtual clusters), and client applications talk directly to the "system interface". This commit adds the functionality necessary for tests to be able to select what deployment modes make sense for them, if they verify something specific to some deployment mode.

In the future, more deployment modes will be added: tenants in shared-process mode, and external process (serverless-style) tenants.

Epic: none

Release note: None

Co-authored-by: Renato Costa <[email protected]>

pkg/cmd/roachtest/roachtestutil/mixedversion/mixedversion.go

@@ -124,6 +124,10 @@ const (
 	// cluster that can use the test fixtures in
 	// `pkg/cmd/roachtest/fixtures`.
 	numNodesInFixtures = 4
+
+	// These `*Deployment` constants are used to indicate different
+	// deployment modes that a test may choose to enable/disable.
+	SystemOnlyDeployment = DeploymentMode("system-only")
 )
 
 var (
@@ -249,6 +253,7 @@ type (
 		minimumSupportedVersion        *clusterupgrade.Version
 		predecessorFunc                predecessorFunc
 		settings                       []install.ClusterSettingOption
+		enabledDeploymentModes         []DeploymentMode
 		overriddenMutatorProbabilities map[string]float64
 	}
 
@@ -301,6 +306,8 @@ type (
 	// authors when they want to stop a background step as part of test
 	// logic itself, without causing the test to fail.
 	StopFunc func()
+
+	DeploymentMode string
 )
 
 // NeverUseFixtures is an option that can be passed to `NewTest` to
@@ -372,6 +379,16 @@ func ClusterSettingOption(opt ...install.ClusterSettingOption) CustomOption {
 	}
 }
 
+// EnabledDeploymentModes restricts the test to only run in the
+// provided deployment modes, in case the test is incompatible with a
+// certain mode. Each test run will pick a random deployment mode
+// within the list of enabled modes.
+func EnabledDeploymentModes(modes ...DeploymentMode) CustomOption {
+	return func(opts *testOptions) {
+		opts.enabledDeploymentModes = modes
+	}
+}
+
 // AlwaysUseLatestPredecessors allows test authors to opt-out of
 // testing upgrades from random predecessor patch releases. The
 // default is to pick a random (non-withdrawn) patch release for
@@ -413,6 +430,7 @@ func defaultTestOptions() testOptions {
 		maxUpgrades:                    4,
 		minimumSupportedVersion:        OldestSupportedVersion,
 		predecessorFunc:                randomPredecessorHistory,
+		enabledDeploymentModes:         []DeploymentMode{SystemOnlyDeployment},
 		overriddenMutatorProbabilities: make(map[string]float64),
 	}
 }
@@ -588,7 +606,7 @@ func (t *Test) Run() {
 		t.rt.Fatal(fmt.Errorf("error creating test plan: %w", err))
 	}
 
-	t.logger.Printf(plan.PrettyPrint())
+	t.logger.Printf("mixed-version test:\n%s", plan.PrettyPrint())
 
 	if err := t.run(plan); err != nil {
 		t.rt.Fatal(err)
@@ -607,9 +625,14 @@ func (t *Test) plan() (*TestPlan, error) {
 		return nil, err
 	}
 
+	// Pick a random deployment mode to use in this test run among the
+	// list of enabled deployment modes enabled for this test.
+	deploymentMode := t.options.enabledDeploymentModes[t.prng.Intn(len(t.options.enabledDeploymentModes))]
 	initialRelease := previousReleases[0]
+
 	planner := testPlanner{
 		versions:       append(previousReleases, clusterupgrade.CurrentVersion()),
+		deploymentMode: deploymentMode,
 		currentContext: newInitialContext(initialRelease, t.crdbNodes, nil /* tenant */),
 		options:        t.options,
 		rt:             t.rt,
@@ -894,20 +917,26 @@ func rngFromRNG(rng *rand.Rand) *rand.Rand {
 }
 
 func assertValidTest(test *Test, fatalFunc func(...interface{})) {
+	fail := func(err error) {
+		fatalFunc(errors.Wrap(err, "mixedversion.NewTest"))
+	}
+
 	if test.options.useFixturesProbability > 0 && len(test.crdbNodes) != numNodesInFixtures {
-		err := fmt.Errorf(
-			"invalid cluster: use of fixtures requires %d cockroach nodes, got %d (%v)",
-			numNodesInFixtures, len(test.crdbNodes), test.crdbNodes,
+		fail(
+			fmt.Errorf(
+				"invalid cluster: use of fixtures requires %d cockroach nodes, got %d (%v)",
+				numNodesInFixtures, len(test.crdbNodes), test.crdbNodes,
+			),
 		)
-		fatalFunc(errors.Wrap(err, "mixedversion.NewTest"))
 	}
 
 	if test.options.minUpgrades > test.options.maxUpgrades {
-		err := fmt.Errorf(
-			"invalid test options: maxUpgrades (%d) must be greater than minUpgrades (%d)",
-			test.options.maxUpgrades, test.options.minUpgrades,
+		fail(
+			fmt.Errorf(
+				"invalid test options: maxUpgrades (%d) must be greater than minUpgrades (%d)",
+				test.options.maxUpgrades, test.options.minUpgrades,
+			),
 		)
-		fatalFunc(errors.Wrap(err, "mixedversion.NewTest"))
 	}
 
 	currentVersion := clusterupgrade.CurrentVersion()
@@ -919,10 +948,15 @@ func assertValidTest(test *Test, fatalFunc func(...interface{})) {
 		(msv.Major() == currentVersion.Major() && msv.Minor() < currentVersion.Minor())
 
 	if !validVersion {
-		err := fmt.Errorf(
-			"invalid test options: minimum supported version (%s) should be from an older release series than current version (%s)",
-			msv.Version.String(), currentVersion.Version.String(),
+		fail(
+			fmt.Errorf(
+				"invalid test options: minimum supported version (%s) should be from an older release series than current version (%s)",
+				msv.Version.String(), currentVersion.Version.String(),
+			),
 		)
-		fatalFunc(errors.Wrap(err, "mixedversion.NewTest"))
+	}
+
+	if len(test.options.enabledDeploymentModes) == 0 {
+		fail(fmt.Errorf("invalid test options: no deployment modes enabled"))
 	}
 }

pkg/cmd/roachtest/roachtestutil/mixedversion/mixedversion_test.go

@@ -86,6 +86,15 @@ func Test_assertValidTest(t *testing.T) {
 		fatalErr.Error(),
 	)
 
+	// no deployment mode is enabled for a test.
+	mvt = newTest(EnabledDeploymentModes())
+	assertValidTest(mvt, fatalFunc())
+	require.Error(t, fatalErr)
+	require.Equal(t,
+		"mixedversion.NewTest: invalid test options: no deployment modes enabled",
+		fatalErr.Error(),
+	)
+
 	mvt = newTest(MinimumSupportedVersion("v22.2.0"))
 	assertValidTest(mvt, fatalFunc())
 	require.NoError(t, fatalErr)

pkg/cmd/roachtest/roachtestutil/mixedversion/planner.go

@@ -41,6 +41,9 @@ type (
 		// the run encoded by this plan. These upgrades happen *after*
 		// every update in `setup` is finished.
 		upgrades []*upgradePlan
+		// deploymentMode is the associated deployment mode used when
+		// generating this test plan.
+		deploymentMode DeploymentMode
 		// enabledMutators is a list of `mutator` implementations that
 		// were applied when generating this test plan.
 		enabledMutators []mutator
@@ -63,6 +66,7 @@ type (
 	// a test plan from the given rng and user-provided hooks.
 	testPlanner struct {
 		versions       []*clusterupgrade.Version
+		deploymentMode DeploymentMode
 		currentContext *Context
 		crdbNodes      option.NodeListOption
 		rt             test.Test
@@ -257,10 +261,11 @@ func (p *testPlanner) Plan() *TestPlan {
 	}
 
 	testPlan := &TestPlan{
-		setup:     setup,
-		initSteps: p.testStartSteps(),
-		upgrades:  testUpgrades,
-		isLocal:   p.isLocal,
+		setup:          setup,
+		initSteps:      p.testStartSteps(),
+		upgrades:       testUpgrades,
+		deploymentMode: p.deploymentMode,
+		isLocal:        p.isLocal,
 	}
 
 	// Probabilistically enable some of of the mutators on the base test
@@ -940,22 +945,30 @@ func (plan *TestPlan) prettyPrintInternal(debug bool) string {
 	versions := plan.Versions()
 	formattedVersions := make([]string, 0, len(versions))
 	for _, v := range versions {
-		formattedVersions = append(formattedVersions, fmt.Sprintf("%q", v.String()))
+		formattedVersions = append(formattedVersions, v.String())
 	}
 
-	var mutatorsDesc string
+	var lines []string
+	addLine := func(title string, val any) {
+		titleWithColon := fmt.Sprintf("%s:", title)
+		lines = append(lines, fmt.Sprintf("%-20s%v", titleWithColon, val))
+	}
+
+	addLine("Upgrades", strings.Join(formattedVersions, " → "))
+	addLine("Deployment mode", plan.deploymentMode)
+
 	if len(plan.enabledMutators) > 0 {
 		mutatorNames := make([]string, 0, len(plan.enabledMutators))
 		for _, mut := range plan.enabledMutators {
 			mutatorNames = append(mutatorNames, mut.Name())
 		}
 
-		mutatorsDesc = fmt.Sprintf(" with mutators {%s}", strings.Join(mutatorNames, ", "))
+		addLine("Mutators", strings.Join(mutatorNames, ", "))
 	}
 
 	return fmt.Sprintf(
-		"mixed-version test plan for upgrading from %s%s:\n%s",
-		strings.Join(formattedVersions, " to "), mutatorsDesc, out.String(),
+		"%s\nPlan:\n%s",
+		strings.Join(lines, "\n"), out.String(),
 	)
 }
 

pkg/cmd/roachtest/roachtestutil/mixedversion/planner_test.go

@@ -82,7 +82,10 @@ func TestTestPlanner(t *testing.T) {
 
 	datadriven.Walk(t, datapathutils.TestDataPath(t, "planner"), func(t *testing.T, path string) {
 		resetMutators()
-		mvt := newTest()
+		// Unless specified, treat every test as a non-UA deployment
+		// test. Tests can use the deployment-mode option in the
+		// mixed-version-test directive to change the deployment mode.
+		mvt := newTest(EnabledDeploymentModes(SystemOnlyDeployment))
 
 		datadriven.RunTest(t, path, func(t *testing.T, d *datadriven.TestData) string {
 			if d.Cmd == "plan" {
@@ -204,7 +207,7 @@ func TestDeterministicHookSeeds(t *testing.T) {
 		plan, err := mvt.plan()
 		require.NoError(t, err)
 
-		upgradeStep := plan.Steps()[3].(sequentialRunStep)
+		upgradeStep := plan.Steps()[2].(sequentialRunStep)
 
 		// We can hardcode these paths since we are using a fixed seed in
 		// these tests.
@@ -441,7 +444,7 @@ func Test_stepSelectorFilter(t *testing.T) {
 			name:                   "no filter",
 			predicate:              func(*singleStep) bool { return true },
 			expectedAllSteps:       true,
-			expectedRandomStepType: runHookStep{},
+			expectedRandomStepType: restartWithNewBinaryStep{},
 		},
 		{
 			name: "filter eliminates all steps",

pkg/cmd/roachtest/roachtestutil/mixedversion/testdata/planner/basic_test_mixed_version_hooks

@@ -23,44 +23,42 @@ ok
 
 plan
 ----
-mixed-version test plan for upgrading from "v22.2.8" to "<current>":
+Upgrades:           v22.2.8 → <current>
+Deployment mode:    system-only
+Plan:
 ├── install fixtures for version "v22.2.8" (1)
 ├── start cluster at version "v22.2.8" (2)
 ├── wait for system tenant on nodes :1-4 to reach cluster version '22.2' (3)
 ├── run startup hooks concurrently
 │   ├── run "initialize bank workload", after 0s delay (4)
-│   └── run "initialize rand workload", after 3m0s delay (5)
+│   └── run "initialize rand workload", after 30s delay (5)
 ├── start background hooks concurrently
-│   ├── run "bank workload", after 500ms delay (6)
+│   ├── run "bank workload", after 0s delay (6)
 │   ├── run "rand workload", after 0s delay (7)
-│   └── run "csv server", after 0s delay (8)
+│   └── run "csv server", after 3m0s delay (8)
 └── upgrade cluster from "v22.2.8" to "<current>"
    ├── prevent auto-upgrades on system tenant by setting `preserve_downgrade_option` (9)
    ├── upgrade nodes :1-4 from "v22.2.8" to "<current>"
-   │   ├── restart node 2 with binary version <current> (10)
-   │   ├── run mixed-version hooks concurrently
-   │   │   ├── run "mixed-version 1", after 3m0s delay (11)
-   │   │   └── run "mixed-version 2", after 5s delay (12)
-   │   ├── restart node 4 with binary version <current> (13)
-   │   ├── restart node 1 with binary version <current> (14)
-   │   └── restart node 3 with binary version <current> (15)
+   │   ├── restart node 4 with binary version <current> (10)
+   │   ├── run "mixed-version 1" (11)
+   │   ├── restart node 3 with binary version <current> (12)
+   │   ├── restart node 2 with binary version <current> (13)
+   │   ├── run "mixed-version 2" (14)
+   │   └── restart node 1 with binary version <current> (15)
    ├── downgrade nodes :1-4 from "<current>" to "v22.2.8"
-   │   ├── restart node 1 with binary version v22.2.8 (16)
-   │   ├── restart node 3 with binary version v22.2.8 (17)
-   │   ├── restart node 4 with binary version v22.2.8 (18)
-   │   ├── run mixed-version hooks concurrently
-   │   │   ├── run "mixed-version 1", after 500ms delay (19)
-   │   │   └── run "mixed-version 2", after 100ms delay (20)
-   │   └── restart node 2 with binary version v22.2.8 (21)
+   │   ├── restart node 3 with binary version v22.2.8 (16)
+   │   ├── run "mixed-version 1" (17)
+   │   ├── restart node 2 with binary version v22.2.8 (18)
+   │   ├── restart node 4 with binary version v22.2.8 (19)
+   │   ├── run "mixed-version 2" (20)
+   │   └── restart node 1 with binary version v22.2.8 (21)
    ├── upgrade nodes :1-4 from "v22.2.8" to "<current>"
-   │   ├── restart node 2 with binary version <current> (22)
-   │   ├── run "mixed-version 2" (23)
-   │   ├── restart node 3 with binary version <current> (24)
-   │   ├── restart node 1 with binary version <current> (25)
-   │   ├── restart node 4 with binary version <current> (26)
-   │   └── run "mixed-version 1" (27)
+   │   ├── restart node 1 with binary version <current> (22)
+   │   ├── restart node 2 with binary version <current> (23)
+   │   ├── restart node 4 with binary version <current> (24)
+   │   ├── run "mixed-version 1" (25)
+   │   ├── restart node 3 with binary version <current> (26)
+   │   └── run "mixed-version 2" (27)
    ├── allow upgrade to happen by resetting `preserve_downgrade_option` (28)
-   ├── run mixed-version hooks concurrently
-   │   ├── run "mixed-version 1", after 30s delay (29)
-   │   └── run "mixed-version 2", after 5s delay (30)
-   └── wait for system tenant on nodes :1-4 to reach cluster version <current> (31)
+   ├── run "mixed-version 2" (29)
+   └── wait for system tenant on nodes :1-4 to reach cluster version <current> (30)