feat: new setting for using GRE mesh for experiments

* `useGREMesh` option in experiment config
* root `--use-gre-mesh` option in CLI (honors server setting)
* honors default bridge setting when creating the GRE mesh

src/go/api/experiment/experiment.go

@@ -64,6 +64,8 @@ func init() {
 				return fmt.Errorf("initializing experiment: %w", err)
 			}
 
+			exp.Spec.SetUseGREMesh(exp.Spec.UseGREMesh() || common.UseGREMesh)
+
 			existing, _ := types.Experiments(false)
 			for _, other := range existing {
 				if other.Metadata.Name == exp.Metadata.Name {
@@ -98,6 +100,8 @@ func init() {
 				return fmt.Errorf("re-initializing experiment (after update): %w", err)
 			}
 
+			exp.Spec.SetUseGREMesh(exp.Spec.UseGREMesh() || common.UseGREMesh)
+
 			existing, _ := types.Experiments(false)
 			for _, other := range existing {
 				if other.Metadata.Name == exp.Metadata.Name {
@@ -323,6 +327,7 @@ func Create(ctx context.Context, opts ...CreateOption) error {
 	exp.Spec.SetVLANRange(o.vlanMin, o.vlanMax, false)
 	exp.Spec.VLANs().SetAliases(o.vlanAliases)
 	exp.Spec.SetSchedule(o.schedules)
+	exp.Spec.SetUseGREMesh(o.useGREMesh)
 
 	c.Spec = structs.MapDefaultCase(exp.Spec, structs.CASESNAKE)
 
@@ -782,6 +787,13 @@ func Reconfigure(name string) error {
 		return fmt.Errorf("updating experiment config: %w", err)
 	}
 
+	// Try deleting the minimega bridge associated with this experiment if we're
+	// not using the GRE mesh, just in case we were using it prior. Ignore any
+	// errors since they will occur if GRE wasn't being used.
+	if !exp.Spec.UseGREMesh() {
+		mm.MeshSend(name, "", fmt.Sprintf("ns del-bridge %s", exp.Spec.DefaultBridge()))
+	}
+
 	return nil
 }
 

src/go/api/experiment/option.go

@@ -19,12 +19,14 @@ type createOptions struct {
 	schedules     map[string]string
 	baseDir       string
 	deployMode    common.DeploymentMode
+	useGREMesh    bool
 	defaultBridge string
 }
 
 func newCreateOptions(opts ...CreateOption) createOptions {
 	o := createOptions{
 		deployMode: common.DeployMode,
+		useGREMesh: common.UseGREMesh,
 	}
 
 	for _, opt := range opts {
@@ -108,6 +110,13 @@ func CreateWithDeployMode(m common.DeploymentMode) CreateOption {
 	}
 }
 
+func CreateWithGREMesh(g bool) CreateOption {
+	return func(o *createOptions) {
+		// Keep use GRE mesh enabled if enabled globally.
+		o.useGREMesh = o.useGREMesh || g
+	}
+}
+
 func CreateWithDefaultBridge(b string) CreateOption {
 	return func(o *createOptions) {
 		o.defaultBridge = b

src/go/cmd/root.go

@@ -39,6 +39,11 @@ var rootCmd = &cobra.Command{
 	PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
 		common.UnixSocket = viper.GetString("unix-socket")
 
+		// Initialize use GRE mesh with option set locally by user. Later it will be
+		// forcefully enabled if it's enabled at the server. This must be done
+		// before getting options from the server (unlike deploy mode option).
+		common.UseGREMesh = viper.GetBool("use-gre-mesh")
+
 		// check for global options set by UI server
 		if common.UnixSocket != "" {
 			cli := http.Client{
@@ -56,32 +61,32 @@ var rootCmd = &cobra.Command{
 					var options map[string]any
 					json.Unmarshal(body, &options)
 
-					if mode, _ := options["deploy-mode"].(string); mode != "" {
-						if deployMode, err := common.ParseDeployMode(mode); err == nil {
-							common.DeployMode = deployMode
-						}
+					mode, _ := options["deploy-mode"].(string)
+					if err := common.SetDeployMode(mode); err != nil {
+						return fmt.Errorf("setting server-specified deploy mode: %w", err)
 					}
+
+					// Enable use GRE mesh if enabled either locally or at server.
+					gre, _ := options["use-gre-mesh"].(bool)
+					common.UseGREMesh = common.UseGREMesh || gre
 				}
 			}
 		}
 
+		// Override deploy mode option from UI server if set locally by user. This
+		// must be done after getting options from the server (unlike use GRE mesh
+		// option).
+		if err := common.SetDeployMode(viper.GetString("deploy-mode")); err != nil {
+			return fmt.Errorf("setting user-specified deploy mode: %w", err)
+		}
+
 		plog.NewPhenixHandler()
 		plog.SetLevelText(viper.GetString("log.level"))
 
 		common.PhenixBase = viper.GetString("base-dir.phenix")
 		common.MinimegaBase = viper.GetString("base-dir.minimega")
 		common.HostnameSuffixes = viper.GetString("hostname-suffixes")
 
-		// if deploy mode option is set locally by user, use it instead of global from UI
-		if opt := viper.GetString("deploy-mode"); opt != "" {
-			mode, err := common.ParseDeployMode(opt)
-			if err != nil {
-				return fmt.Errorf("parsing deploy mode: %w", err)
-			}
-
-			common.DeployMode = mode
-		}
-
 		var (
 			endpoint = viper.GetString("store.endpoint")
 			errFile  = viper.GetString("log.error-file")
@@ -174,6 +179,7 @@ func init() {
 	rootCmd.PersistentFlags().Bool("log.error-stderr", true, "log fatal errors to STDERR")
 	rootCmd.PersistentFlags().String("log.level", "info", "level to log messages at")
 	rootCmd.PersistentFlags().String("deploy-mode", "", "deploy mode for minimega VMs (options: all | no-headnode | only-headnode)")
+	rootCmd.PersistentFlags().Bool("use-gre-mesh", false, "use GRE tunnels between mesh nodes for VLAN trunking")
 	rootCmd.PersistentFlags().String("unix-socket", "/tmp/phenix.sock", "phēnix unix socket to listen on (ui subcommand) or connect to")
 
 	if uid == "0" {

src/go/tmpl/templates/minimega_script.tmpl

@@ -1,10 +1,6 @@
 namespace {{ .ExperimentName }}
 ns queueing true
 
-{{- if ne .DefaultBridge "phenix" }}
-ns bridge {{ .DefaultBridge }} gre
-{{- end }}
-
 {{- if and (ne .VLANs.Min 0) (ne .VLANs.Max 0) }}
 vlans range {{ .VLANs.Min }} {{ .VLANs.Max }}
 {{- end }}
@@ -22,6 +18,10 @@ ns del-host all
 ns add-host localhost
 {{- end }}
 
+{{- if or (ne .DefaultBridge "phenix") .UseGREMesh }}
+ns bridge {{ .DefaultBridge }} gre
+{{- end }}
+
 {{- $basedir := .BaseDir }}
 
 {{- range .Topology.Nodes }}

src/go/types/interfaces/experiment.go

@@ -25,6 +25,7 @@ type ExperimentSpec interface {
 	VLANs() VLANSpec
 	Schedules() map[string]string
 	DeployMode() string
+	UseGREMesh() bool
 
 	SetExperimentName(string)
 	SetBaseDir(string)
@@ -35,6 +36,7 @@ type ExperimentSpec interface {
 	SetTopology(TopologySpec)
 	SetScenario(ScenarioSpec)
 	SetDeployMode(string)
+	SetUseGREMesh(bool)
 
 	VerifyScenario(context.Context) error
 	ScheduleNode(string, string) error

src/go/types/interfaces/topology.go

@@ -16,6 +16,7 @@ type TopologySpec interface {
 
 	HasCommands() bool
 
+	// accepts name of default bridge
 	Init(string) error
 }
 

src/go/types/version/v1/experiment.go

@@ -81,6 +81,7 @@ type ExperimentSpec struct {
 	VLANsF          *VLANSpec         `json:"vlans" yaml:"vlans" structs:"vlans" mapstructure:"vlans"`
 	SchedulesF      map[string]string `json:"schedules" yaml:"schedules" structs:"schedules" mapstructure:"schedules"`
 	DeployModeF     string            `json:"deployMode" yaml:"deployMode" structs:"deployMode" mapstructure:"deployMode"`
+	UseGREMeshF     bool              `json:"useGREMesh" yaml:"useGREMesh" structs:"useGREMesh" mapstructure:"useGREMesh"`
 }
 
 func (this *ExperimentSpec) Init() error {
@@ -196,6 +197,10 @@ func (this *ExperimentSpec) SetDefaultBridge(bridge string) {
 	this.DefaultBridgeF = bridge
 }
 
+func (this ExperimentSpec) UseGREMesh() bool {
+	return this.UseGREMeshF
+}
+
 func (this *ExperimentSpec) SetVLANAlias(a string, i int, f bool) error {
 	if this.VLANsF == nil {
 		this.VLANsF = &VLANSpec{AliasesF: make(map[string]int)}
@@ -256,6 +261,10 @@ func (this *ExperimentSpec) SetScenario(scenario ifaces.ScenarioSpec) {
 	this.ScenarioF = scenario.(*v2.ScenarioSpec)
 }
 
+func (this *ExperimentSpec) SetUseGREMesh(g bool) {
+	this.UseGREMeshF = g
+}
+
 func (this ExperimentSpec) VerifyScenario(ctx context.Context) error {
 	if this.ScenarioF == nil {
 		return nil

src/go/util/common/common.go

@@ -26,6 +26,8 @@ var (
 
 	StoreEndpoint    string
 	HostnameSuffixes string
+
+	UseGREMesh bool
 )
 
 func TrimHostnameSuffixes(str string) string {
@@ -44,7 +46,20 @@ func ParseDeployMode(mode string) (DeploymentMode, error) {
 		return DEPLOY_MODE_ONLY_HEADNODE, nil
 	case "all":
 		return DEPLOY_MODE_ALL, nil
+	case "": // default to current setting
+		return DeployMode, nil
 	}
 
 	return DEPLOY_MODE_UNSET, fmt.Errorf("unknown deploy mode provided: %s", mode)
 }
+
+func SetDeployMode(mode string) error {
+	parsed, err := ParseDeployMode(mode)
+	if err != nil {
+		return fmt.Errorf("setting deploy mode: %w", err)
+	}
+
+	DeployMode = parsed
+
+	return nil
+}

src/go/web/handlers.go

@@ -187,6 +187,7 @@ func CreateExperiment(w http.ResponseWriter, r *http.Request) {
 		experiment.CreatedWithDisabledApplications(req.DisabledApps),
 		experiment.CreateWithDeployMode(deployMode),
 		experiment.CreateWithDefaultBridge(req.DefaultBridge),
+		experiment.CreateWithGREMesh(req.UseGreMesh),
 	}
 
 	if req.WorkflowBranch != "" {

src/go/web/option.go

@@ -188,7 +188,8 @@ func GetOptions(w http.ResponseWriter, r *http.Request) error {
 	}
 
 	options := map[string]any{
-		"deploy-mode": common.DeployMode,
+		"deploy-mode":  common.DeployMode,
+		"use-gre-mesh": common.UseGREMesh,
 	}
 
 	body, err := json.Marshal(options)

src/go/web/proto/experiment.proto

@@ -72,6 +72,7 @@ message CreateExperimentRequest {
 	repeated string disabled_apps = 7 [json_name="disabled_apps"];
 	string deploy_mode = 8 [json_name="deploy_mode"];
 	string default_bridge = 9 [json_name="default_bridge"];
+	bool use_gre_mesh = 10 [json_name="use_gre_mesh"];
 }
 
 message SnapshotRequest {

src/go/web/workflow.go

@@ -140,6 +140,7 @@ func ApplyWorkflow(w http.ResponseWriter, r *http.Request) error {
 			experiment.CreateWithVLANMax(wf.VLANMax()),
 			experiment.CreateWithDeployMode(wf.ExperimentDeployMode()),
 			experiment.CreateWithDefaultBridge(wf.DefaultBridgeName()),
+			experiment.CreateWithGREMesh(wf.UseGREMesh),
 		}
 
 		if err := experiment.Create(ctx, opts...); err != nil {
@@ -317,6 +318,7 @@ func ApplyWorkflow(w http.ResponseWriter, r *http.Request) error {
 		exp.Spec.SetSchedule(schedules)
 		exp.Spec.SetDeployMode(string(wf.ExperimentDeployMode()))
 		exp.Spec.SetVLANRange(wf.VLANMin(), wf.VLANMax(), true)
+		exp.Spec.SetUseGREMesh(wf.UseGREMesh)
 
 		if err := exp.WriteToStore(false); err != nil {
 			err := weberror.NewWebError(err, "unable to write updated experiment %s", expName)
@@ -495,6 +497,7 @@ type workflow struct {
 	VLANs      map[string]int    `mapstructure:"vlans"`
 	Schedules  map[string]string `mapstructue:"schedules"`
 	DeployMode string            `mapstructure:"deployMode"`
+	UseGREMesh bool              `mapstructure:"useGREMesh"`
 
 	VLANRange *struct {
 		Min int `mapstructure:"min"`