fix: don't overwrite exp status after app running stage executes

src/go/api/scorch/scorchexe/execute.go

@@ -51,6 +51,8 @@ func Execute(ctx context.Context, exp *types.Experiment, run int) error {
 		errors = multierror.Append(errors, fmt.Errorf("running Scorch for experiment %s: %w", exp.Metadata.Name, err))
 	}
 
+	exp.Reload() // reload experiment from store in case status was updated during run
+
 	exp.Status.SetAppRunning("scorch", false)
 	exp.Status.SetAppStatus("scorch", nil)
 

src/go/app/app.go

@@ -281,9 +281,11 @@ func ApplyApps(ctx context.Context, exp *types.Experiment, opts ...Option) error
 				err = a.Running(ctx, exp)
 				if err != nil {
 					pubsub.Publish("trigger-app", Publication{Experiment: exp.Spec.ExperimentName(), App: app.Name(), State: "error", Error: err})
+				} else {
+					pubsub.Publish("trigger-app", Publication{Experiment: exp.Spec.ExperimentName(), App: app.Name(), State: "success"})
 				}
 
-				pubsub.Publish("trigger-app", Publication{Experiment: exp.Spec.ExperimentName(), App: app.Name(), State: "success"})
+				exp.Reload() // reload experiment from store in case status was updated during run
 
 				exp.Status.SetAppRunning(app.Name(), false)
 

src/go/types/experiment.go

@@ -40,6 +40,28 @@ func NewExperiment(md store.ConfigMetadata) *Experiment {
 	}
 }
 
+func (this *Experiment) Reload() error {
+	c, err := store.NewConfig("experiment/" + this.Metadata.Name)
+	if err != nil {
+		return fmt.Errorf("getting experiment: %w", err)
+	}
+
+	if err := store.Get(c); err != nil {
+		return fmt.Errorf("getting experiment %s from store: %w", this.Metadata.Name, err)
+	}
+
+	exp, err := DecodeExperimentFromConfig(*c)
+	if err != nil {
+		return fmt.Errorf("decoding experiment %s: %w", this.Metadata.Name, err)
+	}
+
+	this.Metadata = exp.Metadata
+	this.Spec = exp.Spec
+	this.Status = exp.Status
+
+	return nil
+}
+
 func (this Experiment) WriteToStore(statusOnly bool) error {
 	name := this.Metadata.Name