Skip to content

Commit

Permalink
feat(workflow): allow API to accept tags for experiments
Browse files Browse the repository at this point in the history
Users can now send key=value pairs via URL requests to the API when
applying a configuration file. Currently, any query in the form
'tag=key=value' will be stored as an experiment annotation named
`phenix.workflow/tags`, with the value being a simple coma-separated
string of the tags.

Take the following URL as an example.

```
/api/v1/workflow/apply/{BRANCH}?tag=key1=value1&tag=key2=value&foo=bar
```

This will result in the following annotation being added to the
experiment metadata.

```
metadata:
  annotations:
    "phenix.workflow/tags": "key1=value1,key2=value2"
```

Note that the `foo=bar` query parameter is ignored. These tags are then
recorded with other info after each scorch run.
  • Loading branch information
nblair2 authored and activeshadow committed Jul 24, 2023
1 parent d4dcb33 commit 57dbdc3
Show file tree
Hide file tree
Showing 5 changed files with 42 additions and 14 deletions.
5 changes: 4 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,9 @@ repository named `.phenix.yml`) should be applied. Applying the phenix workflow
config file will trigger existing experiments to be updated and restarted,
depending on settings within the workflow config file.

Optional tags can be passed as URL querries when the workflow config is applied.
These tags are stored as a string `key1=value1,key2=value2`.

### Add/Update Topology or Scenario Config

```
Expand All @@ -59,7 +62,7 @@ curl -XPOST -H "Content-Type: application/x-yaml" \
```
curl -XPOST -H "Content-Type: application/x-yaml" \
--data-binary @{/path/to/config/file.yml} \
http://localhost:3000/api/v1/workflow/apply/{branch name}
http://localhost:3000/api/v1/workflow/apply/{branch name}[?tag=key1=value1&tag=key2=value2]
```

### phenix Workflow Config File Documentation
Expand Down
17 changes: 11 additions & 6 deletions src/go/api/scorch/app.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import (
"phenix/api/scorch/scorchexe"
"phenix/api/scorch/scorchmd"
"phenix/app"
"phenix/store"
"phenix/types"
ifaces "phenix/types/interfaces"
"phenix/util"
Expand Down Expand Up @@ -159,7 +160,7 @@ func (this *Scorch) Running(ctx context.Context, exp *types.Experiment) error {
this.stopFilebeat(ctx, cmd, port)
}

if err := this.recordInfo(runID, runDir, exp.Metadata.Name, start); err != nil {
if err := this.recordInfo(runID, runDir, exp.Metadata, start); err != nil {
errors = multierror.Append(errors, err)
}

Expand Down Expand Up @@ -333,7 +334,7 @@ func (this Scorch) stopFilebeat(ctx context.Context, cmd *exec.Cmd, port int) {
}
}

func (this Scorch) recordInfo(runID int, runDir, name string, startTime time.Time) error {
func (this Scorch) recordInfo(runID int, runDir string, md store.ConfigMetadata, startTime time.Time) error {
c := mmcli.NewCommand()
c.Command = "version"

Expand All @@ -344,7 +345,7 @@ func (this Scorch) recordInfo(runID int, runDir, name string, startTime time.Tim

info := []string{
"Experiment Name: %s",
"Experiment Commit Hash: %s",
"Experiment Tags: %s",
"Scorch Run Name: %s",
"Start Time: %s",
"End Time: %s",
Expand All @@ -354,9 +355,13 @@ func (this Scorch) recordInfo(runID int, runDir, name string, startTime time.Tim

body := fmt.Sprintf(
strings.Join(info, "\n"),
name, "TODO", this.md.RunName(runID),
startTime.Format(time.RFC3339), time.Now().UTC().Format(time.RFC3339),
version.Commit, version.Tag, version.Date, mmVersion,
md.Name,
md.Annotations["phenix.workflow/tags"],
this.md.RunName(runID),
startTime.Format(time.RFC3339),
time.Now().UTC().Format(time.RFC3339),
version.Commit, version.Tag, version.Date,
mmVersion,
)

fileName := fmt.Sprintf("info-scorch-run-%d_%s.txt", runID, startTime.Format(time.RFC3339))
Expand Down
14 changes: 7 additions & 7 deletions src/go/store/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,11 +26,11 @@ type (
)

type Config struct {
Version string `json:"apiVersion" yaml:"apiVersion"`
Kind string `json:"kind" yaml:"kind"`
Metadata ConfigMetadata `json:"metadata" yaml:"metadata"`
Spec map[string]interface{} `json:"spec,omitempty" yaml:"spec,omitempty"`
Status map[string]interface{} `json:"status,omitempty" yaml:"status,omitempty"`
Version string `json:"apiVersion" yaml:"apiVersion"`
Kind string `json:"kind" yaml:"kind"`
Metadata ConfigMetadata `json:"metadata" yaml:"metadata"`
Spec map[string]any `json:"spec,omitempty" yaml:"spec,omitempty"`
Status map[string]any `json:"status,omitempty" yaml:"status,omitempty"`
}

type ConfigMetadata struct {
Expand Down Expand Up @@ -217,7 +217,7 @@ type Event struct {
Metadata map[string]string `json:"metadata,omitempty"`
}

func NewEvent(format string, args ...interface{}) *Event {
func NewEvent(format string, args ...any) *Event {
return &Event{
ID: uuid.Must(uuid.NewV4()).String(),
Timestamp: time.Now(),
Expand All @@ -227,7 +227,7 @@ func NewEvent(format string, args ...interface{}) *Event {
}
}

func NewInfoEvent(format string, args ...interface{}) *Event {
func NewInfoEvent(format string, args ...any) *Event {
event := NewEvent(format, args...)
event.Type = EventTypeInfo

Expand Down
3 changes: 3 additions & 0 deletions src/go/types/experiment.go
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,9 @@ func (this Experiment) WriteToStore(statusOnly bool) error {
return fmt.Errorf("getting experiment %s from store: %w", name, err)
}

// limit metadata updates to annotations so name doesn't accidentally get changed
c.Metadata.Annotations = this.Metadata.Annotations

if !statusOnly {
c.Spec = structs.MapDefaultCase(this.Spec, structs.CASESNAKE)
}
Expand Down
17 changes: 17 additions & 0 deletions src/go/web/workflow.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,13 +32,19 @@ func ApplyWorkflow(w http.ResponseWriter, r *http.Request) error {
role = ctx.Value("role").(rbac.Role)
vars = mux.Vars(r)
scope = vars["branch"]
q = r.URL.Query()
tags = ""
)

if !role.Allowed("workflow", "create") {
err := weberror.NewWebError(nil, "applying phenix workflow is not allowed for user %s", ctx.Value("user").(string))
return err.SetStatus(http.StatusForbidden)
}

// currently querries only are used to pass tags. However, this
// is extensible for future fields as well.
tags = strings.Join(q["tag"], ",")

var (
typ = r.Header.Get("Content-Type")
cfg *store.Config
Expand Down Expand Up @@ -116,6 +122,10 @@ func ApplyWorkflow(w http.ResponseWriter, r *http.Request) error {

annotations := map[string]string{"phenix.workflow/branch": scope}

if tags != "" {
annotations["phenix.workflow/tags"] = tags
}

opts := []experiment.CreateOption{
experiment.CreateWithName(expName),
experiment.CreateWithAnnotations(annotations),
Expand Down Expand Up @@ -232,6 +242,13 @@ func ApplyWorkflow(w http.ResponseWriter, r *http.Request) error {
exp.Metadata.Annotations["scenario"] = scenarioName
}

// default is to not override existing tags if no new tags are passed
// TODO: perhaps sorting tags and only updating those that are passed
// while leaving old tags that have not been overriden
if tags != "" {
exp.Metadata.Annotations["phenix.workflow/tags"] = tags
}

var (
aliases = make(map[string]int)
wfAliases = wf.VLANMappings()
Expand Down

0 comments on commit 57dbdc3

Please sign in to comment.