From a22c851d4d19afdf6d5c150b3bff0b8f13405d94 Mon Sep 17 00:00:00 2001 From: vsoch Date: Sun, 24 Sep 2023 00:13:00 -0600 Subject: [PATCH] add support for commands addon this will make it easy (I hope) to download/install/use different performance tools. Signed-off-by: vsoch --- docs/_static/data/addons.json | 10 + docs/getting_started/addons.md | 141 +++-------- examples/addons/commands/metrics-time.yaml | 18 ++ .../perf-lammps-hpctoolkit/metrics-rocky.yaml | 0 .../perf-lammps-hpctoolkit/metrics.yaml | 0 pkg/addons/addons.go | 2 - pkg/addons/commands.go | 229 ++++++++++++++++++ 7 files changed, 294 insertions(+), 106 deletions(-) create mode 100644 examples/addons/commands/metrics-time.yaml rename examples/{tests => addons}/perf-lammps-hpctoolkit/metrics-rocky.yaml (100%) rename examples/{tests => addons}/perf-lammps-hpctoolkit/metrics.yaml (100%) create mode 100644 pkg/addons/commands.go diff --git a/docs/_static/data/addons.json b/docs/_static/data/addons.json index ab039ec..a5a36f3 100644 --- a/docs/_static/data/addons.json +++ b/docs/_static/data/addons.json @@ -4,6 +4,16 @@ "description": "basic application (container) type", "family": "application" }, + { + "name": "commands", + "description": "customize a metric's entrypoints", + "family": "application" + }, + { + "name": "perf-commands", + "description": "customize a metric's entrypoints expecting performance tracing (adding ptrace and admin caps)", + "family": "application" + }, { "name": "perf-hpctoolkit", "description": "performance tools for measurement and analysis", diff --git a/docs/getting_started/addons.md b/docs/getting_started/addons.md index c56eb75..0190b82 100644 --- a/docs/getting_started/addons.md +++ b/docs/getting_started/addons.md @@ -12,6 +12,38 @@ have a request for an addon please [let us know](https://github.com/converged-co +## Command Addons + +The Commands group of addons are some of my favorites, because they allow you to customize entrypoints for existing metrics! + +### Commands + +> Use addon with name "commands" + +The basic "commands" addon allows you to customize: + + - **preBlock**: A custom block of commands to run before the primary entrypoint command. + - **prefix**: a wrapping prefix to the entrypoint + - **suffix**: a wrapping suffix to the entrypoint + - **postBlock**: a block of commands to run after the primary entrypoint command. + +For example, you might want to time something by adding "time" as the prefix. You may want to +install something special to the container (or otherwise customize files or content) before running +the entrypoint. You might also want to run some kind of cleanup or save in the postBlock. The +reason "commands" is so cool is because it's flexible to so many ideas! Here is an example: + + - *[metrics-time.yaml](https://github.com/converged-computing/metrics-operator/tree/main/examples/addons/commands/metrics-time.yaml)* + +### Perf + +> Use addon with name "perf-commands" + +Per commands has the same arguments as [commands](#commands) above, but will additionally add CAP_PTRACE and CAP_SYSADMIN +to your container, which are typically needed for performance benchmarking tools. As an example here, you might +install a performance tool in the preBlock, run it using the "prefix" and then use "suffix" optionally to pipe to +a file, and postBlock to upload somewhere. + + ## Existing Volumes An existing volume addon can be provided to a metric. As an example, it would make sense to run an IO benchmarks with @@ -25,7 +57,7 @@ different kinds of volume addons. The addons for volumes currently include: and for all of the above, you want to create it and provide metadata for the addon to the operator, which will ensure the volume is available for your metric. We will provide examples here to do that. -#### persistent volume claim addon +### persistent volume claim addon As an example, here is how to provide the name of an existing claim (you created separately) to a metric container: TODO add support to specify a specific metric container or replicated job container, if applicable. @@ -45,7 +77,7 @@ spec: The above would add a claim named "data" to the metric container(s). -#### config map addon example +### config map addon example Here is an example of providing a config map to an application container In layman's terms, we are deploying vanilla nginx, but adding a configuration file to `/etc/nginx/conf.d` @@ -86,7 +118,7 @@ data: } ``` -#### secret addon example +### secret addon example Here is an example of providing an existing secret (in the metrics-operator namespace) to the metric container(s): @@ -106,7 +138,7 @@ spec: The above shows an existing secret named "certs" that we will mount into `/etc/certs`. -#### hostpath volume addon example +### hostpath volume addon example Here is how to use a host path: @@ -123,106 +155,7 @@ spec: path: /path/in/container ``` - -TODO convert to addon logic - -### application - -When you want to measure application performance, you'll need to add an "application" section to your MetricSet. This is the container that houses some application that you want to measure performance for. This means that minimally, you are required to define the application container image and command: - - -```yaml -spec: - application: - image: ghcr.io/rse-ops/vanilla-lammps:tag-latest - command: mpirun lmp -v x 1 -v y 1 -v z 1 -in in.reaxc.hns -nocite -``` - -In the above example, we target a container with LAMMPS and mpi, and we are going to run MPIrun. -The command will be used by the metrics sidecar containers to find the PID of interest to measure. - -#### workingDir - -To add a working directory for your application: - -```yaml -spec: - application: - image: ghcr.io/rse-ops/vanilla-lammps:tag-latest - command: mpirun lmp -v x 1 -v y 1 -v z 1 -in in.reaxc.hns -nocite - workingDir: /opt/lammps/examples/reaxff/HNS -``` - -#### volumes - -An application is allowed to have one or more existing volumes. An existing volume can be any of the types described in [existing volumes](#existing-volumes) - -#### resources - -You can define resources for an application or a metric container. Known keys include "memory" and "cpu" (should be provided in some string format that can be parsed) and all others are considered some kind of quantity request. - -```yaml -application: - resources: - memory: 500M - cpu: 4 -``` - -Metrics can also take resource requests. - -```yaml -metrics: - - name: io-fio - resources: - memory: 500M - cpu: 4 -``` - -If you wanted to, for example, request a GPU, that might look like: - -```yaml -resources: - limits: - gpu-vendor.example/example-gpu: 1 -``` - -Or for a particular type of networking fabric: - -```yaml -resources: - limits: - vpc.amazonaws.com/efa: 1 -``` - -Both limits and resources are flexible to accept a string or an integer value, and you'll get an error if you -provide something else. If you need something else, [let us know](https://github.com/converged-computing/metrics-operator/issues). -If you are requesting GPU, [this documentation](https://kubernetes.io/docs/tasks/manage-gpus/scheduling-gpus/) is helpful. - -### storage - -When you want to measure some storage performance, you'll want to add a "storage" section to your MetricSet. This will typically just be a reference to some existing storage (see [existing volumes](#existing-volumes)) that we want to measure, and can also be done for some number of completions and metrics for storage. - -#### commands - -If you need to add some special logic to create or cleanup for a storage volume, you are free to define them for storage in each of pre and post sections, which will happen before and after the metric runs, respectively. - -```yaml -storage: - volume: - claimName: data - path: /data - commands: - pre: | - apt-get update && apt-get install -y mymounter-tool - mymounter-tool mount /data - post: mymounter-tool unmount /data - # Wrap the storage metric in this prefix - prefix: myprefix -``` - -All of the above are strings. The pipe allows for multiple lines, if appropriate. -Note that while a "volume" is typical, you might have a storage setup that is done via a set of custom commands, in which case -you don't need to define the volume too. +**Note that we have support for a custom application container, but haven't written any good examples yet!** ## Performance diff --git a/examples/addons/commands/metrics-time.yaml b/examples/addons/commands/metrics-time.yaml new file mode 100644 index 0000000..6882b64 --- /dev/null +++ b/examples/addons/commands/metrics-time.yaml @@ -0,0 +1,18 @@ +apiVersion: flux-framework.org/v1alpha2 +kind: MetricSet +metadata: + labels: + app.kubernetes.io/name: metricset + app.kubernetes.io/instance: metricset-sample + name: metricset-sample +spec: + # Number of pods for lammps (one launcher, the rest workers) + pods: 4 + metrics: + - name: app-lammps + addons: + - name: commands + options: + preBlock: echo "Hello before LAMMPS" + prefix: time + postBlock: echo "Hello after LAMMPS" \ No newline at end of file diff --git a/examples/tests/perf-lammps-hpctoolkit/metrics-rocky.yaml b/examples/addons/perf-lammps-hpctoolkit/metrics-rocky.yaml similarity index 100% rename from examples/tests/perf-lammps-hpctoolkit/metrics-rocky.yaml rename to examples/addons/perf-lammps-hpctoolkit/metrics-rocky.yaml diff --git a/examples/tests/perf-lammps-hpctoolkit/metrics.yaml b/examples/addons/perf-lammps-hpctoolkit/metrics.yaml similarity index 100% rename from examples/tests/perf-lammps-hpctoolkit/metrics.yaml rename to examples/addons/perf-lammps-hpctoolkit/metrics.yaml diff --git a/pkg/addons/addons.go b/pkg/addons/addons.go index 95a0cb7..030d4c6 100644 --- a/pkg/addons/addons.go +++ b/pkg/addons/addons.go @@ -120,8 +120,6 @@ func GetAddon(a *api.MetricAddon) (Addon, error) { return addon, nil } -// TODO likely we need to carry around entrypoints to customize? - // Register a new addon! func Register(a Addon) { name := a.Name() diff --git a/pkg/addons/commands.go b/pkg/addons/commands.go new file mode 100644 index 0000000..d231e94 --- /dev/null +++ b/pkg/addons/commands.go @@ -0,0 +1,229 @@ +/* +Copyright 2023 Lawrence Livermore National Security, LLC + (c.f. AUTHORS, NOTICE.LLNS, COPYING) + +SPDX-License-Identifier: MIT +*/ + +package addons + +import ( + "fmt" + + api "github.com/converged-computing/metrics-operator/api/v1alpha2" + "github.com/converged-computing/metrics-operator/pkg/specs" + "k8s.io/apimachinery/pkg/util/intstr" + jobset "sigs.k8s.io/jobset/api/jobset/v1alpha2" +) + +// Perf addon expects the same command structure, but adds sys caps for trace and admin +type PerfAddon struct { + CommandAddon +} + +// CustomizeEntrypoint scripts +func (a *PerfAddon) CustomizeEntrypoints( + cs []*specs.ContainerSpec, + rjs []*jobset.ReplicatedJob, +) { + for _, rj := range rjs { + + // Only customize if the replicated job name matches the target + if a.target != "" && a.target != rj.Name { + continue + } + a.customizeEntrypoint(cs, rj) + a.addContainerCaps(cs, rj) + } +} + +// addContainerCaps adds capabilities to a container spec +func (a *PerfAddon) addContainerCaps( + cs []*specs.ContainerSpec, + rj *jobset.ReplicatedJob, +) { + + // We use container names to target specific entrypoint scripts here + for _, containerSpec := range cs { + + // Is this the right replicated job? + if !a.isSelected(containerSpec, rj) { + continue + } + + // Always copy over the pre block - we need the logic to copy software + containerSpec.Attributes.SecurityContext.AllowAdmin = true + containerSpec.Attributes.SecurityContext.AllowPtrace = true + } +} + +// Command addons primarily edit the entrypoint commands +type CommandAddon struct { + AddonBase + + // preBlock is run right before the command + preBlock string + + // prefix is added as a prefix to the command + prefix string + + // add a suffix to the command + suffix string + + // postBlock is run after + postBlock string + + // job name and container name targets + target string + containerTarget string +} + +// Doesn't make sense to have an empty command prefix / pre and post! +func (a *CommandAddon) Validate() bool { + if a.preBlock == "" && a.prefix == "" && a.postBlock == "" && a.suffix == "" { + logger.Error("The command addon requires one of a 'prefix', 'preBlock', 'postBlock' or 'suffix'") + return false + } + return true +} + +// Application family for now... +func (m CommandAddon) Family() string { + return AddonFamilyApplication +} + +// Set custom options / attributes for the metric +func (a *CommandAddon) SetOptions(metric *api.MetricAddon) { + target, ok := metric.Options["target"] + if ok { + a.target = target.StrVal + } + ctarget, ok := metric.Options["containerTarget"] + if ok { + a.containerTarget = ctarget.StrVal + } + prefix, ok := metric.Options["prefix"] + if ok { + a.prefix = prefix.StrVal + } + suffix, ok := metric.Options["suffix"] + if ok { + a.suffix = suffix.StrVal + } + preBlock, ok := metric.Options["preBlock"] + if ok { + a.preBlock = preBlock.StrVal + } + postBlock, ok := metric.Options["postBlock"] + if ok { + a.postBlock = postBlock.StrVal + } +} + +// Underlying function that can be shared +func (a *CommandAddon) DefaultOptions() map[string]intstr.IntOrString { + return map[string]intstr.IntOrString{ + "target": intstr.FromString(a.target), + "prefix": intstr.FromString(a.prefix), + "suffix": intstr.FromString(a.suffix), + "preBlock": intstr.FromString(a.preBlock), + "postBlock": intstr.FromString(a.postBlock), + "containerTarget": intstr.FromString(a.containerTarget), + } +} + +// CustomizeEntrypoint scripts +func (a *CommandAddon) CustomizeEntrypoints( + cs []*specs.ContainerSpec, + rjs []*jobset.ReplicatedJob, +) { + for _, rj := range rjs { + + // Only customize if the replicated job name matches the target + if a.target != "" && a.target != rj.Name { + continue + } + a.customizeEntrypoint(cs, rj) + } + +} + +// isSelected determines if a container spec is targeted based on the container and rj names +func (a *CommandAddon) isSelected( + cs *specs.ContainerSpec, + rj *jobset.ReplicatedJob, +) bool { + if cs.JobName != rj.Name { + return false + } + + // Next check if we have a target set (for the container) + if a.containerTarget != "" && cs.Name != "" && a.containerTarget != cs.Name { + return false + } + return true +} + +// CustomizeEntrypoint for a single replicated job +func (a *CommandAddon) customizeEntrypoint( + cs []*specs.ContainerSpec, + rj *jobset.ReplicatedJob, +) { + + // Generate addon metadata + meta := Metadata(a) + + // This should be run after the pre block of the script, and includes our preblock + preBlock := ` +echo "%s" +%s +` + preBlock = fmt.Sprintf(preBlock, meta, a.preBlock) + + // postBlock to possibly run the hpcstruct command should come right after + postBlock := fmt.Sprintf("\n%s", a.postBlock) + + // We use container names to target specific entrypoint scripts here + for _, containerSpec := range cs { + + // Is this the right replicated job? + if !a.isSelected(containerSpec, rj) { + continue + } + + // Always copy over the pre block - we need the logic to copy software + containerSpec.EntrypointScript.Pre += "\n" + preBlock + + // If the post command ends with sleep infinity, tweak it + isInteractive, updatedPost := deriveUpdatedPost(containerSpec.EntrypointScript.Post) + containerSpec.EntrypointScript.Post = updatedPost + + // The post to run the command across nodes (when the application finishes) + containerSpec.EntrypointScript.Post = containerSpec.EntrypointScript.Post + "\n" + postBlock + containerSpec.EntrypointScript.Command = fmt.Sprintf("%s %s %s", a.prefix, containerSpec.EntrypointScript.Command, a.suffix) + + // If is interactive, add back sleep infinity + if isInteractive { + containerSpec.EntrypointScript.Post += "\nsleep infinity\n" + } + } +} + +func init() { + + // Config map volume type + base := AddonBase{ + Identifier: "commands", + Summary: "customize a metric's entrypoints", + } + app := CommandAddon{AddonBase: base} + Register(&app) + + base = AddonBase{ + Identifier: "perf-commands", + Summary: "customize a metric's entrypoints expecting performance tracing (adding ptrace and admin caps)", + } + cmd := CommandAddon{AddonBase: base} + perf := PerfAddon{CommandAddon: cmd} + Register(&perf) +}