Support min/max backoff annotations and env vars (#341)

When auto auth fails, the agent will automatically retry with an
exponential backoff starting from 1 second up to 5 minutes. Those values
can be overridden in Vault itself. Here, we wire up agent to pass min
and max backoff values from its environment variables or the pod's
annotations.

* If the `AGENT_INJECT_AUTH_MIN_BACKOFF` or `AGENT_INJECT_MAX_BACKOFF`
  environment variables are set on the agent, these will be used as the
  default values.
* If a pod has the annotations `vault.hashicorp.com/auth-min-backoff` or
  `vault.hashicorp.com/auth-max-backoff` set, these will be used as the
  min and max backoff values. These will override any defaults from the
  agent environment variables.

The environment variables can be set via helm as well via, for example,

```sh
helm install vault hashicorp/vault \
  --set "injector.extraEnvironmentVars.AGENT_INJECT_AUTH_MIN_BACKOFF=4s" \
  --set "injector.extraEnvironmentVars.AGENT_INJECT_AUTH_MAX_BACKOFF=5s"
```

This was tested via `helm` and by setting the pod annotations and
confirming that the Vault agent picked up the settings when auto auth
is failing.

CHANGELOG.md

@@ -3,6 +3,7 @@
 Features:
 * Add agent-enable-quit annotation [GH-330](https://github.com/hashicorp/vault-k8s/pull/330)
 * Add go-max-procs annotation [GH-333](https://github.com/hashicorp/vault-k8s/pull/333)
+* Add min and max auth backoff annotations and environment variables [GH-341](https://github.com/hashicorp/vault-k8s/pull/341)
 
 Changes:
 * Only update webhook CA bundles when needed [GH-336](https://github.com/hashicorp/vault-k8s/pull/336)

agent-inject/agent/agent.go

@@ -269,6 +269,12 @@ type Vault struct {
 	// TLSServerName is the name of the Vault server to use when validating Vault's
 	// TLS certificates.
 	TLSServerName string
+
+	// AuthMinBackoff is the minimum time to backoff if auto auth fails.
+	AuthMinBackoff string
+
+	// AuthMinBackoff is the maximum time to backoff if auto auth fails.
+	AuthMaxBackoff string
 }
 
 type VaultAgentCache struct {
@@ -347,6 +353,8 @@ func New(pod *corev1.Pod, patches []*jsonpatch.JsonPatchOperation) (*Agent, erro
 			Role:             pod.Annotations[AnnotationVaultRole],
 			TLSSecret:        pod.Annotations[AnnotationVaultTLSSecret],
 			TLSServerName:    pod.Annotations[AnnotationVaultTLSServerName],
+			AuthMinBackoff:   pod.Annotations[AnnotationAgentAuthMinBackoff],
+			AuthMaxBackoff:   pod.Annotations[AnnotationAgentAuthMaxBackoff],
 		},
 	}
 

agent-inject/agent/annotations.go

@@ -5,6 +5,7 @@ import (
 	"path"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/pkg/errors"
 	corev1 "k8s.io/api/core/v1"
@@ -262,6 +263,14 @@ const (
 	// AnnotationAgentEnableQuit configures whether the quit endpoint is
 	// enabled in the injected agent config
 	AnnotationAgentEnableQuit = "vault.hashicorp.com/agent-enable-quit"
+
+	// AnnotationAgentAuthMinBackoff specifies the minimum backoff duration used when the agent auto auth fails.
+	// Defaults to 1 second.
+	AnnotationAgentAuthMinBackoff = "vault.hashicorp.com/auth-min-backoff"
+
+	// AnnotationAgentAuthMaxBackoff specifies the maximum backoff duration used when the agent auto auth fails.
+	// Defaults to 5 minutes.
+	AnnotationAgentAuthMaxBackoff = "vault.hashicorp.com/auth-max-backoff"
 )
 
 type AgentConfig struct {
@@ -283,6 +292,8 @@ type AgentConfig struct {
 	ResourceLimitMem           string
 	ExitOnRetryFailure         bool
 	StaticSecretRenderInterval string
+	AuthMinBackoff             string
+	AuthMaxBackoff             string
 }
 
 // Init configures the expected annotations required to create a new instance
@@ -451,6 +462,30 @@ func Init(pod *corev1.Pod, cfg AgentConfig) error {
 		pod.ObjectMeta.Annotations[AnnotationTemplateConfigStaticSecretRenderInterval] = cfg.StaticSecretRenderInterval
 	}
 
+	if minBackoffString, ok := pod.ObjectMeta.Annotations[AnnotationAgentAuthMinBackoff]; ok {
+		if minBackoffString != "" {
+			_, err := time.ParseDuration(minBackoffString)
+			if err != nil {
+				return fmt.Errorf("error parsing min backoff as duration: %v", err)
+			}
+		}
+	} else if cfg.AuthMinBackoff != "" {
+		// set default from env/flag
+		pod.ObjectMeta.Annotations[AnnotationAgentAuthMinBackoff] = cfg.AuthMinBackoff
+	}
+
+	if maxBackoffString, ok := pod.ObjectMeta.Annotations[AnnotationAgentAuthMaxBackoff]; ok {
+		if maxBackoffString != "" {
+			_, err := time.ParseDuration(maxBackoffString)
+			if err != nil {
+				return fmt.Errorf("error parsing max backoff as duration: %v", err)
+			}
+		}
+	} else if cfg.AuthMaxBackoff != "" {
+		// set default from env/flag
+		pod.ObjectMeta.Annotations[AnnotationAgentAuthMaxBackoff] = cfg.AuthMaxBackoff
+	}
+
 	return nil
 }
 

agent-inject/agent/annotations_test.go

@@ -732,6 +732,16 @@ func TestCouldErrorAnnotations(t *testing.T) {
 		{AnnotationAgentCacheEnable, "tRuE", false},
 		{AnnotationAgentCacheEnable, "fAlSe", false},
 		{AnnotationAgentCacheEnable, "", false},
+
+		{AnnotationAgentAuthMinBackoff, "", true},
+		{AnnotationAgentAuthMinBackoff, "1s", true},
+		{AnnotationAgentAuthMinBackoff, "1m", true},
+		{AnnotationAgentAuthMinBackoff, "x", false},
+
+		{AnnotationAgentAuthMaxBackoff, "", true},
+		{AnnotationAgentAuthMaxBackoff, "1s", true},
+		{AnnotationAgentAuthMaxBackoff, "1m", true},
+		{AnnotationAgentAuthMaxBackoff, "x", false},
 	}
 
 	for i, tt := range tests {
@@ -741,7 +751,11 @@ func TestCouldErrorAnnotations(t *testing.T) {
 		agentConfig := basicAgentConfig()
 		err := Init(pod, agentConfig)
 		if err != nil {
-			t.Errorf("got error, shouldn't have: %s", err)
+			if tt.valid {
+				t.Errorf("got error, shouldn't have: %s", err)
+			}
+			// if !tt.valid, that is okay, we expected an error
+			continue
 		}
 
 		_, err = New(pod, patches)
@@ -1121,3 +1135,23 @@ func TestDefaultTemplateOverride(t *testing.T) {
 		}
 	}
 }
+
+func TestAuthMinMaxBackoff(t *testing.T) {
+	pod := testPod(map[string]string{
+		"vault.hashicorp.com/auth-min-backoff": "5s",
+		"vault.hashicorp.com/auth-max-backoff": "10s",
+	})
+	agentConfig := basicAgentConfig()
+	err := Init(pod, agentConfig)
+	if err != nil {
+		t.Errorf("got error, shouldn't have: %s", err)
+	}
+
+	agent, err := New(pod, nil)
+	if err != nil {
+		t.Errorf("got error, shouldn't have: %s", err)
+	}
+
+	require.Equal(t, "5s", agent.Vault.AuthMinBackoff, "expected 5s, got %v", agent.Vault.AuthMinBackoff)
+	require.Equal(t, "10s", agent.Vault.AuthMaxBackoff, "expected 10s, got %v", agent.Vault.AuthMaxBackoff)
+}

agent-inject/agent/config.go

@@ -52,6 +52,8 @@ type Method struct {
 	MountPath  string                 `json:"mount_path,omitempty"`
 	WrapTTLRaw interface{}            `json:"wrap_ttl,omitempty"`
 	WrapTTL    time.Duration          `json:"-"`
+	MinBackoff string                 `json:"min_backoff,omitempty"`
+	MaxBackoff string                 `json:"max_backoff,omitempty"`
 	Namespace  string                 `json:"namespace,omitempty"`
 	Config     map[string]interface{} `json:"config,omitempty"`
 }
@@ -167,10 +169,12 @@ func (a *Agent) newConfig(init bool) ([]byte, error) {
 		},
 		AutoAuth: &AutoAuth{
 			Method: &Method{
-				Type:      a.Vault.AuthType,
-				Namespace: a.Vault.Namespace,
-				MountPath: a.Vault.AuthPath,
-				Config:    a.Vault.AuthConfig,
+				Type:       a.Vault.AuthType,
+				Namespace:  a.Vault.Namespace,
+				MountPath:  a.Vault.AuthPath,
+				Config:     a.Vault.AuthConfig,
+				MinBackoff: a.Vault.AuthMinBackoff,
+				MaxBackoff: a.Vault.AuthMaxBackoff,
 			},
 			Sinks: []*Sink{
 				{

agent-inject/handler.go

@@ -64,6 +64,8 @@ type Handler struct {
 	ResourceLimitMem           string
 	ExitOnRetryFailure         bool
 	StaticSecretRenderInterval string
+	AuthMinBackoff             string
+	AuthMaxBackoff             string
 }
 
 // Handle is the http.HandlerFunc implementation that actually handles the
@@ -194,6 +196,8 @@ func (h *Handler) Mutate(req *admissionv1.AdmissionRequest) *admissionv1.Admissi
 		ResourceLimitMem:           h.ResourceLimitMem,
 		ExitOnRetryFailure:         h.ExitOnRetryFailure,
 		StaticSecretRenderInterval: h.StaticSecretRenderInterval,
+		AuthMinBackoff:             h.AuthMinBackoff,
+		AuthMaxBackoff:             h.AuthMaxBackoff,
 	}
 	err = agent.Init(&pod, cfg)
 	if err != nil {

subcommand/injector/command.go

@@ -67,6 +67,8 @@ type Command struct {
 	flagResourceLimitMem           string // Set Memory limit in the injected containers
 	flagTLSMinVersion              string // Minimum TLS version supported by the webhook server
 	flagTLSCipherSuites            string // Comma-separated list of supported cipher suites
+	flagAuthMinBackoff             string // Auth min backoff on failure
+	flagAuthMaxBackoff             string // Auth min backoff on failure
 
 	flagSet *flag.FlagSet
 
@@ -199,6 +201,8 @@ func (c *Command) Run(args []string) int {
 		ResourceLimitMem:           c.flagResourceLimitMem,
 		ExitOnRetryFailure:         c.flagExitOnRetryFailure,
 		StaticSecretRenderInterval: c.flagStaticSecretRenderInterval,
+		AuthMinBackoff:             c.flagAuthMinBackoff,
+		AuthMaxBackoff:             c.flagAuthMaxBackoff,
 	}
 
 	mux := http.NewServeMux()

subcommand/injector/flags.go

@@ -6,6 +6,7 @@ import (
 	"sort"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/hashicorp/go-hclog"
 	"github.com/hashicorp/go-secure-stdlib/tlsutil"
@@ -109,6 +110,12 @@ type Specification struct {
 
 	// TLSCipherSuites is the AGENT_INJECT_TLS_CIPHER_SUITES environment variable
 	TLSCipherSuites string `envconfig:"tls_cipher_suites"`
+
+	// AuthMinBackoff is the AGENT_MIN_BACKOFF environment variable
+	AuthMinBackoff string `envconfig:"AGENT_INJECT_AUTH_MIN_BACKOFF"`
+
+	// AuthMaxBackoff is the AGENT_MAX_BACKOFF environment variable
+	AuthMaxBackoff string `envconfig:"AGENT_INJECT_AUTH_MAX_BACKOFF"`
 }
 
 func (c *Command) init() {
@@ -168,6 +175,10 @@ func (c *Command) init() {
 		fmt.Sprintf("CPU resource limit set in injected containers. Defaults to %s", agent.DefaultResourceLimitCPU))
 	c.flagSet.StringVar(&c.flagResourceLimitMem, "memory-limit", agent.DefaultResourceLimitMem,
 		fmt.Sprintf("Memory resource limit set in injected containers. Defaults to %s", agent.DefaultResourceLimitMem))
+	c.flagSet.StringVar(&c.flagAuthMinBackoff, "auth-min-backoff", "",
+		"Sets the minimum backoff on auto-auth failure. Default is 1s")
+	c.flagSet.StringVar(&c.flagAuthMaxBackoff, "auth-max-backoff", "",
+		"Sets the maximum backoff on auto-auth failure. Default is 5m")
 
 	tlsVersions := []string{}
 	for v := range tlsutil.TLSLookup {
@@ -339,5 +350,27 @@ func (c *Command) parseEnvs() error {
 		c.flagTLSCipherSuites = envs.TLSCipherSuites
 	}
 
+	if envs.AuthMinBackoff != "" {
+		c.flagAuthMinBackoff = envs.AuthMinBackoff
+	}
+
+	if c.flagAuthMinBackoff != "" {
+		_, err = time.ParseDuration(c.flagAuthMinBackoff)
+		if err != nil {
+			return err
+		}
+	}
+
+	if envs.AuthMaxBackoff != "" {
+		c.flagAuthMaxBackoff = envs.AuthMaxBackoff
+	}
+
+	if c.flagAuthMaxBackoff != "" {
+		_, err = time.ParseDuration(c.flagAuthMaxBackoff)
+		if err != nil {
+			return err
+		}
+	}
+
 	return nil
 }

subcommand/injector/flags_test.go

@@ -133,6 +133,8 @@ func TestCommandEnvs(t *testing.T) {
 		{env: "AGENT_INJECT_TEMPLATE_STATIC_SECRET_RENDER_INTERVAL", value: "12s", cmdPtr: &cmd.flagStaticSecretRenderInterval},
 		{env: "AGENT_INJECT_TLS_MIN_VERSION", value: "tls13", cmdPtr: &cmd.flagTLSMinVersion},
 		{env: "AGENT_INJECT_TLS_CIPHER_SUITES", value: "TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA", cmdPtr: &cmd.flagTLSCipherSuites},
+		{env: "AGENT_INJECT_AUTH_MIN_BACKOFF", value: "5s", cmdPtr: &cmd.flagAuthMinBackoff},
+		{env: "AGENT_INJECT_AUTH_MAX_BACKOFF", value: "5s", cmdPtr: &cmd.flagAuthMaxBackoff},
 	}
 
 	for _, tt := range tests {