diff --git a/internal/servermon/monitoring.go b/internal/servermon/monitoring.go index f353973d3..0e09ea320 100644 --- a/internal/servermon/monitoring.go +++ b/internal/servermon/monitoring.go @@ -50,6 +50,19 @@ var ( Name: "error_total", Help: "The number of database errors.", }, []string{"method"}) + VaultQueryDurationHistogram = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: "jimm", + Subsystem: "vault", + Name: "query_duration_seconds", + Help: "Histogram of database query time in seconds", + Buckets: []float64{.005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10}, + }, []string{"method"}) + VaultQueryErrorCount = promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: "jimm", + Subsystem: "vault", + Name: "error_total", + Help: "The number of database errors.", + }, []string{"method"}) ConcurrentWebsocketConnections = promauto.NewGauge(prometheus.GaugeOpts{ Namespace: "jimm", Subsystem: "websocket", diff --git a/internal/vault/vault.go b/internal/vault/vault.go index 4393ea635..9b4c1390b 100644 --- a/internal/vault/vault.go +++ b/internal/vault/vault.go @@ -19,6 +19,7 @@ import ( "github.com/lestrrat-go/jwx/v2/jwk" "github.com/canonical/jimm/internal/errors" + "github.com/canonical/jimm/internal/servermon" ) const ( @@ -59,9 +60,13 @@ type VaultStore struct { // Get retrieves the attributes for the given cloud credential from a vault // service. -func (s *VaultStore) Get(ctx context.Context, tag names.CloudCredentialTag) (map[string]string, error) { +func (s *VaultStore) Get(ctx context.Context, tag names.CloudCredentialTag) (_ map[string]string, err error) { const op = errors.Op("vault.Get") + durationObserver := servermon.DurationObserver(servermon.VaultQueryDurationHistogram, string(op)) + defer durationObserver() + defer servermon.ErrorCounter(servermon.VaultQueryErrorCount, &err, string(op)) + client, err := s.client(ctx) if err != nil { return nil, errors.E(op, err) @@ -89,12 +94,17 @@ func (s *VaultStore) Get(ctx context.Context, tag names.CloudCredentialTag) (map // Put stores the attributes associated with a cloud-credential in a vault // service. -func (s *VaultStore) Put(ctx context.Context, tag names.CloudCredentialTag, attr map[string]string) error { +func (s *VaultStore) Put(ctx context.Context, tag names.CloudCredentialTag, attr map[string]string) (err error) { if len(attr) == 0 { return s.delete(ctx, tag) } const op = errors.Op("vault.Put") + + durationObserver := servermon.DurationObserver(servermon.VaultQueryDurationHistogram, string(op)) + defer durationObserver() + defer servermon.ErrorCounter(servermon.VaultQueryErrorCount, &err, string(op)) + client, err := s.client(ctx) if err != nil { return errors.E(op, err) @@ -113,9 +123,13 @@ func (s *VaultStore) Put(ctx context.Context, tag names.CloudCredentialTag, attr // delete removes the attributes associated with the cloud-credential in // the vault service. -func (s *VaultStore) delete(ctx context.Context, tag names.CloudCredentialTag) error { +func (s *VaultStore) delete(ctx context.Context, tag names.CloudCredentialTag) (err error) { const op = errors.Op("vault.delete") + durationObserver := servermon.DurationObserver(servermon.VaultQueryDurationHistogram, string(op)) + defer durationObserver() + defer servermon.ErrorCounter(servermon.VaultQueryErrorCount, &err, string(op)) + client, err := s.client(ctx) if err != nil { return errors.E(op, err) @@ -133,9 +147,13 @@ func (s *VaultStore) delete(ctx context.Context, tag names.CloudCredentialTag) e // GetControllerCredentials retrieves the credentials for the given controller from a vault // service. -func (s *VaultStore) GetControllerCredentials(ctx context.Context, controllerName string) (string, string, error) { +func (s *VaultStore) GetControllerCredentials(ctx context.Context, controllerName string) (_ string, _ string, err error) { const op = errors.Op("vault.GetControllerCredentials") + durationObserver := servermon.DurationObserver(servermon.VaultQueryDurationHistogram, string(op)) + defer durationObserver() + defer servermon.ErrorCounter(servermon.VaultQueryErrorCount, &err, string(op)) + client, err := s.client(ctx) if err != nil { return "", "", errors.E(op, err) @@ -162,12 +180,17 @@ func (s *VaultStore) GetControllerCredentials(ctx context.Context, controllerNam // PutControllerCredentials stores the controller credentials in a vault // service. -func (s *VaultStore) PutControllerCredentials(ctx context.Context, controllerName string, username string, password string) error { +func (s *VaultStore) PutControllerCredentials(ctx context.Context, controllerName string, username string, password string) (err error) { if username == "" || password == "" { return s.deleteControllerCredentials(ctx, controllerName) } const op = errors.Op("vault.PutControllerCredentials") + + durationObserver := servermon.DurationObserver(servermon.VaultQueryDurationHistogram, string(op)) + defer durationObserver() + defer servermon.ErrorCounter(servermon.VaultQueryErrorCount, &err, string(op)) + client, err := s.client(ctx) if err != nil { return errors.E(op, err) @@ -185,9 +208,13 @@ func (s *VaultStore) PutControllerCredentials(ctx context.Context, controllerNam } // CleanupJWKS removes all secrets associated with the JWKS process. -func (s *VaultStore) CleanupJWKS(ctx context.Context) error { +func (s *VaultStore) CleanupJWKS(ctx context.Context) (err error) { const op = errors.Op("vault.CleanupJWKS") + durationObserver := servermon.DurationObserver(servermon.VaultQueryDurationHistogram, string(op)) + defer durationObserver() + defer servermon.ErrorCounter(servermon.VaultQueryErrorCount, &err, string(op)) + client, err := s.client(ctx) if err != nil { return errors.E(op, err) @@ -203,9 +230,13 @@ func (s *VaultStore) CleanupJWKS(ctx context.Context) error { } // GetJWKS returns the current key set stored within the credential store. -func (s *VaultStore) GetJWKS(ctx context.Context) (jwk.Set, error) { +func (s *VaultStore) GetJWKS(ctx context.Context) (_ jwk.Set, err error) { const op = errors.Op("vault.GetJWKS") + durationObserver := servermon.DurationObserver(servermon.VaultQueryDurationHistogram, string(op)) + defer durationObserver() + defer servermon.ErrorCounter(servermon.VaultQueryErrorCount, &err, string(op)) + client, err := s.client(ctx) if err != nil { return nil, errors.E(op, err) @@ -239,9 +270,13 @@ func (s *VaultStore) GetJWKS(ctx context.Context) (jwk.Set, error) { } // GetJWKSPrivateKey returns the current private key for the active JWKS -func (s *VaultStore) GetJWKSPrivateKey(ctx context.Context) ([]byte, error) { +func (s *VaultStore) GetJWKSPrivateKey(ctx context.Context) (_ []byte, err error) { const op = errors.Op("vault.GetJWKSPrivateKey") + durationObserver := servermon.DurationObserver(servermon.VaultQueryDurationHistogram, string(op)) + defer durationObserver() + defer servermon.ErrorCounter(servermon.VaultQueryErrorCount, &err, string(op)) + client, err := s.client(ctx) if err != nil { return nil, errors.E(op, err) @@ -269,8 +304,13 @@ func (s *VaultStore) GetJWKSPrivateKey(ctx context.Context) ([]byte, error) { } // GetJWKSExpiry returns the expiry of the active JWKS. -func (s *VaultStore) GetJWKSExpiry(ctx context.Context) (time.Time, error) { +func (s *VaultStore) GetJWKSExpiry(ctx context.Context) (_ time.Time, err error) { const op = errors.Op("vault.getJWKSExpiry") + + durationObserver := servermon.DurationObserver(servermon.VaultQueryDurationHistogram, string(op)) + defer durationObserver() + defer servermon.ErrorCounter(servermon.VaultQueryErrorCount, &err, string(op)) + now := time.Now() client, err := s.client(ctx) if err != nil { @@ -305,9 +345,13 @@ func (s *VaultStore) GetJWKSExpiry(ctx context.Context) (time.Time, error) { // // The pathing is similar to the controllers credentials // in that we understand RS256 keys as credentials, rather than crytographic keys. -func (s *VaultStore) PutJWKS(ctx context.Context, jwks jwk.Set) error { +func (s *VaultStore) PutJWKS(ctx context.Context, jwks jwk.Set) (err error) { const op = errors.Op("vault.PutJWKS") + durationObserver := servermon.DurationObserver(servermon.VaultQueryDurationHistogram, string(op)) + defer durationObserver() + defer servermon.ErrorCounter(servermon.VaultQueryErrorCount, &err, string(op)) + client, err := s.client(ctx) if err != nil { return errors.E(op, err) @@ -327,9 +371,13 @@ func (s *VaultStore) PutJWKS(ctx context.Context, jwks jwk.Set) error { } // PutJWKSPrivateKey persists the private key associated with the current JWKS within the store. -func (s *VaultStore) PutJWKSPrivateKey(ctx context.Context, pem []byte) error { +func (s *VaultStore) PutJWKSPrivateKey(ctx context.Context, pem []byte) (err error) { const op = errors.Op("vault.PutJWKSPrivateKey") + durationObserver := servermon.DurationObserver(servermon.VaultQueryDurationHistogram, string(op)) + defer durationObserver() + defer servermon.ErrorCounter(servermon.VaultQueryErrorCount, &err, string(op)) + client, err := s.client(ctx) if err != nil { return errors.E(op, err) @@ -343,9 +391,13 @@ func (s *VaultStore) PutJWKSPrivateKey(ctx context.Context, pem []byte) error { } // PutJWKSExpiry sets the expiry time for the current JWKS within the store. -func (s *VaultStore) PutJWKSExpiry(ctx context.Context, expiry time.Time) error { +func (s *VaultStore) PutJWKSExpiry(ctx context.Context, expiry time.Time) (err error) { const op = errors.Op("vault.PutJWKSExpiry") + durationObserver := servermon.DurationObserver(servermon.VaultQueryDurationHistogram, string(op)) + defer durationObserver() + defer servermon.ErrorCounter(servermon.VaultQueryErrorCount, &err, string(op)) + client, err := s.client(ctx) if err != nil { return errors.E(op, err) @@ -380,9 +432,13 @@ func (s *VaultStore) getJWKSExpiryPath() string { } // CleanupOAuthSecrets removes all secrets associated with OAuth. -func (s *VaultStore) CleanupOAuthSecrets(ctx context.Context) error { +func (s *VaultStore) CleanupOAuthSecrets(ctx context.Context) (err error) { const op = errors.Op("vault.CleanupOAuthSecrets") + durationObserver := servermon.DurationObserver(servermon.VaultQueryDurationHistogram, string(op)) + defer durationObserver() + defer servermon.ErrorCounter(servermon.VaultQueryErrorCount, &err, string(op)) + client, err := s.client(ctx) if err != nil { return errors.E(op, err) @@ -406,9 +462,13 @@ func (s *VaultStore) GetOAuthSecretsBasePath() string { } // GetOAuthSecret returns the current HS256 (symmetric encryption) secret used to sign OAuth session tokens. -func (s *VaultStore) GetOAuthSecret(ctx context.Context) ([]byte, error) { +func (s *VaultStore) GetOAuthSecret(ctx context.Context) (_ []byte, err error) { const op = errors.Op("vault.GetOAuthSecret") + durationObserver := servermon.DurationObserver(servermon.VaultQueryDurationHistogram, string(op)) + defer durationObserver() + defer servermon.ErrorCounter(servermon.VaultQueryErrorCount, &err, string(op)) + client, err := s.client(ctx) if err != nil { return nil, errors.E(op, err) @@ -447,9 +507,13 @@ func (s *VaultStore) GetOAuthSecret(ctx context.Context) ([]byte, error) { } // PutOAuthSecret puts a HS256 (symmetric encryption) secret into the credentials store for signing OAuth session tokens. -func (s *VaultStore) PutOAuthSecret(ctx context.Context, raw []byte) error { +func (s *VaultStore) PutOAuthSecret(ctx context.Context, raw []byte) (err error) { const op = errors.Op("vault.PutOAuthSecret") + durationObserver := servermon.DurationObserver(servermon.VaultQueryDurationHistogram, string(op)) + defer durationObserver() + defer servermon.ErrorCounter(servermon.VaultQueryErrorCount, &err, string(op)) + client, err := s.client(ctx) if err != nil { return errors.E(op, err) @@ -469,9 +533,13 @@ func (s *VaultStore) GetOAuthSecretPath() string { } // GetOAuthSessionStoreSecret returns the current secret used to store session tokens. -func (s *VaultStore) GetOAuthSessionStoreSecret(ctx context.Context) ([]byte, error) { +func (s *VaultStore) GetOAuthSessionStoreSecret(ctx context.Context) (_ []byte, err error) { const op = errors.Op("vault.GetOAuthSessionStoreSecret") + durationObserver := servermon.DurationObserver(servermon.VaultQueryDurationHistogram, string(op)) + defer durationObserver() + defer servermon.ErrorCounter(servermon.VaultQueryErrorCount, &err, string(op)) + client, err := s.client(ctx) if err != nil { return nil, errors.E(op, err) @@ -510,9 +578,13 @@ func (s *VaultStore) GetOAuthSessionStoreSecret(ctx context.Context) ([]byte, er } // PutOAuthSessionStoreSecret puts a secret into the credentials store for secure storage of session tokens. -func (s *VaultStore) PutOAuthSessionStoreSecret(ctx context.Context, raw []byte) error { +func (s *VaultStore) PutOAuthSessionStoreSecret(ctx context.Context, raw []byte) (err error) { const op = errors.Op("vault.PutOAuthSessionStoreSecret") + durationObserver := servermon.DurationObserver(servermon.VaultQueryDurationHistogram, string(op)) + defer durationObserver() + defer servermon.ErrorCounter(servermon.VaultQueryErrorCount, &err, string(op)) + client, err := s.client(ctx) if err != nil { return errors.E(op, err) @@ -533,9 +605,13 @@ func (s *VaultStore) GetOAuthSessionStoreSecretPath() string { // deleteControllerCredentials removes the credentials associated with the controller in // the vault service. -func (s *VaultStore) deleteControllerCredentials(ctx context.Context, controllerName string) error { +func (s *VaultStore) deleteControllerCredentials(ctx context.Context, controllerName string) (err error) { const op = errors.Op("vault.deleteControllerCredentials") + durationObserver := servermon.DurationObserver(servermon.VaultQueryDurationHistogram, string(op)) + defer durationObserver() + defer servermon.ErrorCounter(servermon.VaultQueryErrorCount, &err, string(op)) + client, err := s.client(ctx) if err != nil { return errors.E(op, err)