Skip to content

Commit

Permalink
Initial monitoring cleanup.
Browse files Browse the repository at this point in the history
  • Loading branch information
@alesstimec
alesstimec committed Jun 11, 2024
1 parent a365444 commit b094671
Show file tree
Hide file tree
Showing 5 changed files with 62 additions and 182 deletions.
38 changes: 33 additions & 5 deletions internal/auth/oauth2.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ import (
"github.com/canonical/jimm/api/params"
"github.com/canonical/jimm/internal/dbmodel"
"github.com/canonical/jimm/internal/errors"
"github.com/canonical/jimm/internal/servermon"
)

const (
Expand Down Expand Up @@ -305,8 +306,20 @@ func (as *AuthenticationService) MintSessionToken(email string, secretKey string
}

// VerifySessionToken calls the exported VerifySessionToken function.
func (as *AuthenticationService) VerifySessionToken(token string, secretKey string) (jwt.Token, error) {
return VerifySessionToken(token, secretKey)
func (as *AuthenticationService) VerifySessionToken(token string, secretKey string) (_ jwt.Token, err error) {
defer func() {
if err != nil {
servermon.AuthenticationFailCount.WithLabelValues("VerifySessionToken").Inc()
} else {
servermon.AuthenticationSuccessCount.WithLabelValues("VerifySessionToken").Inc()
}
}()

jwt, err := VerifySessionToken(token, secretKey)
if err != nil {
return nil, errors.E(err)
}
return jwt, nil
}

// UpdateIdentity updates the database with the display name and access token set for the user.
Expand Down Expand Up @@ -377,7 +390,15 @@ func VerifySessionToken(token string, secretKey string) (jwt.Token, error) {
}

// VerifyClientCredentials verifies the provided client ID and client secret.
func (as *AuthenticationService) VerifyClientCredentials(ctx context.Context, clientID string, clientSecret string) error {
func (as *AuthenticationService) VerifyClientCredentials(ctx context.Context, clientID string, clientSecret string) (err error) {
defer func() {
if err != nil {
servermon.AuthenticationFailCount.WithLabelValues("VerifyClientCredentials").Inc()
} else {
servermon.AuthenticationSuccessCount.WithLabelValues("VerifyClientCredentials").Inc()
}
}()

cfg := clientcredentials.Config{
ClientID: clientID,
ClientSecret: clientSecret,
Expand All @@ -386,7 +407,7 @@ func (as *AuthenticationService) VerifyClientCredentials(ctx context.Context, cl
AuthStyle: oauth2.AuthStyle(as.oauthConfig.Endpoint.AuthStyle),
}

_, err := cfg.Token(ctx)
_, err = cfg.Token(ctx)
if err != nil {
zapctx.Error(ctx, "client credential verification failed", zap.Error(err))
return errors.E(errors.CodeUnauthorized, "invalid client credentials")
Expand Down Expand Up @@ -425,8 +446,15 @@ func (as *AuthenticationService) CreateBrowserSession(
// AuthenticateBrowserSession updates the session for a browser, additionally
// retrieving new access tokens upon expiry. If this cannot be done, the cookie
// is deleted and an error is returned.
func (as *AuthenticationService) AuthenticateBrowserSession(ctx context.Context, w http.ResponseWriter, req *http.Request) (context.Context, error) {
func (as *AuthenticationService) AuthenticateBrowserSession(ctx context.Context, w http.ResponseWriter, req *http.Request) (_ context.Context, err error) {
const op = errors.Op("auth.AuthenticationService.AuthenticateBrowserSession")
defer func() {
if err != nil {
servermon.AuthenticationFailCount.WithLabelValues("AuthenticateBrowserSession").Inc()
} else {
servermon.AuthenticationSuccessCount.WithLabelValues("AuthenticateBrowserSession").Inc()
}
}()

session, err := as.sessionStore.Get(req, SessionName)
if err != nil {
Expand Down
2 changes: 2 additions & 0 deletions internal/jimm/watcher.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import (
"github.com/canonical/jimm/internal/db"
"github.com/canonical/jimm/internal/dbmodel"
"github.com/canonical/jimm/internal/errors"
"github.com/canonical/jimm/internal/servermon"
)

// Publisher defines the interface used by the Watcher
Expand Down Expand Up @@ -353,6 +354,7 @@ func (w *Watcher) watchController(ctx context.Context, ctl *dbmodel.Controller)
if err != nil {
return errors.E(op, err)
}
servermon.MonitorDeltasReceivedCount.WithLabelValues(ctl.UUID).Add(float64(len(deltas)))
for _, d := range deltas {
eid := d.Entity.EntityId()
ctx := zapctx.WithFields(ctx, zap.String("model-uuid", eid.ModelUUID), zap.String("kind", eid.Kind), zap.String("id", eid.Id))
Expand Down
17 changes: 9 additions & 8 deletions internal/jujuapi/modelmanager.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -153,15 +153,16 @@ func (r *controllerRoot) CreateModel(ctx context.Context, args jujuparams.ModelC
return jujuparams.ModelInfo{}, errors.E(op, err)
}
info, err := r.jimm.AddModel(ctx, r.user, &mca)
if err == nil {
servermon.ModelsCreatedCount.Inc()
if r.controllerUUIDMasking {
info.ControllerUUID = r.params.ControllerUUID
}
return *info, nil
if err != nil {
servermon.ModelsCreatedFailCount.Inc()
return jujuparams.ModelInfo{}, errors.E(op, err)
}

servermon.ModelsCreatedCount.Inc()
if r.controllerUUIDMasking {
info.ControllerUUID = r.params.ControllerUUID
}
servermon.ModelsCreatedFailCount.Inc()
return jujuparams.ModelInfo{}, errors.E(op, err)
return *info, nil
}

// DestroyModels implements the ModelManager facade's DestroyModels
Expand Down
158 changes: 18 additions & 140 deletions internal/servermon/monitoring.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,171 +5,64 @@
package servermon

import (
"github.com/juju/mgomonitor"
"github.com/prometheus/client_golang/prometheus"
)

var (
AuthenticationFailCount = prometheus.NewCounterVec(prometheus.CounterOpts{
Namespace: "jimm",
Name: "auth",
Help: "The number of failed authentications.",
}, []string{"method"})
AuthenticationSuccessCount = prometheus.NewCounterVec(prometheus.CounterOpts{
Namespace: "jimm",
Name: "auth",
Help: "The number of successful authentications.",
}, []string{"method"})
QueryTimeAuditLogCleanUpHistogram = prometheus.NewHistogram(prometheus.HistogramOpts{
Namespace: "jem",
Namespace: "jimm",
Name: "db_query_audit_clean_up_duration_seconds",
Help: "Histogram of query time for audit_log clean up in seconds",
Buckets: []float64{.005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10},
})
AuthenticationFailCount = prometheus.NewCounter(prometheus.CounterOpts{
Namespace: "jem",
Subsystem: "auth",
Name: "authentication_fail",
Help: "The number of failed authentications.",
})
AuthenticationSuccessCount = prometheus.NewCounter(prometheus.CounterOpts{
Namespace: "jem",
Subsystem: "auth",
Name: "authentication_success",
Help: "The number of successful authentications.",
})
AuthenticatorPoolGet = prometheus.NewCounter(prometheus.CounterOpts{
Namespace: "jem",
Subsystem: "auth",
Name: "pool_get",
Help: "The number of times an Authenticator has been retrieved from the pool.",
})
AuthenticatorPoolNew = prometheus.NewCounter(prometheus.CounterOpts{
Namespace: "jem",
Subsystem: "auth",
Name: "pool_new",
Help: "The number of times a new Authenticator has been created by the pool.",
})
AuthenticatorPoolPut = prometheus.NewCounter(prometheus.CounterOpts{
Namespace: "jem",
Subsystem: "auth",
Name: "pool_put",
Help: "The number of times an Authenticator has been replaced into the pool.",
})
ConcurrentWebsocketConnections = prometheus.NewGauge(prometheus.GaugeOpts{
Namespace: "jem",
Namespace: "jimm",
Subsystem: "websocket",
Name: "concurrent_connections",
Help: "The number of concurrent websocket connections",
})
DatabaseFailCount = prometheus.NewCounter(prometheus.CounterOpts{
Namespace: "jem",
Subsystem: "database",
Name: "fail_count",
Help: "The number of times a database error was considered fatal.",
})
DeployedUnitCount = prometheus.NewCounter(prometheus.CounterOpts{
Namespace: "jem",
Subsystem: "websocket",
Name: "deployed_unit_count",
Help: "The number of deployed units.",
})
LoginFailCount = prometheus.NewCounter(prometheus.CounterOpts{
Namespace: "jem",
Subsystem: "websocket",
Name: "login_fail_count",
Help: "The number of failed logins attempted.",
})
LoginRedirectCount = prometheus.NewCounter(prometheus.CounterOpts{
Namespace: "jem",
Subsystem: "websocket",
Name: "login_redirect_count",
Help: "The number of logins redirected to another controller.",
})
LoginSuccessCount = prometheus.NewCounter(prometheus.CounterOpts{
Namespace: "jem",
Subsystem: "websocket",
Name: "login_success_count",
Help: "The number of successful logins completed.",
})
ModelLifetime = prometheus.NewHistogram(prometheus.HistogramOpts{
Namespace: "jem",
Subsystem: "health",
Name: "model_lifetime",
Help: "The length of time (in hours) models had existed at the point they are destroyed.",
// Buckets are in hours for this histogram.
Buckets: []float64{
1.0 / 6,
1.0 / 2,
1,
6,
24,
7 * 24,
28 * 24,
6 * 28 * 24,
365 * 24,
2 * 365 * 24,
5 * 365 * 24,
},
})
ModelsCreatedCount = prometheus.NewCounter(prometheus.CounterOpts{
Namespace: "jem",
Namespace: "jimm",
Subsystem: "websocket",
Name: "models_created_count",
Help: "The number of models created.",
})
ModelsCreatedFailCount = prometheus.NewCounter(prometheus.CounterOpts{
Namespace: "jem",
Namespace: "jimm",
Subsystem: "websocket",
Name: "models_created_fail_count",
Help: "The number of fails attempting to create models.",
})
ModelsDestroyedCount = prometheus.NewCounter(prometheus.CounterOpts{
Namespace: "jem",
Subsystem: "websocket",
Name: "models_destroyed_count",
Help: "The number of models destroyed.",
})
MonitorDeltasReceivedCount = prometheus.NewCounterVec(prometheus.CounterOpts{
Namespace: "jem",
Namespace: "jimm",
Subsystem: "monitor",
Name: "deltas_received_count",
Help: "The number of watcher deltas received.",
}, []string{"controller"})
MonitorDeltaBatchesReceivedCount = prometheus.NewCounterVec(prometheus.CounterOpts{
Namespace: "jem",
Subsystem: "monitor",
Name: "delta_batches_received_count",
Help: "The number of watcher delta batches received.",
}, []string{"controller"})
MonitorErrorsCount = prometheus.NewCounterVec(prometheus.CounterOpts{
Namespace: "jem",
Namespace: "jimm",
Subsystem: "monitor",
Name: "errors_count",
Help: "The number of monitoring errors found.",
}, []string{"controller"})
MonitorLeaseGauge = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Namespace: "jem",
Subsystem: "monitor",
Name: "lease_gauge",
Help: "The number of current monitor leases held",
}, []string{"controller"})
requestDuration = prometheus.NewSummaryVec(prometheus.SummaryOpts{
Namespace: "jem",
Subsystem: "handler",
Name: "request_duration",
Help: "The duration of a web request in seconds.",
}, []string{"path_pattern"})
StatsCollectFailCount = prometheus.NewCounter(prometheus.CounterOpts{
Namespace: "jem",
Subsystem: "health",
Name: "stats_collect_fail_count",
Help: "The number of times we failed to collect stats from mongo.",
})
VaultConfigured = prometheus.NewGauge(prometheus.GaugeOpts{
Namespace: "jem",
Namespace: "jimm",
Subsystem: "vault",
Name: "configured",
Help: "Indicator that a vault is configured.",
})
VaultSecretRefreshes = prometheus.NewCounter(prometheus.CounterOpts{
Namespace: "jem",
Subsystem: "vault",
Name: "secret_refreshes",
Help: "The number of times the secret has been refreshed.",
})
WebsocketRequestDuration = prometheus.NewSummaryVec(prometheus.SummaryOpts{
Namespace: "jem",
Namespace: "jimm",
Subsystem: "websocket",
Name: "request_duration",
Help: "The duration of a websocket request in seconds.",
Expand All @@ -179,26 +72,11 @@ var (
func init() {
prometheus.MustRegister(AuthenticationFailCount)
prometheus.MustRegister(AuthenticationSuccessCount)
prometheus.MustRegister(AuthenticatorPoolGet)
prometheus.MustRegister(AuthenticatorPoolNew)
prometheus.MustRegister(AuthenticatorPoolPut)
prometheus.MustRegister(ConcurrentWebsocketConnections)
prometheus.MustRegister(DatabaseFailCount)
prometheus.MustRegister(DeployedUnitCount)
prometheus.MustRegister(LoginFailCount)
prometheus.MustRegister(LoginRedirectCount)
prometheus.MustRegister(LoginSuccessCount)
prometheus.MustRegister(ModelLifetime)
prometheus.MustRegister(ModelsCreatedCount)
prometheus.MustRegister(ModelsCreatedFailCount)
prometheus.MustRegister(MonitorDeltasReceivedCount)
prometheus.MustRegister(MonitorDeltaBatchesReceivedCount)
prometheus.MustRegister(MonitorErrorsCount)
prometheus.MustRegister(MonitorLeaseGauge)
prometheus.MustRegister(requestDuration)
prometheus.MustRegister(StatsCollectFailCount)
prometheus.MustRegister(VaultConfigured)
prometheus.MustRegister(VaultSecretRefreshes)
prometheus.MustRegister(WebsocketRequestDuration)
prometheus.MustRegister(mgomonitor.NewCollector("jem"))
}
29 changes: 0 additions & 29 deletions internal/servermon/request.go
View file

This file was deleted.

0 comments on commit b094671

Please sign in to comment.