diff --git a/auth/client/client_test.go b/auth/client/client_test.go index 6fa3eb5eea..63b25209ee 100644 --- a/auth/client/client_test.go +++ b/auth/client/client_test.go @@ -65,6 +65,7 @@ var _ = Describe("Client", func() { config.Config.UserAgent = testHttp.NewUserAgent() config.Config.ServiceSecret = authTest.NewServiceSecret() config.ExternalConfig.Address = testHttp.NewAddress() + config.ExternalConfig.PathPrefix = "auth" config.ExternalConfig.UserAgent = testHttp.NewUserAgent() config.ExternalConfig.ServerSessionTokenSecret = serverTokenSecret config.ExternalConfig.ServerSessionTokenTimeout = time.Duration(serverTokenTimeout) * time.Second @@ -126,6 +127,7 @@ var _ = Describe("Client", func() { config.Config.UserAgent = testHttp.NewUserAgent() config.Config.ServiceSecret = authTest.NewServiceSecret() config.ExternalConfig.Address = server.URL() + config.ExternalConfig.PathPrefix = "auth" config.ExternalConfig.UserAgent = testHttp.NewUserAgent() config.ExternalConfig.ServerSessionTokenSecret = serverTokenSecret authorizeAs = platform.AuthorizeAsService diff --git a/auth/client/external.go b/auth/client/external.go index 9c6c363bc6..6096bc3a76 100644 --- a/auth/client/external.go +++ b/auth/client/external.go @@ -68,6 +68,8 @@ type ExternalConfig struct { *platform.Config ServerSessionTokenSecret string `envconfig:"TIDEPOOL_AUTH_CLIENT_EXTERNAL_SERVER_SESSION_TOKEN_SECRET"` ServerSessionTokenTimeout time.Duration `envconfig:"TIDEPOOL_AUTH_CLIENT_EXTERNAL_SERVER_SESSION_TOKEN_TIMEOUT" default:"1h"` + // PathPrefix to prepend to the path of any service calls (if any). + PathPrefix string `envconfig:"TIDEPOOL_AUTH_CLIENT_EXTERNAL_PATH_PREFIX" default:"auth"` } func NewExternalConfig() *ExternalConfig { @@ -107,6 +109,7 @@ type External struct { serverSessionTokenMutex sync.Mutex serverSessionTokenSafe string closingChannel chan chan bool + PathPrefix string // PathPrefix is the prefix to prepend to all external URL path calls to the auth service (if any) } func NewExternal(cfg *ExternalConfig, authorizeAs platform.AuthorizeAs, name string, lgr log.Logger) (*External, error) { @@ -135,6 +138,7 @@ func NewExternal(cfg *ExternalConfig, authorizeAs platform.AuthorizeAs, name str name: name, serverSessionTokenSecret: cfg.ServerSessionTokenSecret, serverSessionTokenTimeout: cfg.ServerSessionTokenTimeout, + PathPrefix: cfg.PathPrefix, }, nil } @@ -200,7 +204,7 @@ func (e *External) ValidateSessionToken(ctx context.Context, token string) (requ IsServer bool UserID string } - if err := e.client.RequestData(ctx, "GET", e.client.ConstructURL("auth", "token", token), nil, nil, &result); err != nil { + if err := e.client.RequestData(ctx, "GET", e.client.ConstructURL(e.PathPrefix, "token", token), nil, nil, &result); err != nil { return nil, err } @@ -329,7 +333,7 @@ func (e *External) refreshServerSessionToken() error { e.logger.Debug("Refreshing server session token") requestMethod := "POST" - requestURL := e.client.ConstructURL("auth", "serverlogin") + requestURL := e.client.ConstructURL(e.PathPrefix, "serverlogin") request, err := http.NewRequest(requestMethod, requestURL, nil) if err != nil { return errors.Wrapf(err, "unable to create new request for %s %s", requestMethod, requestURL) diff --git a/auth/service/service/service.go b/auth/service/service/service.go index f78ed0412f..8ca06f8239 100644 --- a/auth/service/service/service.go +++ b/auth/service/service/service.go @@ -129,7 +129,7 @@ func (s *Service) Initialize(provider application.Provider) error { if err := s.initializeTwiistServiceAccountAuthorizer(); err != nil { return err } - return s.initializeUserEventsHandler() + return s.initializeUserEventsHandler(provider) } func (s *Service) Terminate() { @@ -468,13 +468,20 @@ func (s *Service) terminateAuthClient() { } } -func (s *Service) initializeUserEventsHandler() error { +func (s *Service) initializeUserEventsHandler(provider application.Provider) error { s.Logger().Debug("Initializing user events handler") - ctx := logInternal.NewContextWithLogger(context.Background(), s.Logger()) - handler := authEvents.NewUserDataDeletionHandler(ctx, s.authClient) - handlers := []eventsCommon.EventHandler{handler} - runner := events.NewRunner(handlers) + var runner events.Runner + + configReporter := provider.ConfigReporter().WithScopes("user", "events", "handler") + if configReporter.GetWithDefault("disable", "") != "true" { + ctx := logInternal.NewContextWithLogger(context.Background(), s.Logger()) + handler := authEvents.NewUserDataDeletionHandler(ctx, s.authClient) + handlers := []eventsCommon.EventHandler{handler} + runner = events.NewRunner(handlers) + } else { + runner = events.NewNoopRunner() + } if err := runner.Initialize(); err != nil { return errors.Wrap(err, "unable to initialize events runner") diff --git a/auth/service/service/service_test.go b/auth/service/service/service_test.go index e98473d5bc..b1d69876d6 100644 --- a/auth/service/service/service_test.go +++ b/auth/service/service/service_test.go @@ -49,7 +49,7 @@ var _ = Describe("Service", func() { server = NewServer() server.AppendHandlers( CombineHandlers( - VerifyRequest("POST", "/auth/serverlogin"), + VerifyRequest("POST", "/serverlogin"), // by default the path prefix is empty to the auth service unless set in the env var TIDEPOOL_AUTH_CLIENT_EXTERNAL_PATH_PREFIX VerifyHeaderKV("X-Tidepool-Server-Name", *provider.NameOutput), VerifyHeaderKV("X-Tidepool-Server-Secret", serverSecret), VerifyBody(nil), diff --git a/blob/service/service.go b/blob/service/service.go index a8381e2bd4..ee99021c65 100644 --- a/blob/service/service.go +++ b/blob/service/service.go @@ -69,7 +69,7 @@ func (s *Service) Initialize(provider application.Provider) error { if err := s.initializeBlobClient(); err != nil { return err } - if err := s.initializeUserEventsHandler(); err != nil { + if err := s.initializeUserEventsHandler(provider); err != nil { return err } return s.initializeRouter() @@ -211,13 +211,20 @@ func (s *Service) terminateDeviceLogsUnstructuredStore() { } } -func (s *Service) initializeUserEventsHandler() error { +func (s *Service) initializeUserEventsHandler(provider application.Provider) error { s.Logger().Debug("Initializing user events handler") - ctx := logInternal.NewContextWithLogger(context.Background(), s.Logger()) - handler := blobEvents.NewUserDataDeletionHandler(ctx, s.blobClient) - handlers := []eventsCommon.EventHandler{handler} - runner := events.NewRunner(handlers) + var runner events.Runner + + configReporter := provider.ConfigReporter().WithScopes("user", "events", "handler") + if configReporter.GetWithDefault("disable", "") != "true" { + ctx := logInternal.NewContextWithLogger(context.Background(), s.Logger()) + handler := blobEvents.NewUserDataDeletionHandler(ctx, s.blobClient) + handlers := []eventsCommon.EventHandler{handler} + runner = events.NewRunner(handlers) + } else { + runner = events.NewNoopRunner() + } if err := runner.Initialize(); err != nil { return errors.Wrap(err, "unable to initialize events runner") diff --git a/blob/service/service_test.go b/blob/service/service_test.go index 251c376a17..b0246c707c 100644 --- a/blob/service/service_test.go +++ b/blob/service/service_test.go @@ -47,7 +47,7 @@ var _ = Describe("Service", func() { server = NewServer() server.AppendHandlers( CombineHandlers( - VerifyRequest(http.MethodPost, "/auth/serverlogin"), + VerifyRequest(http.MethodPost, "/serverlogin"), VerifyHeaderKV("X-Tidepool-Server-Name", *provider.NameOutput), VerifyHeaderKV("X-Tidepool-Server-Secret", serverSecret), VerifyBody(nil), diff --git a/client/client.go b/client/client.go index fe9298f97c..47406fbf49 100644 --- a/client/client.go +++ b/client/client.go @@ -58,7 +58,11 @@ func NewWithErrorParser(cfg *Config, errorResponseParser ErrorResponseParser) (* func (c *Client) ConstructURL(paths ...string) string { segments := []string{} for _, path := range paths { - segments = append(segments, url.PathEscape(strings.Trim(path, "/"))) + escapedPath := url.PathEscape(strings.Trim(path, "/")) + if escapedPath == "" { + continue + } + segments = append(segments, escapedPath) } return fmt.Sprintf("%s/%s", strings.TrimRight(c.address, "/"), strings.Join(segments, "/")) } diff --git a/client/config.go b/client/config.go index bbc5115f93..aa1a003aa3 100644 --- a/client/config.go +++ b/client/config.go @@ -1,6 +1,7 @@ package client import ( + "fmt" "net/url" "github.com/kelseyhightower/envconfig" @@ -37,7 +38,7 @@ func (c *Config) Validate() error { if c.Address == "" { return errors.New("address is missing") } else if _, err := url.Parse(c.Address); err != nil { - return errors.New("address is invalid") + return fmt.Errorf("address is invalid: %w", err) } return nil diff --git a/client/config_test.go b/client/config_test.go index 4c22fbcf29..62090aca71 100644 --- a/client/config_test.go +++ b/client/config_test.go @@ -86,7 +86,7 @@ var _ = Describe("Config", func() { It("returns an error if the address is not a parseable URL", func() { cfg.Address = "Not%Parseable" - Expect(cfg.Validate()).To(MatchError("address is invalid")) + Expect(cfg.Validate()).To(MatchError("address is invalid: parse \"Not%Parseable\": invalid URL escape \"%Pa\"")) }) It("returns success", func() { diff --git a/data/service/api/v1/datasets_data_create.go b/data/service/api/v1/datasets_data_create.go index 9339b445c9..cc5e56d8f5 100644 --- a/data/service/api/v1/datasets_data_create.go +++ b/data/service/api/v1/datasets_data_create.go @@ -149,7 +149,8 @@ func CollectProvenanceInfo(ctx context.Context, req *rest.Request, authDetails r if token != "" && shouldHaveJWT(authDetails) { claims := &TokenClaims{} if _, _, err := jwt.NewParser().ParseUnverified(token, claims); err != nil { - lgr.WithError(err).Warn("Unable to parse access token for provenance") + // temp comment out because makes realtime viewing rough + // lgr.WithError(err).Warn("Unable to parse access token for provenance") } else { provenance.ClientID = claims.ClientID } diff --git a/data/service/service/standard.go b/data/service/service/standard.go index 3d549e083d..50297c1f8f 100644 --- a/data/service/service/standard.go +++ b/data/service/service/standard.go @@ -90,7 +90,7 @@ func (s *Standard) Initialize(provider application.Provider) error { if err := s.initializeDataSourceClient(); err != nil { return err } - if err := s.initializeUserEventsHandler(); err != nil { + if err := s.initializeUserEventsHandler(provider); err != nil { return err } if err := s.initializeTwiistServiceAccountAuthorizer(); err != nil { @@ -430,14 +430,22 @@ func (s *Standard) initializeServer() error { return nil } -func (s *Standard) initializeUserEventsHandler() error { +func (s *Standard) initializeUserEventsHandler(provider application.Provider) error { s.Logger().Debug("Initializing user events handler") sarama.Logger = log.New(os.Stdout, "SARAMA ", log.LstdFlags|log.Lshortfile) - ctx := logInternal.NewContextWithLogger(context.Background(), s.Logger()) - handler := dataEvents.NewUserDataDeletionHandler(ctx, s.dataStore, s.dataSourceStructuredStore) - handlers := []eventsCommon.EventHandler{handler} - runner := events.NewRunner(handlers) + var runner events.Runner + + configReporter := provider.ConfigReporter().WithScopes("user", "events", "handler") + if configReporter.GetWithDefault("disable", "") != "true" { + ctx := logInternal.NewContextWithLogger(context.Background(), s.Logger()) + handler := dataEvents.NewUserDataDeletionHandler(ctx, s.dataStore, s.dataSourceStructuredStore) + handlers := []eventsCommon.EventHandler{handler} + runner = events.NewRunner(handlers) + } else { + runner = events.NewNoopRunner() + } + if err := runner.Initialize(); err != nil { return errors.Wrap(err, "unable to initialize user events handler runner") } diff --git a/data/store/mongo/mongo_data_set.go b/data/store/mongo/mongo_data_set.go index 5a3a5db386..ee71dbfc56 100644 --- a/data/store/mongo/mongo_data_set.go +++ b/data/store/mongo/mongo_data_set.go @@ -40,16 +40,6 @@ func (d *DataSetRepository) EnsureIndexes() error { Options: options.Index(). SetName("UserIdTypeWeighted_v2"), }, - { - Keys: bson.D{ - {Key: "origin.id", Value: 1}, - {Key: "type", Value: 1}, - {Key: "deletedTime", Value: -1}, - {Key: "_active", Value: 1}, - }, - Options: options.Index(). - SetName("OriginId"), - }, { Keys: bson.D{ {Key: "uploadId", Value: 1}, @@ -330,14 +320,14 @@ func (d *DataSetRepository) GetDataSetsForUserByID(ctx context.Context, userID s SetSort(bson.M{"createdTime": -1}) cursor, err := d.Find(ctx, selector, opts) - loggerFields := log.Fields{"userId": userID, "dataSetsCount": len(dataSets), "duration": time.Since(now) / time.Microsecond} - log.LoggerFromContext(ctx).WithFields(loggerFields).WithError(err).Debug("GetDataSetsForUserByID") - if err != nil { return nil, errors.Wrap(err, "unable to get data sets for user by id") } - if err = cursor.All(ctx, &dataSets); err != nil { + err = cursor.All(ctx, &dataSets) + loggerFields := log.Fields{"userId": userID, "dataSetsCount": len(dataSets), "duration": time.Since(now) / time.Microsecond} + log.LoggerFromContext(ctx).WithFields(loggerFields).WithError(err).Debug("GetDataSetsForUserByID") + if err != nil { return nil, errors.Wrap(err, "unable to decode data sets for user by id") } diff --git a/data/store/mongo/mongo_datum.go b/data/store/mongo/mongo_datum.go index 29776f9d97..7c007acccb 100644 --- a/data/store/mongo/mongo_datum.go +++ b/data/store/mongo/mongo_datum.go @@ -471,7 +471,8 @@ func (d *DatumRepository) ArchiveDeviceDataUsingHashesFromDataSet(ctx context.Co "modifiedTime": timestamp, } unset := bson.M{} - updateInfo, err = d.UpdateMany(ctx, selector, d.ConstructUpdate(set, unset)) + opts := options.Update() + updateInfo, err = d.UpdateMany(ctx, selector, d.ConstructUpdate(set, unset), opts) } loggerFields := log.Fields{"userId": dataSet.UserID, "deviceId": *dataSet.DeviceID, "updateInfo": updateInfo, "duration": time.Since(now) / time.Microsecond} diff --git a/data/store/mongo/mongo_test.go b/data/store/mongo/mongo_test.go index f941348090..4f0c121142 100644 --- a/data/store/mongo/mongo_test.go +++ b/data/store/mongo/mongo_test.go @@ -371,10 +371,6 @@ var _ = Describe("Mongo", Label("mongodb", "slow", "integration"), func() { "Key": Equal(storeStructuredMongoTest.MakeKeySlice("_userId", "_active", "type", "-time")), "Name": Equal("UserIdTypeWeighted_v2"), }), - MatchFields(IgnoreExtras, Fields{ - "Key": Equal(storeStructuredMongoTest.MakeKeySlice("origin.id", "type", "-deletedTime", "_active")), - "Name": Equal("OriginId"), - }), MatchFields(IgnoreExtras, Fields{ "Key": Equal(storeStructuredMongoTest.MakeKeySlice("uploadId", "type", "-deletedTime", "_active")), "Name": Equal("UploadId"), diff --git a/env.sh b/env.sh index 13f8a66e31..22ec573817 100644 --- a/env.sh +++ b/env.sh @@ -32,6 +32,7 @@ export TIDEPOOL_USER_CLIENT_ADDRESS="http://localhost:8009" export TIDEPOOL_AUTH_CLIENT_EXTERNAL_ADDRESS="http://localhost:8009" export TIDEPOOL_AUTH_CLIENT_EXTERNAL_SERVER_SESSION_TOKEN_SECRET="This needs to be the same secret everywhere. YaHut75NsK1f9UKUXuWqxNN0RUwHFBCy" +export TIDEPOOL_AUTH_CLIENT_EXTERNAL_PATH_PREFIX="auth" export TIDEPOOL_AUTH_SERVICE_SERVER_ADDRESS=":9222" export TIDEPOOL_BLOB_SERVICE_SERVER_ADDRESS=":9225" diff --git a/events/events.go b/events/events.go index e1d3129300..240850c756 100644 --- a/events/events.go +++ b/events/events.go @@ -47,3 +47,28 @@ func (r *runner) Terminate() error { } return nil } + +type noopRunner struct { + terminate chan struct{} +} + +func (n *noopRunner) Initialize() error { + n.terminate = make(chan struct{}, 0) + return nil +} + +func (n *noopRunner) Run() error { + <-n.terminate + return nil +} + +func (n *noopRunner) Terminate() error { + n.terminate <- struct{}{} + return nil +} + +var _ Runner = &noopRunner{} + +func NewNoopRunner() Runner { + return &noopRunner{} +} diff --git a/platform/config_test.go b/platform/config_test.go index edba20a423..841e15294a 100644 --- a/platform/config_test.go +++ b/platform/config_test.go @@ -110,7 +110,7 @@ var _ = Describe("Config", func() { It("returns an error if the address is not a parseable URL", func() { cfg.Address = "Not%Parseable" - Expect(cfg.Validate()).To(MatchError("address is invalid")) + Expect(cfg.Validate()).To(MatchError("address is invalid: parse \"Not%Parseable\": invalid URL escape \"%Pa\"")) }) It("returns success", func() { diff --git a/prescription/application/application_test.go b/prescription/application/application_test.go index 9e2d511bd2..6f27e00087 100644 --- a/prescription/application/application_test.go +++ b/prescription/application/application_test.go @@ -53,7 +53,7 @@ var _ = Describe("Application", func() { server = NewServer() server.AppendHandlers( CombineHandlers( - VerifyRequest(http.MethodPost, "/auth/serverlogin"), + VerifyRequest(http.MethodPost, "/serverlogin"), VerifyHeaderKV("X-Tidepool-Server-Name", *prvdr.NameOutput), VerifyHeaderKV("X-Tidepool-Server-Secret", serverSecret), VerifyBody(nil), diff --git a/service/service/DEPRECATED_service_test.go b/service/service/DEPRECATED_service_test.go index 95e148f7be..621256e107 100644 --- a/service/service/DEPRECATED_service_test.go +++ b/service/service/DEPRECATED_service_test.go @@ -39,7 +39,7 @@ var _ = Describe("DEPRECATEDService", func() { Expect(server).ToNot(BeNil()) server.AppendHandlers( CombineHandlers( - VerifyRequest("POST", "/auth/serverlogin"), + VerifyRequest("POST", "/serverlogin"), VerifyHeaderKV("X-Tidepool-Server-Name", *provider.NameOutput), VerifyHeaderKV("X-Tidepool-Server-Secret", serverSecret), VerifyBody(nil), diff --git a/store/structured/mongo/config.go b/store/structured/mongo/config.go index 72bdf08031..8212f760d1 100644 --- a/store/structured/mongo/config.go +++ b/store/structured/mongo/config.go @@ -28,15 +28,16 @@ func LoadConfig() (*Config, error) { // Config describe parameters need to make a connection to a Mongo database type Config struct { - Scheme string `json:"scheme" envconfig:"TIDEPOOL_STORE_SCHEME"` - Addresses []string `json:"addresses" envconfig:"TIDEPOOL_STORE_ADDRESSES" required:"true"` - TLS bool `json:"tls" envconfig:"TIDEPOOL_STORE_TLS" default:"true"` - Database string `json:"database" envconfig:"TIDEPOOL_STORE_DATABASE" required:"true"` - CollectionPrefix string `json:"collectionPrefix" envconfig:"TIDEPOOL_STORE_COLLECTION_PREFIX"` - Username *string `json:"-" envconfig:"TIDEPOOL_STORE_USERNAME"` - Password *string `json:"-" envconfig:"TIDEPOOL_STORE_PASSWORD"` - Timeout time.Duration `json:"timeout" envconfig:"TIDEPOOL_STORE_TIMEOUT" default:"60s"` - OptParams *string `json:"optParams" envconfig:"TIDEPOOL_STORE_OPT_PARAMS"` + Scheme string `json:"scheme" envconfig:"TIDEPOOL_STORE_SCHEME"` + Addresses []string `json:"addresses" envconfig:"TIDEPOOL_STORE_ADDRESSES" required:"true"` + TLS bool `json:"tls" envconfig:"TIDEPOOL_STORE_TLS" default:"true"` + Database string `json:"database" envconfig:"TIDEPOOL_STORE_DATABASE" required:"true"` + CollectionPrefix string `json:"collectionPrefix" envconfig:"TIDEPOOL_STORE_COLLECTION_PREFIX"` + Username *string `json:"-" envconfig:"TIDEPOOL_STORE_USERNAME"` + Password *string `json:"-" envconfig:"TIDEPOOL_STORE_PASSWORD"` + Timeout time.Duration `json:"timeout" envconfig:"TIDEPOOL_STORE_TIMEOUT" default:"60s"` + OptParams *string `json:"optParams" envconfig:"TIDEPOOL_STORE_OPT_PARAMS"` + DisableIndexCreation bool `json:"disableIndexCreation" envconfig:"TIDEPOOL_DISABLE_INDEX_CREATION"` } // AsConnectionString constructs a MongoDB connection string from a Config diff --git a/store/structured/mongo/repository.go b/store/structured/mongo/repository.go index 274d9217db..16f2ba79f2 100644 --- a/store/structured/mongo/repository.go +++ b/store/structured/mongo/repository.go @@ -15,15 +15,25 @@ import ( type Repository struct { *mongo.Collection + config RepositoryConfig } -func NewRepository(collection *mongo.Collection) *Repository { +type RepositoryConfig struct { + DisableIndexCreation bool +} + +func NewRepository(collection *mongo.Collection, config RepositoryConfig) *Repository { return &Repository{ collection, + config, } } func (r *Repository) CreateAllIndexes(ctx context.Context, indexes []mongo.IndexModel) error { + if r.config.DisableIndexCreation { + return nil + } + if ctx == nil { ctx = context.Background() } diff --git a/store/structured/mongo/store.go b/store/structured/mongo/store.go index a1b2a2086a..e9a7f437dc 100644 --- a/store/structured/mongo/store.go +++ b/store/structured/mongo/store.go @@ -66,7 +66,10 @@ func AppendLifecycleHooksToStore(store *Store, lifecycle fx.Lifecycle) { } func (o *Store) GetRepository(collection string) *Repository { - return NewRepository(o.GetCollection(collection)) + config := RepositoryConfig{ + DisableIndexCreation: o.config.DisableIndexCreation, + } + return NewRepository(o.GetCollection(collection), config) } func (o *Store) GetCollection(collection string) *mongo.Collection { diff --git a/task/service/service/service_test.go b/task/service/service/service_test.go index 8fe529cca5..69f4f2efc2 100644 --- a/task/service/service/service_test.go +++ b/task/service/service/service_test.go @@ -46,7 +46,7 @@ var _ = Describe("Service", func() { server = NewServer() server.AppendHandlers( CombineHandlers( - VerifyRequest("POST", "/auth/serverlogin"), + VerifyRequest("POST", "/serverlogin"), VerifyHeaderKV("X-Tidepool-Server-Name", *provider.NameOutput), VerifyHeaderKV("X-Tidepool-Server-Secret", serverSecret), VerifyBody(nil),