diff --git a/pkg/application/app.go b/pkg/application/app.go index 52e06f877..2f93a03ae 100644 --- a/pkg/application/app.go +++ b/pkg/application/app.go @@ -45,6 +45,7 @@ func Default(options ...Option) kernel.Kernel { WithMetadataServer, WithConsumerMessagesPerRunnerMetrics, WithKernelSettingsFromConfig, + WithLoggerGroupTag, WithLoggerApplicationTag, WithLoggerContextFieldsMessageEncoder, WithLoggerContextFieldsResolver(log.ContextLoggerFieldsResolver), diff --git a/pkg/application/options.go b/pkg/application/options.go index d5762a697..1fe202cee 100644 --- a/pkg/application/options.go +++ b/pkg/application/options.go @@ -178,6 +178,18 @@ func WithKernelSettingsFromConfig(app *App) { }) } +func WithLoggerGroupTag(app *App) { + app.addLoggerOption(func(config cfg.GosoConf, logger log.GosoLogger) error { + if !config.IsSet("app_group") { + return errors.New("can not get application group from config to set it on logger") + } + + return logger.Option(log.WithFields(map[string]interface{}{ + "group": config.GetString("app_group"), + })) + }) +} + func WithLoggerApplicationTag(app *App) { app.addLoggerOption(func(config cfg.GosoConf, logger log.GosoLogger) error { if !config.IsSet("app_name") { diff --git a/pkg/application/testdata/config.dist.yml b/pkg/application/testdata/config.dist.yml index 881ffed61..4a2e771ef 100644 --- a/pkg/application/testdata/config.dist.yml +++ b/pkg/application/testdata/config.dist.yml @@ -1,5 +1,6 @@ env: test app_name: test_app +app_group: test_group app_project: test_project app_family: test_family diff --git a/pkg/cfg/application_identifiers.go b/pkg/cfg/application_identifiers.go index ee31a2540..d5f006ba1 100644 --- a/pkg/cfg/application_identifiers.go +++ b/pkg/cfg/application_identifiers.go @@ -1,11 +1,16 @@ package cfg -import "fmt" +import ( + "strings" + + "github.com/justtrackio/gosoline/pkg/funk" +) type AppId struct { Project string `cfg:"project" default:"{app_project}" json:"project"` Environment string `cfg:"environment" default:"{env}" json:"environment"` Family string `cfg:"family" default:"{app_family}" json:"family"` + Group string `cfg:"group" default:"{app_group}" json:"group"` Application string `cfg:"application" default:"{app_name}" json:"application"` } @@ -14,6 +19,7 @@ func GetAppIdFromConfig(config Config) AppId { Project: config.GetString("app_project"), Environment: config.GetString("env"), Family: config.GetString("app_family"), + Group: config.GetString("app_group"), Application: config.GetString("app_name"), } } @@ -31,11 +37,20 @@ func (i *AppId) PadFromConfig(config Config) { i.Family = config.GetString("app_family") } + if len(i.Group) == 0 { + i.Group = config.GetString("app_group") + } + if len(i.Application) == 0 { i.Application = config.GetString("app_name") } } func (i *AppId) String() string { - return fmt.Sprintf("%s-%s-%s-%s", i.Project, i.Environment, i.Family, i.Application) + elements := []string{i.Project, i.Environment, i.Family, i.Group, i.Application} + elements = funk.Filter(elements, func(element string) bool { + return len(element) > 0 + }) + + return strings.Join(elements, "-") } diff --git a/pkg/cfg/application_identifiers_test.go b/pkg/cfg/application_identifiers_test.go index 2cb176b49..911aef55c 100644 --- a/pkg/cfg/application_identifiers_test.go +++ b/pkg/cfg/application_identifiers_test.go @@ -12,6 +12,7 @@ func TestGetAppIdFromConfig(t *testing.T) { config := new(cfgMocks.Config) config.On("GetString", "app_project").Return("prj") config.On("GetString", "app_family").Return("fam") + config.On("GetString", "app_group").Return("grp") config.On("GetString", "app_name").Return("name") config.On("GetString", "env").Return("test") @@ -21,6 +22,7 @@ func TestGetAppIdFromConfig(t *testing.T) { Project: "prj", Environment: "test", Family: "fam", + Group: "grp", Application: "name", }, appId) @@ -31,6 +33,7 @@ func TestAppId_PadFromConfig(t *testing.T) { config := new(cfgMocks.Config) config.On("GetString", "app_project").Return("prj") config.On("GetString", "app_family").Return("fam") + config.On("GetString", "app_group").Return("grp") config.On("GetString", "app_name").Return("name") config.On("GetString", "env").Return("test") @@ -41,6 +44,7 @@ func TestAppId_PadFromConfig(t *testing.T) { Project: "prj", Environment: "test", Family: "fam", + Group: "grp", Application: "name", }, appId) diff --git a/pkg/cloud/aws/kinesis/naming.go b/pkg/cloud/aws/kinesis/naming.go index 2fb323026..3837f13ff 100644 --- a/pkg/cloud/aws/kinesis/naming.go +++ b/pkg/cloud/aws/kinesis/naming.go @@ -15,7 +15,7 @@ type StreamNameSettingsAware interface { } type StreamNamingSettings struct { - Pattern string `cfg:"pattern,nodecode" default:"{project}-{env}-{family}-{app}-{name}"` + Pattern string `cfg:"pattern,nodecode" default:"{project}-{env}-{family}-{group}-{streamName}"` } func GetStreamName(config cfg.Config, settings StreamNameSettingsAware) (Stream, error) { @@ -31,11 +31,12 @@ func GetStreamName(config cfg.Config, settings StreamNameSettingsAware) (Stream, name := namingSettings.Pattern values := map[string]string{ - "project": appId.Project, - "env": appId.Environment, - "family": appId.Family, - "app": appId.Application, - "name": settings.GetStreamName(), + "project": appId.Project, + "env": appId.Environment, + "family": appId.Family, + "group": appId.Group, + "app": appId.Application, + "streamName": settings.GetStreamName(), } for key, val := range values { diff --git a/pkg/cloud/aws/kinesis/naming_test.go b/pkg/cloud/aws/kinesis/naming_test.go index 1747f3a6f..8aa7eb242 100644 --- a/pkg/cloud/aws/kinesis/naming_test.go +++ b/pkg/cloud/aws/kinesis/naming_test.go @@ -23,8 +23,9 @@ func (s *GetStreamNameTestSuite) SetupTest() { s.settings = &kinesis.Settings{ AppId: cfg.AppId{ Project: "justtrack", - Environment: "test", + Environment: "env", Family: "gosoline", + Group: "grp", Application: "producer", }, ClientName: "default", @@ -40,12 +41,12 @@ func (s *GetStreamNameTestSuite) setupConfig(settings map[string]interface{}) { func (s *GetStreamNameTestSuite) TestDefault() { name, err := kinesis.GetStreamName(s.config, s.settings) s.NoError(err) - s.EqualValues("justtrack-test-gosoline-producer-event", name) + s.EqualValues("justtrack-env-gosoline-grp-event", string(name)) } func (s *GetStreamNameTestSuite) TestDefaultWithPattern() { s.setupConfig(map[string]interface{}{ - "cloud.aws.kinesis.clients.default.naming.pattern": "{app}-{name}", + "cloud.aws.kinesis.clients.default.naming.pattern": "{app}-{streamName}", }) name, err := kinesis.GetStreamName(s.config, s.settings) @@ -56,7 +57,7 @@ func (s *GetStreamNameTestSuite) TestDefaultWithPattern() { func (s *GetStreamNameTestSuite) TestSpecificClientWithPattern() { s.settings.ClientName = "specific" s.setupConfig(map[string]interface{}{ - "cloud.aws.kinesis.clients.specific.naming.pattern": "{app}-{name}", + "cloud.aws.kinesis.clients.specific.naming.pattern": "{app}-{streamName}", }) name, err := kinesis.GetStreamName(s.config, s.settings) diff --git a/pkg/cloud/aws/sns/naming.go b/pkg/cloud/aws/sns/naming.go index 9d2f25d31..734d691c5 100644 --- a/pkg/cloud/aws/sns/naming.go +++ b/pkg/cloud/aws/sns/naming.go @@ -33,7 +33,7 @@ func (s TopicNameSettings) GetTopicId() string { } type TopicNamingSettings struct { - Pattern string `cfg:"pattern,nodecode" default:"{project}-{env}-{family}-{app}-{topicId}"` + Pattern string `cfg:"pattern,nodecode" default:"{project}-{env}-{family}-{group}-{topicId}"` } func GetTopicName(config cfg.Config, topicSettings TopicNameSettingsAware) (string, error) { @@ -51,6 +51,7 @@ func GetTopicName(config cfg.Config, topicSettings TopicNameSettingsAware) (stri "project": appId.Project, "env": appId.Environment, "family": appId.Family, + "group": appId.Group, "app": appId.Application, "topicId": topicSettings.GetTopicId(), } diff --git a/pkg/cloud/aws/sns/naming_test.go b/pkg/cloud/aws/sns/naming_test.go index 97f86df85..b02ff3a5a 100644 --- a/pkg/cloud/aws/sns/naming_test.go +++ b/pkg/cloud/aws/sns/naming_test.go @@ -25,6 +25,7 @@ func (s *GetTopicNameTestSuite) SetupTest() { Project: "justtrack", Environment: "test", Family: "gosoline", + Group: "group", Application: "producer", }, ClientName: "default", @@ -40,7 +41,7 @@ func (s *GetTopicNameTestSuite) setupConfig(settings map[string]interface{}) { func (s *GetTopicNameTestSuite) TestDefault() { name, err := sns.GetTopicName(s.config, s.settings) s.NoError(err) - s.Equal("justtrack-test-gosoline-producer-event", name) + s.Equal("justtrack-test-gosoline-group-event", name) } func (s *GetTopicNameTestSuite) TestDefaultWithPattern() { diff --git a/pkg/cloud/aws/sqs/naming.go b/pkg/cloud/aws/sqs/naming.go index 049d27df9..32c64e531 100644 --- a/pkg/cloud/aws/sqs/naming.go +++ b/pkg/cloud/aws/sqs/naming.go @@ -39,7 +39,7 @@ func (s QueueNameSettings) GetQueueId() string { } type QueueNamingSettings struct { - Pattern string `cfg:"pattern,nodecode" default:"{project}-{env}-{family}-{app}-{queueId}"` + Pattern string `cfg:"pattern,nodecode" default:"{project}-{env}-{family}-{group}-{queueId}"` } func GetQueueName(config cfg.Config, queueSettings QueueNameSettingsAware) (string, error) { @@ -57,6 +57,7 @@ func GetQueueName(config cfg.Config, queueSettings QueueNameSettingsAware) (stri "project": appId.Project, "env": appId.Environment, "family": appId.Family, + "group": appId.Group, "app": appId.Application, "queueId": queueSettings.GetQueueId(), } diff --git a/pkg/cloud/aws/sqs/naming_test.go b/pkg/cloud/aws/sqs/naming_test.go index 9a154330c..99b258b37 100644 --- a/pkg/cloud/aws/sqs/naming_test.go +++ b/pkg/cloud/aws/sqs/naming_test.go @@ -25,6 +25,7 @@ func (s *GetSqsQueueNameTestSuite) SetupTest() { Project: "justtrack", Environment: "test", Family: "gosoline", + Group: "group", Application: "producer", }, ClientName: "default", @@ -40,7 +41,7 @@ func (s *GetSqsQueueNameTestSuite) setupConfig(settings map[string]interface{}) func (s *GetSqsQueueNameTestSuite) TestDefault() { name, err := sqs.GetQueueName(s.config, s.settings) s.NoError(err) - s.Equal("justtrack-test-gosoline-producer-event", name) + s.Equal("justtrack-test-gosoline-group-event", name) } func (s *GetSqsQueueNameTestSuite) TestDefaultFifo() { @@ -48,7 +49,7 @@ func (s *GetSqsQueueNameTestSuite) TestDefaultFifo() { name, err := sqs.GetQueueName(s.config, s.settings) s.NoError(err) - s.Equal("justtrack-test-gosoline-producer-event.fifo", name) + s.Equal("justtrack-test-gosoline-group-event.fifo", name) } func (s *GetSqsQueueNameTestSuite) TestDefaultWithPattern() { diff --git a/pkg/conc/ddb/leader_election_ddb.go b/pkg/conc/ddb/leader_election_ddb.go index 14cf6fb10..a9cf2d2c2 100644 --- a/pkg/conc/ddb/leader_election_ddb.go +++ b/pkg/conc/ddb/leader_election_ddb.go @@ -19,10 +19,14 @@ type DdbLeaderElectionItem struct { LeadingUntil int64 `json:"leadingUntil" ddb:"ttl=enabled"` } +type TableNamingSettings struct { + Pattern string `cfg:"pattern,nodecode" default:"{project}-{env}-{family}-leader-elections"` +} + type DdbLeaderElectionSettings struct { - TableName string `cfg:"table_name" default:"{app_project}-{env}-{app_family}-leader-elections"` - GroupId string `cfg:"group_id" default:"{app_name}"` - LeaseDuration time.Duration `cfg:"lease_duration" default:"1m"` + Naming TableNamingSettings `cfg:"naming"` + GroupId string `cfg:"group_id" default:"{app_name}"` + LeaseDuration time.Duration `cfg:"lease_duration" default:"1m"` } type DdbLeaderElection struct { @@ -44,7 +48,7 @@ func NewDdbLeaderElectionWithSettings(ctx context.Context, config cfg.Config, lo repository, err := ddb.NewRepository(ctx, config, logger, &ddb.Settings{ ModelId: mdl.ModelId{}, TableNamingSettings: ddb.TableNamingSettings{ - Pattern: settings.TableName, + Pattern: settings.Naming.Pattern, }, DisableTracing: true, Main: ddb.MainSettings{ diff --git a/pkg/conc/ddb/leader_election_ddb_test.go b/pkg/conc/ddb/leader_election_ddb_test.go index 060e4d8fb..e8c3804ef 100644 --- a/pkg/conc/ddb/leader_election_ddb_test.go +++ b/pkg/conc/ddb/leader_election_ddb_test.go @@ -30,7 +30,9 @@ func (s *DdbLeaderElectionTestCase) SetupTest() { var err error s.election, err = concDdb.NewDdbLeaderElectionWithInterfaces(s.logger, s.clock, s.repository, &concDdb.DdbLeaderElectionSettings{ - TableName: "gosoline-leader-election", + Naming: concDdb.TableNamingSettings{ + Pattern: "gosoline-leader-election", + }, GroupId: "test", LeaseDuration: time.Minute, }) diff --git a/pkg/db-repo/notification_publisher_test.go b/pkg/db-repo/notification_publisher_test.go index 85718e3c8..2721d763d 100644 --- a/pkg/db-repo/notification_publisher_test.go +++ b/pkg/db-repo/notification_publisher_test.go @@ -3,6 +3,7 @@ package db_repo_test import ( "context" "testing" + "time" "github.com/justtrackio/gosoline/pkg/cfg" db_repo "github.com/justtrackio/gosoline/pkg/db-repo" @@ -12,6 +13,23 @@ import ( "github.com/stretchr/testify/assert" ) +type modelBased struct { + value string + createdAt, updatedAt *time.Time +} + +func (m *modelBased) GetId() *uint { + return mdl.Box(uint(3)) +} + +func (m *modelBased) SetUpdatedAt(updatedAt *time.Time) { + m.updatedAt = updatedAt +} + +func (m *modelBased) SetCreatedAt(createdAt *time.Time) { + m.createdAt = createdAt +} + func Test_Publish_Notifier(t *testing.T) { input := &modelBased{ value: "my test input", @@ -33,6 +51,7 @@ func Test_Publish_Notifier(t *testing.T) { Name: "myTest", Application: "testApp", Family: "testFamily", + Group: "grp", Environment: "test", } diff --git a/pkg/db-repo/notification_sns.go b/pkg/db-repo/notification_sns.go deleted file mode 100644 index e08a60f2e..000000000 --- a/pkg/db-repo/notification_sns.go +++ /dev/null @@ -1,88 +0,0 @@ -package db_repo - -import ( - "context" - "fmt" - - "github.com/justtrackio/gosoline/pkg/cfg" - "github.com/justtrackio/gosoline/pkg/exec" - "github.com/justtrackio/gosoline/pkg/log" - "github.com/justtrackio/gosoline/pkg/mdl" - "github.com/justtrackio/gosoline/pkg/stream" -) - -type streamNotifier struct { - notifier - encoder stream.MessageEncoder - output stream.Output - transformer mdl.TransformerResolver -} - -func NewStreamNotifier(logger log.Logger, output stream.Output, modelId mdl.ModelId, version int, transformer mdl.TransformerResolver) *streamNotifier { - notifier := newNotifier(logger, modelId, version) - - encoder := stream.NewMessageEncoder(&stream.MessageEncoderSettings{ - Encoding: stream.EncodingJson, - }) - - return &streamNotifier{ - notifier: notifier, - encoder: encoder, - output: output, - transformer: transformer, - } -} - -func (n *streamNotifier) Send(ctx context.Context, notificationType string, value ModelBased) error { - logger := n.logger.WithContext(ctx) - modelId := n.modelId.String() - - out := n.transformer("api", n.version, value) - - msg, err := n.encoder.Encode(ctx, out, map[string]interface{}{ - "type": notificationType, - "version": n.version, - "modelId": modelId, - }) - if err != nil { - return fmt.Errorf("can not encode notification message: %w", err) - } - - err = n.output.WriteOne(ctx, msg) - - if exec.IsRequestCanceled(err) { - logger.Info("request canceled while executing notification on %s for model %s with id %d", notificationType, modelId, *value.GetId()) - n.writeMetric(err) - return err - } - - if err != nil { - n.writeMetric(err) - return fmt.Errorf("error executing notification on %s for model %s with id %d: %w", notificationType, modelId, *value.GetId(), err) - } - - logger.Info("sent on %s successful for model %s with id %d", notificationType, modelId, *value.GetId()) - n.writeMetric(nil) - - return nil -} - -func NewSnsNotifier(ctx context.Context, config cfg.Config, logger log.Logger, modelId mdl.ModelId, version int, transformer mdl.TransformerResolver) (*streamNotifier, error) { - modelId.PadFromConfig(config) - - output, err := stream.NewSnsOutput(ctx, config, logger, &stream.SnsOutputSettings{ - TopicId: modelId.Name, - AppId: cfg.AppId{ - Project: modelId.Project, - Environment: modelId.Environment, - Family: modelId.Family, - Application: modelId.Application, - }, - ClientName: "default", - }) - if err != nil { - return nil, fmt.Errorf("can not create sns output: %w", err) - } - - return NewStreamNotifier(logger, output, modelId, version, transformer), nil -} diff --git a/pkg/db-repo/notification_sns_test.go b/pkg/db-repo/notification_sns_test.go deleted file mode 100644 index db749988f..000000000 --- a/pkg/db-repo/notification_sns_test.go +++ /dev/null @@ -1,69 +0,0 @@ -package db_repo_test - -import ( - "context" - "testing" - "time" - - db_repo "github.com/justtrackio/gosoline/pkg/db-repo" - logMocks "github.com/justtrackio/gosoline/pkg/log/mocks" - "github.com/justtrackio/gosoline/pkg/mdl" - "github.com/justtrackio/gosoline/pkg/stream" - streamMocks "github.com/justtrackio/gosoline/pkg/stream/mocks" - "github.com/stretchr/testify/assert" -) - -func Test_SNS_Notifier(t *testing.T) { - input := &modelBased{ - value: "my test input", - } - modelId := mdl.ModelId{ - Project: "testProject", - Name: "myTest", - Application: "testApp", - Family: "testFamily", - Environment: "test", - } - streamMessage := stream.Message{ - Attributes: map[string]interface{}{ - "encoding": stream.EncodingJson, - "modelId": modelId.String(), - "type": "CREATE", - "version": 55, - }, - Body: `{}`, - } - output := streamMocks.Output{} - output.On("WriteOne", context.Background(), &streamMessage).Return(nil).Once() - transformer := func(view string, version int, in interface{}) (out interface{}) { - assert.Equal(t, mdl.Box(uint(3)), in.(db_repo.ModelBased).GetId()) - assert.Equal(t, "api", view) - assert.Equal(t, 55, version) - - return in - } - - notifier := db_repo.NewStreamNotifier(logMocks.NewLoggerMockedAll(), &output, modelId, 55, transformer) - - err := notifier.Send(context.Background(), "CREATE", input) - assert.NoError(t, err) - - output.AssertExpectations(t) -} - -type modelBased struct { - value string - createdAt, updatedAt *time.Time -} - -func (m *modelBased) GetId() *uint { - return mdl.Box(uint(3)) -} - -func (m *modelBased) SetUpdatedAt(updatedAt *time.Time) { - m.updatedAt = updatedAt -} - -func (m *modelBased) SetCreatedAt(createdAt *time.Time) { - m.createdAt = createdAt -} diff --git a/pkg/ddb/naming.go b/pkg/ddb/naming.go index fe701e297..e6d89cc81 100644 --- a/pkg/ddb/naming.go +++ b/pkg/ddb/naming.go @@ -8,12 +8,8 @@ import ( "github.com/justtrackio/gosoline/pkg/cloud/aws" ) -const ( - DefaultNamingPattern = "{project}-{env}-{family}-{app}-{modelId}" -) - type TableNamingSettings struct { - Pattern string `cfg:"pattern,nodecode" default:"{project}-{env}-{family}-{app}-{modelId}"` + Pattern string `cfg:"pattern,nodecode" default:"{project}-{env}-{family}-{group}-{modelId}"` } func TableName(config cfg.Config, settings *Settings) string { @@ -45,6 +41,7 @@ func GetTableNameWithSettings(tableSettings *Settings, namingSettings *TableNami "project": tableSettings.ModelId.Project, "env": tableSettings.ModelId.Environment, "family": tableSettings.ModelId.Family, + "group": tableSettings.ModelId.Group, "app": tableSettings.ModelId.Application, "modelId": tableSettings.ModelId.Name, } diff --git a/pkg/ddb/naming_test.go b/pkg/ddb/naming_test.go index 97bbcc97d..977599154 100644 --- a/pkg/ddb/naming_test.go +++ b/pkg/ddb/naming_test.go @@ -26,6 +26,7 @@ func (s *TableNameTestSuite) SetupTest() { Project: "justtrack", Environment: "test", Family: "gosoline", + Group: "group", Application: "producer", Name: "event", }, @@ -40,7 +41,7 @@ func (s *TableNameTestSuite) setupConfig(settings map[string]interface{}) { func (s *TableNameTestSuite) TestDefault() { name := ddb.TableName(s.config, s.settings) - s.Equal("justtrack-test-gosoline-producer-event", name) + s.Equal("justtrack-test-gosoline-group-event", name) } func (s *TableNameTestSuite) TestDefaultWithPattern() { diff --git a/pkg/fixtures/writer_ddb_kvstore.go b/pkg/fixtures/writer_ddb_kvstore.go index d94a8df3b..9bb9700e3 100644 --- a/pkg/fixtures/writer_ddb_kvstore.go +++ b/pkg/fixtures/writer_ddb_kvstore.go @@ -31,6 +31,7 @@ func DynamoDbKvStoreFixtureWriterFactory[T any](modelId *mdl.ModelId) FixtureWri Project: modelId.Project, Environment: modelId.Environment, Family: modelId.Family, + Group: modelId.Group, Application: modelId.Application, }, Name: modelId.Name, diff --git a/pkg/fixtures/writer_redis_kvstore.go b/pkg/fixtures/writer_redis_kvstore.go index c1739d5af..d4613d01d 100644 --- a/pkg/fixtures/writer_redis_kvstore.go +++ b/pkg/fixtures/writer_redis_kvstore.go @@ -23,6 +23,7 @@ func RedisKvStoreFixtureWriterFactory[T any](modelId *mdl.ModelId) FixtureWriter Project: modelId.Project, Environment: modelId.Environment, Family: modelId.Family, + Group: modelId.Group, Application: modelId.Application, }, Name: modelId.Name, diff --git a/pkg/kvstore/chain_test.go b/pkg/kvstore/chain_test.go index b759a70fc..d7ca345cc 100644 --- a/pkg/kvstore/chain_test.go +++ b/pkg/kvstore/chain_test.go @@ -467,6 +467,7 @@ func buildTestableChainStore[T any](missingCacheEnabled bool) (kvstore.KvStore[T Project: "applike", Environment: "test", Family: "gosoline", + Group: "grp", Application: "kvstore", }, Name: "test", diff --git a/pkg/kvstore/configurable.go b/pkg/kvstore/configurable.go index 2d56be4cc..c80ec615e 100644 --- a/pkg/kvstore/configurable.go +++ b/pkg/kvstore/configurable.go @@ -21,6 +21,7 @@ const ( type ChainConfiguration struct { Project string `cfg:"project"` Family string `cfg:"family"` + Group string `cfg:"group"` Application string `cfg:"application"` Type string `cfg:"type" default:"chain" validate:"eq=chain"` Elements []string `cfg:"elements" validate:"min=1"` @@ -76,6 +77,7 @@ func newKvStoreChainFromConfig[T any](ctx context.Context, config cfg.Config, lo AppId: cfg.AppId{ Project: configuration.Project, Family: configuration.Family, + Group: configuration.Group, Application: configuration.Application, }, Name: name, diff --git a/pkg/kvstore/ddb.go b/pkg/kvstore/ddb.go index cc8e59da2..c443eac1c 100644 --- a/pkg/kvstore/ddb.go +++ b/pkg/kvstore/ddb.go @@ -44,6 +44,7 @@ func NewDdbKvStore[T any](ctx context.Context, config cfg.Config, logger log.Log Project: settings.Project, Environment: settings.Environment, Family: settings.Family, + Group: settings.Group, Application: settings.Application, Name: name, }, diff --git a/pkg/kvstore/ddb_test.go b/pkg/kvstore/ddb_test.go index c3a4d3aad..903b6f16e 100644 --- a/pkg/kvstore/ddb_test.go +++ b/pkg/kvstore/ddb_test.go @@ -280,6 +280,7 @@ func buildTestableDdbStore[T any]() (kvstore.KvStore[T], *ddbMocks.Repository) { Project: "applike", Environment: "test", Family: "gosoline", + Group: "grp", Application: "kvstore", }, Name: "test", diff --git a/pkg/kvstore/metric.go b/pkg/kvstore/metric.go index f9666fd2a..4ceaa2b61 100644 --- a/pkg/kvstore/metric.go +++ b/pkg/kvstore/metric.go @@ -40,6 +40,7 @@ func NewMetricStoreWithInterfaces[T any](store KvStore[T], settings *Settings) K Project: settings.Project, Environment: settings.Environment, Family: settings.Family, + Group: settings.Group, Application: settings.Application, Name: settings.Name, }).String() diff --git a/pkg/kvstore/redis.go b/pkg/kvstore/redis.go index ffa940594..ce378fc28 100644 --- a/pkg/kvstore/redis.go +++ b/pkg/kvstore/redis.go @@ -274,7 +274,7 @@ func (s *redisKvStore[T]) key(key any) (string, error) { keyStr = strings.Join([]string{ s.settings.Project, s.settings.Family, - s.settings.Application, + s.settings.Group, "kvstore", s.settings.Name, keyStr, diff --git a/pkg/kvstore/redis_test.go b/pkg/kvstore/redis_test.go index d9dba58f7..9951ed0f9 100644 --- a/pkg/kvstore/redis_test.go +++ b/pkg/kvstore/redis_test.go @@ -22,8 +22,8 @@ type Item struct { func TestRedisKvStore_Contains(t *testing.T) { store, client := buildTestableRedisStore[Item]() - client.On("Exists", mock.AnythingOfType("*context.emptyCtx"), "applike-gosoline-kvstore-kvstore-test-foo").Return(int64(0), nil) - client.On("Exists", mock.AnythingOfType("*context.emptyCtx"), "applike-gosoline-kvstore-kvstore-test-bar").Return(int64(1), nil) + client.On("Exists", mock.AnythingOfType("*context.emptyCtx"), "justtrack-gosoline-grp-kvstore-test-foo").Return(int64(0), nil) + client.On("Exists", mock.AnythingOfType("*context.emptyCtx"), "justtrack-gosoline-grp-kvstore-test-bar").Return(int64(1), nil) exists, err := store.Contains(context.Background(), "foo") assert.NoError(t, err) @@ -38,7 +38,7 @@ func TestRedisKvStore_Contains(t *testing.T) { func TestRedisKvStore_Get(t *testing.T) { store, client := buildTestableRedisStore[Item]() - client.On("Get", mock.AnythingOfType("*context.emptyCtx"), "applike-gosoline-kvstore-kvstore-test-foo").Return(`{"id":"foo","body":"bar"}`, nil) + client.On("Get", mock.AnythingOfType("*context.emptyCtx"), "justtrack-gosoline-grp-kvstore-test-foo").Return(`{"id":"foo","body":"bar"}`, nil) item := &Item{} found, err := store.Get(context.Background(), "foo", item) @@ -54,7 +54,7 @@ func TestRedisKvStore_Get(t *testing.T) { func TestRedisKvStore_GetBatch(t *testing.T) { store, client := buildTestableRedisStore[Item]() - args := []interface{}{mock.AnythingOfType("*context.emptyCtx"), "applike-gosoline-kvstore-kvstore-test-foo", "applike-gosoline-kvstore-kvstore-test-fuu"} + args := []interface{}{mock.AnythingOfType("*context.emptyCtx"), "justtrack-gosoline-grp-kvstore-test-foo", "justtrack-gosoline-grp-kvstore-test-fuu"} returns := []interface{}{`{"id":"foo","body":"bar"}`, nil} client.On("MGet", args...).Return(returns, nil) @@ -77,7 +77,7 @@ func TestRedisKvStore_GetBatch(t *testing.T) { func TestRedisKvStore_Put(t *testing.T) { store, client := buildTestableRedisStore[Item]() - client.On("Set", mock.AnythingOfType("*context.emptyCtx"), "applike-gosoline-kvstore-kvstore-test-foo", []byte(`{"id":"foo","body":"bar"}`), time.Duration(0)).Return(nil) + client.On("Set", mock.AnythingOfType("*context.emptyCtx"), "justtrack-gosoline-grp-kvstore-test-foo", []byte(`{"id":"foo","body":"bar"}`), time.Duration(0)).Return(nil) item := Item{ Id: "foo", @@ -95,16 +95,16 @@ func TestRedisKvStore_PutBatch(t *testing.T) { pipe := &redisMocks.Pipeliner{} pipe.On("MSet", mock.AnythingOfType("*context.emptyCtx"), mock.MatchedBy(func(input []interface{}) bool { - possibleInput1 := `[applike-gosoline-kvstore-kvstore-test-foo {"id":"foo","body":"bar"} applike-gosoline-kvstore-kvstore-test-fuu {"id":"fuu","body":"baz"}]` - possibleInput2 := `[applike-gosoline-kvstore-kvstore-test-fuu {"id":"fuu","body":"baz"} applike-gosoline-kvstore-kvstore-test-foo {"id":"foo","body":"bar"}]` + possibleInput1 := `[justtrack-gosoline-grp-kvstore-test-foo {"id":"foo","body":"bar"} justtrack-gosoline-grp-kvstore-test-fuu {"id":"fuu","body":"baz"}]` + possibleInput2 := `[justtrack-gosoline-grp-kvstore-test-fuu {"id":"fuu","body":"baz"} justtrack-gosoline-grp-kvstore-test-foo {"id":"foo","body":"bar"}]` inputStr := fmt.Sprintf("%s", input) return inputStr == possibleInput1 || inputStr == possibleInput2 })).Return(nil) client.On("Pipeline").Return(pipe) pipe.On("TxPipeline").Return(pipe) - pipe.On("Expire", mock.AnythingOfType("*context.emptyCtx"), "applike-gosoline-kvstore-kvstore-test-foo", mock.AnythingOfType("time.Duration")).Return(nil) - pipe.On("Expire", mock.AnythingOfType("*context.emptyCtx"), "applike-gosoline-kvstore-kvstore-test-fuu", mock.AnythingOfType("time.Duration")).Return(nil) + pipe.On("Expire", mock.AnythingOfType("*context.emptyCtx"), "justtrack-gosoline-grp-kvstore-test-foo", mock.AnythingOfType("time.Duration")).Return(nil) + pipe.On("Expire", mock.AnythingOfType("*context.emptyCtx"), "justtrack-gosoline-grp-kvstore-test-fuu", mock.AnythingOfType("time.Duration")).Return(nil) pipe.On("Exec", mock.AnythingOfType("*context.emptyCtx")).Return(nil, nil) items := map[string]Item{ @@ -129,8 +129,8 @@ func TestRedisKvStore_PutBatchSkipExpire(t *testing.T) { pipe := &redisMocks.Pipeliner{} pipe.On("MSet", mock.AnythingOfType("*context.emptyCtx"), mock.MatchedBy(func(input []interface{}) bool { - possibleInput1 := `[applike-gosoline-kvstore-kvstore-test-foo {"id":"foo","body":"bar"} applike-gosoline-kvstore-kvstore-test-fuu {"id":"fuu","body":"baz"}]` - possibleInput2 := `[applike-gosoline-kvstore-kvstore-test-fuu {"id":"fuu","body":"baz"} applike-gosoline-kvstore-kvstore-test-foo {"id":"foo","body":"bar"}]` + possibleInput1 := `[justtrack-gosoline-grp-kvstore-test-foo {"id":"foo","body":"bar"} justtrack-gosoline-grp-kvstore-test-fuu {"id":"fuu","body":"baz"}]` + possibleInput2 := `[justtrack-gosoline-grp-kvstore-test-fuu {"id":"fuu","body":"baz"} justtrack-gosoline-grp-kvstore-test-foo {"id":"foo","body":"bar"}]` inputStr := fmt.Sprintf("%s", input) return inputStr == possibleInput1 || inputStr == possibleInput2 @@ -168,7 +168,7 @@ func TestRedisKvStore_EstimateSize(t *testing.T) { func TestRedisKvStore_Delete(t *testing.T) { store, client := buildTestableRedisStore[Item]() - client.On("Del", mock.AnythingOfType("*context.emptyCtx"), "applike-gosoline-kvstore-kvstore-test-foo").Return(int64(1), nil) + client.On("Del", mock.AnythingOfType("*context.emptyCtx"), "justtrack-gosoline-grp-kvstore-test-foo").Return(int64(1), nil) err := store.Delete(context.Background(), "foo") @@ -178,7 +178,7 @@ func TestRedisKvStore_Delete(t *testing.T) { func TestRedisKvStore_DeleteBatch(t *testing.T) { store, client := buildTestableRedisStore[Item]() - client.On("Del", mock.AnythingOfType("*context.emptyCtx"), "applike-gosoline-kvstore-kvstore-test-foo", "applike-gosoline-kvstore-kvstore-test-fuu").Return(int64(2), nil) + client.On("Del", mock.AnythingOfType("*context.emptyCtx"), "justtrack-gosoline-grp-kvstore-test-foo", "justtrack-gosoline-grp-kvstore-test-fuu").Return(int64(2), nil) items := []string{"foo", "fuu"} @@ -193,10 +193,11 @@ func buildTestableRedisStore[T any]() (kvstore.KvStore[T], *redisMocks.Client) { store := kvstore.NewRedisKvStoreWithInterfaces[T](client, &kvstore.Settings{ AppId: cfg.AppId{ - Project: "applike", - Environment: "test", + Project: "justtrack", + Environment: "env", Family: "gosoline", - Application: "kvstore", + Group: "grp", + Application: "app", }, Name: "test", BatchSize: 100, @@ -211,9 +212,10 @@ func buildTestableRedisStoreWithTTL[T any]() (kvstore.KvStore[T], *redisMocks.Cl store := kvstore.NewRedisKvStoreWithInterfaces[T](client, &kvstore.Settings{ AppId: cfg.AppId{ - Project: "applike", + Project: "justtrack", Environment: "test", Family: "gosoline", + Group: "grp", Application: "kvstore", }, Name: "test", diff --git a/pkg/log/status/module_test.go b/pkg/log/status/module_test.go index 9320e4411..e03f687d3 100644 --- a/pkg/log/status/module_test.go +++ b/pkg/log/status/module_test.go @@ -121,6 +121,13 @@ func TestModuleExample(t *testing.T) { exitCodeHandler, application.WithModuleFactory("status", status.NewModule(status.ProvideManager())), application.WithModuleFactory("main", NewTestModule), + application.WithConfigMap(map[string]interface{}{ + "env": "test", + "app_project": "justtrack", + "app_family": "fam", + "app_group": "grp", + "app_name": "name", + }), ) app.Run() } diff --git a/pkg/mdl/model.go b/pkg/mdl/model.go index 837a44f81..042c88f1a 100644 --- a/pkg/mdl/model.go +++ b/pkg/mdl/model.go @@ -13,12 +13,13 @@ type ModelId struct { Project string `cfg:"project" default:"{app_project}"` Environment string `cfg:"environment" default:"{env}"` Family string `cfg:"family" default:"{app_family}"` + Group string `cfg:"group" default:"{app_group}"` Application string `cfg:"application" default:"{app_name}"` Name string `cfg:"name"` } func (m *ModelId) String() string { - return fmt.Sprintf("%v.%v.%v.%v", m.Project, m.Family, m.Application, m.Name) + return fmt.Sprintf("%s.%s.%s.%s", m.Project, m.Family, m.Group, m.Name) } func (m *ModelId) PadFromConfig(config ConfigProvider) { @@ -34,6 +35,10 @@ func (m *ModelId) PadFromConfig(config ConfigProvider) { m.Family = config.GetString("app_family") } + if len(m.Group) == 0 { + m.Group = config.GetString("app_group") + } + if len(m.Application) == 0 { m.Application = config.GetString("app_name") } @@ -47,10 +52,10 @@ func ModelIdFromString(str string) (ModelId, error) { } modelId := ModelId{ - Project: parts[0], - Family: parts[1], - Application: parts[2], - Name: parts[3], + Project: parts[0], + Family: parts[1], + Group: parts[2], + Name: parts[3], } return modelId, nil diff --git a/pkg/mdlsub/config_postprocessor_publisher.go b/pkg/mdlsub/config_postprocessor_publisher.go index 137426a14..d9685deda 100644 --- a/pkg/mdlsub/config_postprocessor_publisher.go +++ b/pkg/mdlsub/config_postprocessor_publisher.go @@ -74,7 +74,7 @@ func PublisherConfigPostProcessor(config cfg.GosoConf) (bool, error) { return true, nil } -func handlePublisherOutputTypeInMemory(config cfg.Config, publisherSettings *PublisherSettings, producerSettings *stream.ProducerSettings) stream.BaseOutputConfigurationAware { +func handlePublisherOutputTypeInMemory(config cfg.Config, _ *PublisherSettings, _ *stream.ProducerSettings) stream.BaseOutputConfigurationAware { outputSettings := &stream.InMemoryOutputConfiguration{} config.UnmarshalDefaults(outputSettings) @@ -96,6 +96,7 @@ func handlePublisherOutputTypeKinesis(config cfg.Config, publisherSettings *Publ outputSettings.Project = publisherSettings.Project outputSettings.Family = publisherSettings.Family + outputSettings.Group = publisherSettings.Group outputSettings.Application = publisherSettings.Application outputSettings.StreamName = publisherSettings.Name outputSettings.Tracing.Enabled = false @@ -103,12 +104,13 @@ func handlePublisherOutputTypeKinesis(config cfg.Config, publisherSettings *Publ return outputSettings } -func handlePublisherOutputTypeSns(config cfg.Config, publisherSettings *PublisherSettings, producerSettings *stream.ProducerSettings) stream.BaseOutputConfigurationAware { +func handlePublisherOutputTypeSns(config cfg.Config, publisherSettings *PublisherSettings, _ *stream.ProducerSettings) stream.BaseOutputConfigurationAware { outputSettings := &stream.SnsOutputConfiguration{} config.UnmarshalDefaults(outputSettings) outputSettings.Project = publisherSettings.Project outputSettings.Family = publisherSettings.Family + outputSettings.Group = publisherSettings.Group outputSettings.Application = publisherSettings.Application outputSettings.TopicId = publisherSettings.Name @@ -119,12 +121,13 @@ func handlePublisherOutputTypeSns(config cfg.Config, publisherSettings *Publishe return outputSettings } -func handlePublisherOutputTypeSqs(config cfg.Config, publisherSettings *PublisherSettings, producerSettings *stream.ProducerSettings) stream.BaseOutputConfigurationAware { +func handlePublisherOutputTypeSqs(config cfg.Config, publisherSettings *PublisherSettings, _ *stream.ProducerSettings) stream.BaseOutputConfigurationAware { outputSettings := &stream.SqsOutputConfiguration{} config.UnmarshalDefaults(outputSettings) outputSettings.Project = publisherSettings.Project outputSettings.Family = publisherSettings.Family + outputSettings.Group = publisherSettings.Group outputSettings.Application = publisherSettings.Application outputSettings.QueueId = publisherSettings.Name diff --git a/pkg/mdlsub/config_postprocessor_subscriber.go b/pkg/mdlsub/config_postprocessor_subscriber.go index c6bc5a5fb..bf6aefd3d 100644 --- a/pkg/mdlsub/config_postprocessor_subscriber.go +++ b/pkg/mdlsub/config_postprocessor_subscriber.go @@ -99,6 +99,7 @@ func snsSubscriberInputConfigPostProcessor(config cfg.GosoConf, name string, sub inputSettings.Targets = []stream.SnsInputTargetConfiguration{ { Family: subscriberSettings.SourceModel.Family, + Group: subscriberSettings.SourceModel.Group, Application: subscriberSettings.SourceModel.Application, TopicId: topicId, }, @@ -120,6 +121,7 @@ func kinesisSubscriberInputConfigPostProcessor(config cfg.GosoConf, name string, inputSettings.Project = subscriberSettings.SourceModel.Project inputSettings.Family = subscriberSettings.SourceModel.Family + inputSettings.Group = subscriberSettings.SourceModel.Group inputSettings.Application = subscriberSettings.SourceModel.Application inputSettings.StreamName = streamName @@ -134,6 +136,7 @@ func kvstoreSubscriberOutputConfigPostProcessor(config cfg.GosoConf, name string kvstoreSettings.Project = subscriberSettings.TargetModel.Project kvstoreSettings.Family = subscriberSettings.TargetModel.Family + kvstoreSettings.Group = subscriberSettings.TargetModel.Group kvstoreSettings.Application = subscriberSettings.TargetModel.Application kvstoreSettings.Elements = []string{kvstore.TypeRedis, kvstore.TypeDdb} @@ -145,7 +148,7 @@ func GetSubscriberFQN(name string, sourceModel SubscriberModel) string { return fmt.Sprintf("subscriber-%s", name) } - return fmt.Sprintf("subscriber-%s-%s-%s-%s", sourceModel.Project, sourceModel.Family, sourceModel.Application, sharedName) + return fmt.Sprintf("subscriber-%s-%s-%s-%s-%s", sourceModel.Project, sourceModel.Family, sourceModel.Group, sourceModel.Application, sharedName) } func getInputConfigKey(name string, sourceModel SubscriberModel) string { diff --git a/pkg/mdlsub/output.go b/pkg/mdlsub/output.go index 389cbd41d..0d67935c6 100644 --- a/pkg/mdlsub/output.go +++ b/pkg/mdlsub/output.go @@ -31,17 +31,17 @@ func initOutputs(ctx context.Context, config cfg.Config, logger log.Logger, subs modelId = settings.SourceModel.String() if outputFactory, ok = outputFactories[settings.Output]; !ok { - return nil, fmt.Errorf("there is no output of type %s for subscriber %s with modelId %s", settings.Output, name, modelId) + return nil, fmt.Errorf("can not create outputs: there is no output of type %s for subscriber %s with modelId %s", settings.Output, name, modelId) } if versionedModelTransformers, ok = transformers[modelId]; !ok { - return nil, fmt.Errorf("there is no transformer for subscriber %s with modelId %s", name, modelId) + return nil, fmt.Errorf("can not create transformer: there is no transformer for subscriber %s with modelId %s", name, modelId) } modelId := settings.SourceModel.String() if outputs[modelId], err = outputFactory(ctx, config, logger, settings, versionedModelTransformers); err != nil { - return nil, fmt.Errorf("can not create output for subscriber %s with modelId %s", name, modelId) + return nil, fmt.Errorf("can not create output for subscriber %s with modelId %s: %w", name, modelId, err) } } diff --git a/pkg/mdlsub/publisher_test.go b/pkg/mdlsub/publisher_test.go index 068e0e0c2..d2ed06533 100644 --- a/pkg/mdlsub/publisher_test.go +++ b/pkg/mdlsub/publisher_test.go @@ -25,6 +25,7 @@ func (s *PublisherTestSuite) SetupTest() { ModelId: mdl.ModelId{ Project: "gosoline", Family: "test", + Group: "grp", Application: "app", Name: "event", }, @@ -46,7 +47,7 @@ func (s *PublisherTestSuite) TestPublish() { expectedAttributes := map[string]interface{}{ "type": mdlsub.TypeCreate, "version": 0, - "modelId": "gosoline.test.app.event", + "modelId": "gosoline.test.grp.event", } s.producer.On("WriteOne", ctx, event, expectedAttributes).Return(nil) diff --git a/pkg/mdlsub/transformer.go b/pkg/mdlsub/transformer.go index c1363816b..953cf432e 100644 --- a/pkg/mdlsub/transformer.go +++ b/pkg/mdlsub/transformer.go @@ -43,7 +43,7 @@ func initTransformers(ctx context.Context, config cfg.Config, logger log.Logger, modelId := settings.SourceModel.String() if _, ok := transformerFactories[modelId]; !ok { - return nil, fmt.Errorf("there is no transformer for subscriber %s with modelId %s", name, modelId) + return nil, fmt.Errorf("can not create transformers: there is no transformer for subscriber %s with modelId %s", name, modelId) } } diff --git a/pkg/metric/daemon.go b/pkg/metric/daemon.go index 9355743f8..946c5418d 100644 --- a/pkg/metric/daemon.go +++ b/pkg/metric/daemon.go @@ -14,11 +14,20 @@ import ( const defaultTimeFormat = "2006-01-02T15:04Z07:00" +type NamingSettings struct { + Pattern string `cfg:"pattern,nodecode" default:"{project}/{env}/{family}/{group}-{app}"` +} + +type Cloudwatch struct { + Naming NamingSettings `cfg:"naming"` +} + type Settings struct { cfg.AppId - Enabled bool `cfg:"enabled" default:"false"` - Interval time.Duration `cfg:"interval" default:"60s"` - Writer string `cfg:"writer"` + Enabled bool `cfg:"enabled" default:"false"` + Interval time.Duration `cfg:"interval" default:"60s"` + Cloudwatch Cloudwatch `cfg:"cloudwatch"` + Writer string `cfg:"writer"` } func getMetricSettings(config cfg.Config) *Settings { diff --git a/pkg/metric/naming.go b/pkg/metric/naming.go index 088062359..caa9415cf 100644 --- a/pkg/metric/naming.go +++ b/pkg/metric/naming.go @@ -9,7 +9,7 @@ import ( type NamingFactory func(modelId cfg.AppId) string var promNSNamingStrategy = func(modelId cfg.AppId) string { - return fmt.Sprintf("%v:%v:%v:%v", modelId.Project, modelId.Environment, modelId.Family, modelId.Application) + return fmt.Sprintf("%s:%s:%s:%s-%s", modelId.Project, modelId.Environment, modelId.Family, modelId.Group, modelId.Application) } func WithPromNSNamingStrategy(strategy NamingFactory) { diff --git a/pkg/metric/writer_cw.go b/pkg/metric/writer_cw.go index 1a5b201e4..928fe6b23 100644 --- a/pkg/metric/writer_cw.go +++ b/pkg/metric/writer_cw.go @@ -133,12 +133,26 @@ func (w *cwWriter) Write(batch Data) { } metricData, err := w.buildMetricData(batch) - namespace := fmt.Sprintf("%s/%s/%s/%s", w.settings.Project, w.settings.Environment, w.settings.Family, w.settings.Application) + namespace := w.settings.Cloudwatch.Naming.Pattern + values := map[string]string{ + "project": w.settings.Project, + "env": w.settings.Environment, + "family": w.settings.Family, + "group": w.settings.Group, + "app": w.settings.Application, + } + + for key, val := range values { + templ := fmt.Sprintf("{%s}", key) + namespace = strings.ReplaceAll(namespace, templ, val) + } + + logger := w.logger.WithFields(log.Fields{ + "namespace": namespace, + }) if err != nil { - w.logger.WithFields(log.Fields{ - "namespace": namespace, - }).Info("could not build metric data: %w", err) + logger.Info("could not build metric data: %w", err) return } @@ -156,12 +170,12 @@ func (w *cwWriter) Write(batch Data) { } if _, err = w.client.PutMetricData(context.Background(), &input); err != nil { - w.logger.Info("could not write metric data: %s", err) + logger.Info("could not write metric data: %s", err) continue } } - w.logger.Debug("written %d metric data sets to cloudwatch", len(metricData)) + logger.Debug("written %d metric data sets to cloudwatch", len(metricData)) } func (w *cwWriter) buildMetricData(batch Data) ([]types.MetricDatum, error) { diff --git a/pkg/metric/writer_cw_test.go b/pkg/metric/writer_cw_test.go index c8f82eec1..bae1c396b 100644 --- a/pkg/metric/writer_cw_test.go +++ b/pkg/metric/writer_cw_test.go @@ -38,7 +38,7 @@ func buildMocksAndWrite(now time.Time, metricTimeStamp time.Time) *cloudwatchMoc cwClient := new(cloudwatchMocks.Client) cwClient.On("PutMetricData", context.Background(), &cloudwatch.PutMetricDataInput{ - Namespace: aws.String("my/test/namespace/app"), + Namespace: aws.String("my/test/namespace/grp/app"), MetricData: []types.MetricDatum{{ MetricName: aws.String("my-test-metric-name"), Dimensions: []types.Dimension{ @@ -58,8 +58,14 @@ func buildMocksAndWrite(now time.Time, metricTimeStamp time.Time) *cloudwatchMoc Project: "my", Environment: "test", Family: "namespace", + Group: "grp", Application: "app", }, + Cloudwatch: metric.Cloudwatch{ + Naming: metric.NamingSettings{ + Pattern: "{project}/{env}/{family}/{group}/{app}", + }, + }, Enabled: true, }) diff --git a/pkg/metric/writer_es.go b/pkg/metric/writer_es.go index cb276858a..a01f2460c 100644 --- a/pkg/metric/writer_es.go +++ b/pkg/metric/writer_es.go @@ -32,7 +32,7 @@ func NewEsWriter(config cfg.Config, logger log.Logger) (*esWriter, error) { testClock := clock.NewRealClock() appId := cfg.GetAppIdFromConfig(config) - namespace := fmt.Sprintf("%s/%s/%s/%s", appId.Project, appId.Environment, appId.Family, appId.Application) + namespace := fmt.Sprintf("%s/%s/%s/%s-%s", appId.Project, appId.Environment, appId.Family, appId.Group, appId.Application) return NewEsWriterWithInterfaces(logger, client, testClock, namespace), nil } diff --git a/pkg/parquet/reader.go b/pkg/parquet/reader.go index 99918cb9e..8cc8d900e 100644 --- a/pkg/parquet/reader.go +++ b/pkg/parquet/reader.go @@ -371,6 +371,7 @@ func (r *s3Reader) getBucketName() string { Project: r.modelId.Project, Environment: r.modelId.Environment, Family: r.modelId.Family, + Group: r.modelId.Group, Application: r.modelId.Application, }) } diff --git a/pkg/parquet/writer.go b/pkg/parquet/writer.go index aae4be2d4..1fd8670e5 100644 --- a/pkg/parquet/writer.go +++ b/pkg/parquet/writer.go @@ -75,6 +75,7 @@ func NewWriterWithInterfaces( "Project": modelId.Project, "Environment": modelId.Environment, "Family": modelId.Family, + "Group": modelId.Group, "Application": modelId.Application, "Model": modelId.Name, } @@ -185,6 +186,7 @@ func (w *s3Writer) getBucketName() string { Project: w.modelId.Project, Environment: w.modelId.Environment, Family: w.modelId.Family, + Group: w.modelId.Group, Application: w.modelId.Application, }) } diff --git a/pkg/redis/client.go b/pkg/redis/client.go index d7c07f5f0..6b8653a86 100644 --- a/pkg/redis/client.go +++ b/pkg/redis/client.go @@ -52,7 +52,7 @@ type Pipeliner interface { } func GetFullyQualifiedKey(appId cfg.AppId, key string) string { - return fmt.Sprintf("%v-%v-%v-%v-%v", appId.Project, appId.Environment, appId.Family, appId.Application, key) + return fmt.Sprintf("%s-%s-%s-%s-%s-%s", appId.Project, appId.Environment, appId.Family, appId.Group, appId.Application, key) } //go:generate mockery --name Client diff --git a/pkg/redis/dialer.go b/pkg/redis/dialer.go index 668933a80..6250a1e9c 100644 --- a/pkg/redis/dialer.go +++ b/pkg/redis/dialer.go @@ -25,7 +25,7 @@ type ( ) var srvNamingStrategy = func(appId cfg.AppId, name string) string { - return fmt.Sprintf("%s.%s.redis.%s.%s", name, appId.Application, appId.Environment, appId.Family) + return fmt.Sprintf("%s.%s.redis.%s.%s", name, appId.Group, appId.Environment, appId.Family) } func dialerSrv(logger log.Logger, settings *Settings) func(ctx context.Context, network, addr string) (net.Conn, error) { diff --git a/pkg/redis/factory.go b/pkg/redis/factory.go index 68114927c..7a8c8583a 100644 --- a/pkg/redis/factory.go +++ b/pkg/redis/factory.go @@ -27,18 +27,18 @@ func ProvideClient(config cfg.Config, logger log.Logger, name string) (Client, e defer mutex.Unlock() settings := ReadSettings(config, name) - cachKey := fmt.Sprintf("%s:%s", settings.Address, name) + cacheKey := fmt.Sprintf("%s:%s", settings.Address, name) - if client, ok := clients[cachKey]; ok { + if client, ok := clients[cacheKey]; ok { return client, nil } var err error - if clients[cachKey], err = NewClient(config, logger, name); err != nil { + if clients[cacheKey], err = NewClient(config, logger, name); err != nil { return nil, err } - return clients[cachKey], nil + return clients[cacheKey], nil } func ReadSettings(config cfg.Config, name string) *Settings { diff --git a/pkg/redis/factory_test.go b/pkg/redis/factory_test.go index 881cb6694..34325d112 100644 --- a/pkg/redis/factory_test.go +++ b/pkg/redis/factory_test.go @@ -22,9 +22,10 @@ func (s *FactoryTestSuite) SetupTest() { func (s *FactoryTestSuite) initConfig(settings map[string]interface{}) { appIdConfig := cfg.WithConfigMap(map[string]interface{}{ "app_project": "gosoline", - "app_family": "test", + "app_family": "fam", + "app_group": "grp", "app_name": "redis", - "env": "test", + "env": "env", }) if err := s.config.Option(cfg.WithConfigMap(settings), appIdConfig); err != nil { @@ -40,8 +41,9 @@ func (s *FactoryTestSuite) TestDefault() { expected := &redis.Settings{ AppId: cfg.AppId{ Project: "gosoline", - Environment: "test", - Family: "test", + Environment: "env", + Family: "fam", + Group: "grp", Application: "redis", }, Name: "default", @@ -76,8 +78,9 @@ func (s *FactoryTestSuite) TestDedicated() { expected := &redis.Settings{ AppId: cfg.AppId{ Project: "gosoline", - Environment: "test", - Family: "test", + Environment: "env", + Family: "fam", + Group: "grp", Application: "redis", }, Name: "dedicated", @@ -114,8 +117,9 @@ func (s *FactoryTestSuite) TestWithDefaults() { expected := &redis.Settings{ AppId: cfg.AppId{ Project: "gosoline", - Environment: "test", - Family: "test", + Environment: "env", + Family: "fam", + Group: "grp", Application: "redis", }, Name: "partial", diff --git a/pkg/stream/consumer_base.go b/pkg/stream/consumer_base.go index b2fa3fb4d..a373725ae 100644 --- a/pkg/stream/consumer_base.go +++ b/pkg/stream/consumer_base.go @@ -152,7 +152,7 @@ func NewBaseConsumerWithInterfaces( ) *baseConsumer { return &baseConsumer{ name: name, - id: fmt.Sprintf("consumer-%s-%s-%s", appId.Family, appId.Application, name), + id: fmt.Sprintf("consumer-%s-%s-%s-%s", appId.Family, appId.Group, appId.Application, name), clock: clock.Provider, uuidGen: uuidGen, logger: logger, diff --git a/pkg/stream/input_configurable.go b/pkg/stream/input_configurable.go index 52f2c030f..9615eb8dd 100644 --- a/pkg/stream/input_configurable.go +++ b/pkg/stream/input_configurable.go @@ -111,6 +111,7 @@ func newKinesisInputFromConfig(ctx context.Context, config cfg.Config, logger lo type redisInputConfiguration struct { Project string `cfg:"project"` Family string `cfg:"family"` + Group string `cfg:"group"` Application string `cfg:"application"` ServerName string `cfg:"server_name" default:"default" validate:"min=1"` Key string `cfg:"key" validate:"required,min=1"` @@ -127,6 +128,7 @@ func newRedisInputFromConfig(_ context.Context, config cfg.Config, logger log.Lo AppId: cfg.AppId{ Project: configuration.Project, Family: configuration.Family, + Group: configuration.Group, Application: configuration.Application, }, ServerName: configuration.ServerName, @@ -139,6 +141,7 @@ func newRedisInputFromConfig(_ context.Context, config cfg.Config, logger log.Lo type SnsInputTargetConfiguration struct { Family string `cfg:"family"` + Group string `cfg:"group" validate:"required"` Application string `cfg:"application" validate:"required"` TopicId string `cfg:"topic_id" validate:"required"` Attributes map[string]interface{} `cfg:"attributes"` @@ -149,6 +152,7 @@ type SnsInputConfiguration struct { Type string `cfg:"type" default:"sns"` ConsumerId string `cfg:"id" validate:"required"` Family string `cfg:"family" default:""` + Group string `cfg:"group" default:""` Application string `cfg:"application" default:""` Targets []SnsInputTargetConfiguration `cfg:"targets" validate:"min=1"` MaxNumberOfMessages int32 `cfg:"max_number_of_messages" default:"10" validate:"min=1,max=10"` @@ -168,6 +172,7 @@ func readSnsInputSettings(config cfg.Config, name string) (*SnsInputSettings, [] settings := &SnsInputSettings{ AppId: cfg.AppId{ Family: configuration.Family, + Group: configuration.Group, Application: configuration.Application, }, QueueId: configuration.ConsumerId, @@ -185,6 +190,7 @@ func readSnsInputSettings(config cfg.Config, name string) (*SnsInputSettings, [] for i, t := range configuration.Targets { targetAppId := cfg.AppId{ Family: t.Family, + Group: t.Group, Application: t.Application, } @@ -214,6 +220,7 @@ func newSnsInputFromConfig(ctx context.Context, config cfg.Config, logger log.Lo type sqsInputConfiguration struct { Family string `cfg:"target_family"` + Group string `cfg:"target_group"` Application string `cfg:"target_application"` QueueId string `cfg:"target_queue_id" validate:"min=1"` MaxNumberOfMessages int32 `cfg:"max_number_of_messages" default:"10" validate:"min=1,max=10"` @@ -235,6 +242,7 @@ func readSqsInputSettings(config cfg.Config, name string) *SqsInputSettings { settings := &SqsInputSettings{ AppId: cfg.AppId{ Family: configuration.Family, + Group: configuration.Group, Application: configuration.Application, }, QueueId: configuration.QueueId, diff --git a/pkg/stream/mpr_config_postprocessor.go b/pkg/stream/mpr_config_postprocessor.go index f45a05de7..98bab17dd 100644 --- a/pkg/stream/mpr_config_postprocessor.go +++ b/pkg/stream/mpr_config_postprocessor.go @@ -2,6 +2,7 @@ package stream import ( "fmt" + "strings" "time" "github.com/justtrackio/gosoline/pkg/cfg" @@ -13,29 +14,40 @@ func init() { } func mprConfigPostprocessor(config cfg.GosoConf) (bool, error) { - enabled := config.GetBool(configKey+".enabled", false) - if !enabled { + settings := readMessagesPerRunnerMetricSettings(config) + if !settings.Enabled { return false, nil } - settings := readMessagesPerRunnerMetricSettings(config) - - key := ddb.GetLeaderElectionConfigKey(settings.LeaderElection) - typKey := ddb.GetLeaderElectionConfigKeyType(settings.LeaderElection) + electionKey := ddb.GetLeaderElectionConfigKey(settings.LeaderElection) + electionKeyType := ddb.GetLeaderElectionConfigKeyType(settings.LeaderElection) - if config.IsSet(key) { + if config.IsSet(electionKey) { return true, nil } + pattern := settings.DynamoDb.Naming.Pattern + + values := map[string]string{ + "modelId": "stream-metric-writer-leaders", + } + + for key, val := range values { + templ := fmt.Sprintf("{%s}", key) + pattern = strings.ReplaceAll(pattern, templ, val) + } + leaderElectionSettings := &ddb.DdbLeaderElectionSettings{ - TableName: fmt.Sprintf("%s-%s-%s-stream-metric-writer-leaders", config.GetString("app_project"), config.GetString("env"), config.GetString("app_family")), - GroupId: config.GetString("app_name"), + Naming: ddb.TableNamingSettings{ + Pattern: pattern, + }, + GroupId: fmt.Sprintf("%s-%s", config.GetString("app_group"), config.GetString("app_name")), LeaseDuration: time.Minute, } configOptions := []cfg.Option{ - cfg.WithConfigSetting(typKey, ddb.LeaderElectionTypeDdb), - cfg.WithConfigSetting(key, leaderElectionSettings), + cfg.WithConfigSetting(electionKeyType, ddb.LeaderElectionTypeDdb), + cfg.WithConfigSetting(electionKey, leaderElectionSettings), } if err := config.Option(configOptions...); err != nil { diff --git a/pkg/stream/mpr_metric_module.go b/pkg/stream/mpr_metric_module.go index e40226ed5..995ddc7c2 100644 --- a/pkg/stream/mpr_metric_module.go +++ b/pkg/stream/mpr_metric_module.go @@ -3,6 +3,7 @@ package stream import ( "context" "fmt" + "strings" "time" "github.com/aws/aws-sdk-go-v2/aws" @@ -24,14 +25,14 @@ const ( ) type MessagesPerRunnerMetricWriterSettings struct { - QueueNames []string - Ecs MessagesPerRunnerEcsSettings - UpdatePeriod time.Duration - TargetValue float64 - MaxIncreasePercent float64 - MaxIncreasePeriod time.Duration - AppId cfg.AppId - MemberId string + Ecs MessagesPerRunnerEcsSettings + MaxIncreasePeriod time.Duration + UpdatePeriod time.Duration + CloudwatchNamespace string + MaxIncreasePercent float64 + MemberId string + QueueNames []string + TargetValue float64 } func MessagesPerRunnerMetricWriterFactory(_ context.Context, config cfg.Config, _ log.Logger) (map[string]kernel.ModuleFactory, error) { @@ -78,17 +79,18 @@ func NewMessagesPerRunnerMetricWriter(settings *MessagesPerRunnerMetricSettings) return nil, fmt.Errorf("failed to detect any SQS queues to monitor") } + cwNamespace := getCloudwatchNamespace(config, settings.Cloudwatch.Naming.Pattern) + writerSettings := &MessagesPerRunnerMetricWriterSettings{ - QueueNames: queueNames, - UpdatePeriod: settings.Period, - TargetValue: settings.TargetValue, - MaxIncreasePercent: settings.MaxIncreasePercent, - MaxIncreasePeriod: settings.MaxIncreasePeriod, - Ecs: settings.Ecs, - AppId: cfg.AppId{}, - MemberId: uuid.New().NewV4(), + CloudwatchNamespace: cwNamespace, + QueueNames: queueNames, + UpdatePeriod: settings.Period, + TargetValue: settings.TargetValue, + MaxIncreasePercent: settings.MaxIncreasePercent, + MaxIncreasePeriod: settings.MaxIncreasePeriod, + Ecs: settings.Ecs, + MemberId: uuid.New().NewV4(), } - writerSettings.AppId.PadFromConfig(config) if leaderElection, err = ddb.NewLeaderElection(ctx, config, logger, settings.LeaderElection); err != nil { return nil, fmt.Errorf("can not create leader election for stream-metric-messages-per-runner writer: %w", err) @@ -105,6 +107,26 @@ func NewMessagesPerRunnerMetricWriter(settings *MessagesPerRunnerMetricSettings) } } +func getCloudwatchNamespace(config cfg.Config, cwNamespacePattern string) string { + appId := cfg.AppId{} + appId.PadFromConfig(config) + + values := map[string]string{ + "project": appId.Project, + "env": appId.Environment, + "family": appId.Family, + "group": appId.Group, + "app": appId.Application, + } + + for key, val := range values { + templ := fmt.Sprintf("{%s}", key) + cwNamespacePattern = strings.ReplaceAll(cwNamespacePattern, templ, val) + } + + return cwNamespacePattern +} + func NewMessagesPerRunnerMetricWriterWithInterfaces(logger log.Logger, leaderElection ddb.LeaderElection, cwClient gosoCloudwatch.Client, metricWriter metric.Writer, clock clock.Clock, ticker clock.Ticker, settings *MessagesPerRunnerMetricWriterSettings) (*MessagesPerRunnerMetricWriter, error) { writer := &MessagesPerRunnerMetricWriter{ logger: logger, @@ -192,7 +214,7 @@ func (u *MessagesPerRunnerMetricWriter) calculateMessagesPerRunner(ctx context.C } if currentMpr, err = u.getStreamMprMetric(ctx, metricNameStreamMprMessagesPerRunner, types.StatisticAverage, u.settings.MaxIncreasePeriod); err != nil { - u.logger.Warn("can not get current messages per runner metric: defaulting to 0") + u.logger.Warn("can not get current messages per runner metric: %s, defaulting to 0", err.Error()) currentMpr = 0 } @@ -274,8 +296,7 @@ func (u *MessagesPerRunnerMetricWriter) getQueueMetrics(ctx context.Context, met } func (u *MessagesPerRunnerMetricWriter) getStreamMprMetric(ctx context.Context, name string, stat types.Statistic, period time.Duration) (float64, error) { - appId := u.settings.AppId - namespace := fmt.Sprintf("%s/%s/%s/%s", appId.Project, appId.Environment, appId.Family, appId.Application) + namespace := u.settings.CloudwatchNamespace startTime := u.clock.Now().Add(-1 * period) endTime := u.clock.Now().Add(-1 * u.settings.UpdatePeriod) diff --git a/pkg/stream/mpr_metric_module_test.go b/pkg/stream/mpr_metric_module_test.go index 53179cf54..ea05462d9 100644 --- a/pkg/stream/mpr_metric_module_test.go +++ b/pkg/stream/mpr_metric_module_test.go @@ -10,7 +10,6 @@ import ( "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/cloudwatch" "github.com/aws/aws-sdk-go-v2/service/cloudwatch/types" - "github.com/justtrackio/gosoline/pkg/cfg" "github.com/justtrackio/gosoline/pkg/clock" cloudwatchMocks "github.com/justtrackio/gosoline/pkg/cloud/aws/cloudwatch/mocks" concMocks "github.com/justtrackio/gosoline/pkg/conc/mocks" @@ -142,7 +141,7 @@ var mprMetricModuleTestCases = map[string]mprMetricModuleTestCase{ err := fmt.Errorf("unknown error") s.mockGetMetricMessagesPerRunner(500, err) - s.logger.On("Warn", "can not get current messages per runner metric: defaulting to 0") + s.logger.On("Warn", "can not get current messages per runner metric: %s, defaulting to 0", "can not get metric: unknown error") s.mockSuccessLogger(1000, 0, 2, 500) s.mockMetricWriteMessagesPerRunner(500) @@ -194,19 +193,14 @@ func (s *MprMetricModuleTestSuite) SetupTestCase() { s.ticker = s.clock.NewTicker(time.Minute) s.settings = &stream.MessagesPerRunnerMetricWriterSettings{ - QueueNames: []string{"queueName"}, - UpdatePeriod: time.Minute, - MaxIncreasePercent: 200, - MaxIncreasePeriod: time.Minute * 5, - AppId: cfg.AppId{ - Project: "gosoline", - Environment: "test", - Family: "stream", - Application: "mprMetric", - }, + QueueNames: []string{"queueName"}, + CloudwatchNamespace: "gosoline/test/stream/grp/mprMetric", + UpdatePeriod: time.Minute, + MaxIncreasePercent: 200, + MaxIncreasePeriod: time.Minute * 5, Ecs: stream.MessagesPerRunnerEcsSettings{ Cluster: "gosoline-test-stream", - Service: "mprMetric", + Service: "grp/mprMetric", }, MemberId: "e7c6003c-66df-11eb-9bdf-af0dafba2813", } @@ -313,7 +307,7 @@ func (s *MprMetricModuleTestSuite) mockGetMetricMpr(metric string, stat types.St Id: aws.String("m1"), MetricStat: &types.MetricStat{ Metric: &types.Metric{ - Namespace: aws.String("gosoline/test/stream/mprMetric"), + Namespace: aws.String("gosoline/test/stream/grp/mprMetric"), MetricName: aws.String(metric), }, Period: aws.Int32(60), @@ -355,7 +349,7 @@ func (s *MprMetricModuleTestSuite) mockGetMetricEcs(metric string, stat types.St }, { Name: aws.String("ServiceName"), - Value: aws.String("mprMetric"), + Value: aws.String("grp/mprMetric"), }, }, }, diff --git a/pkg/stream/mpr_settings.go b/pkg/stream/mpr_settings.go index 06a27431d..c99c8f30b 100644 --- a/pkg/stream/mpr_settings.go +++ b/pkg/stream/mpr_settings.go @@ -12,23 +12,39 @@ const ( type MessagesPerRunnerEcsSettings struct { Cluster string `cfg:"cluster" default:"{app_project}-{env}-{app_family}"` - Service string `cfg:"service" default:"{app_name}"` + Service string `cfg:"service" default:"{app_group}-{app_name}"` +} + +type MessagesPerRunnerDdbServiceNamingSettings struct { + Naming MessagesPerRunnerDdbNamingSettings `cfg:"naming"` +} + +type MessagesPerRunnerDdbNamingSettings struct { + Pattern string `cfg:"pattern,nodecode" default:"{project}-{env}-{family}-{modelId}"` +} + +type MessagesPerRunnerCwServiceNamingSettings struct { + Naming MessagesPerRunnerCwNamingSettings `cfg:"naming"` +} + +type MessagesPerRunnerCwNamingSettings struct { + Pattern string `cfg:"pattern,nodecode" default:"{project}/{env}/{family}/{group}-{app}"` } type MessagesPerRunnerMetricSettings struct { - Enabled bool `cfg:"enabled"` - Ecs MessagesPerRunnerEcsSettings `cfg:"ecs"` - LeaderElection string `cfg:"leader_election" default:"streamMprMetrics"` - MaxIncreasePercent float64 `cfg:"max_increase_percent" default:"200"` - MaxIncreasePeriod time.Duration `cfg:"max_increase_period" default:"5m"` - Period time.Duration `cfg:"period" default:"1m"` - TargetValue float64 `cfg:"target_value" default:"0"` + Enabled bool `cfg:"enabled"` + Ecs MessagesPerRunnerEcsSettings `cfg:"ecs"` + LeaderElection string `cfg:"leader_election" default:"streamMprMetrics"` + MaxIncreasePercent float64 `cfg:"max_increase_percent" default:"200"` + MaxIncreasePeriod time.Duration `cfg:"max_increase_period" default:"5m"` + DynamoDb MessagesPerRunnerDdbServiceNamingSettings `cfg:"dynamodb"` + Cloudwatch MessagesPerRunnerCwServiceNamingSettings `cfg:"cloudwatch"` + Period time.Duration `cfg:"period" default:"1m"` + TargetValue float64 `cfg:"target_value" default:"0"` } func readMessagesPerRunnerMetricSettings(config cfg.Config) *MessagesPerRunnerMetricSettings { - mprSettings := &MessagesPerRunnerMetricSettings{ - Ecs: MessagesPerRunnerEcsSettings{}, - } + mprSettings := &MessagesPerRunnerMetricSettings{} config.UnmarshalKey(configKey, mprSettings) return mprSettings diff --git a/pkg/stream/output_configurable.go b/pkg/stream/output_configurable.go index 8b2311a6c..18d9bff8f 100644 --- a/pkg/stream/output_configurable.go +++ b/pkg/stream/output_configurable.go @@ -100,6 +100,7 @@ type KinesisOutputConfiguration struct { Type string `cfg:"type" default:"kinesis"` Project string `cfg:"project"` Family string `cfg:"family"` + Group string `cfg:"group"` Application string `cfg:"application"` ClientName string `cfg:"client_name" default:"default"` StreamName string `cfg:"stream_name"` @@ -114,6 +115,7 @@ func newKinesisOutputFromConfig(ctx context.Context, config cfg.Config, logger l AppId: cfg.AppId{ Project: configuration.Project, Family: configuration.Family, + Group: configuration.Group, Application: configuration.Application, }, ClientName: configuration.ClientName, @@ -124,6 +126,7 @@ func newKinesisOutputFromConfig(ctx context.Context, config cfg.Config, logger l type redisListOutputConfiguration struct { Project string `cfg:"project"` Family string `cfg:"family"` + Group string `cfg:"group"` Application string `cfg:"application"` ServerName string `cfg:"server_name" default:"default" validate:"required,min=1"` Key string `cfg:"key" validate:"required,min=1"` @@ -140,6 +143,7 @@ func newRedisListOutputFromConfig(_ context.Context, config cfg.Config, logger l AppId: cfg.AppId{ Project: configuration.Project, Family: configuration.Family, + Group: configuration.Group, Application: configuration.Application, }, ServerName: configuration.ServerName, @@ -153,6 +157,7 @@ type SnsOutputConfiguration struct { Type string `cfg:"type" default:"sns"` Project string `cfg:"project"` Family string `cfg:"family"` + Group string `cfg:"group"` Application string `cfg:"application"` TopicId string `cfg:"topic_id" validate:"required"` ClientName string `cfg:"client_name" default:"default"` @@ -167,6 +172,7 @@ func newSnsOutputFromConfig(ctx context.Context, config cfg.Config, logger log.L AppId: cfg.AppId{ Project: configuration.Project, Family: configuration.Family, + Group: configuration.Group, Application: configuration.Application, }, TopicId: configuration.TopicId, @@ -179,6 +185,7 @@ type SqsOutputConfiguration struct { Type string `cfg:"type" default:"sqs"` Project string `cfg:"project"` Family string `cfg:"family"` + Group string `cfg:"group"` Application string `cfg:"application"` QueueId string `cfg:"queue_id" validate:"required"` VisibilityTimeout int `cfg:"visibility_timeout" default:"30" validate:"gt=0"` @@ -196,6 +203,7 @@ func newSqsOutputFromConfig(ctx context.Context, config cfg.Config, logger log.L AppId: cfg.AppId{ Project: configuration.Project, Family: configuration.Family, + Group: configuration.Group, Application: configuration.Application, }, QueueId: configuration.QueueId, diff --git a/pkg/stream/output_redis_list_test.go b/pkg/stream/output_redis_list_test.go index 544b4456e..73bb747ee 100644 --- a/pkg/stream/output_redis_list_test.go +++ b/pkg/stream/output_redis_list_test.go @@ -15,7 +15,7 @@ import ( func TestRedisListOutput_WriteOne(t *testing.T) { output, redisMock := setup(1) - redisMock.On("RPush", mock.AnythingOfType("*context.emptyCtx"), "mcoins-test-analytics-app-my-list", mock.AnythingOfType("[]uint8")).Return(int64(1), nil).Once() + redisMock.On("RPush", mock.AnythingOfType("*context.emptyCtx"), "mcoins-test-fam-grp-app-my-list", mock.AnythingOfType("[]uint8")).Return(int64(1), nil).Once() record := stream.NewMessage("bla") err := output.WriteOne(context.Background(), record) @@ -26,7 +26,7 @@ func TestRedisListOutput_WriteOne(t *testing.T) { func TestRedisListOutput_Write(t *testing.T) { output, redisMock := setup(2) - redisMock.On("RPush", mock.AnythingOfType("*context.emptyCtx"), "mcoins-test-analytics-app-my-list", mock.AnythingOfType("[]uint8"), mock.AnythingOfType("[]uint8")).Return(int64(2), nil).Once() + redisMock.On("RPush", mock.AnythingOfType("*context.emptyCtx"), "mcoins-test-fam-grp-app-my-list", mock.AnythingOfType("[]uint8"), mock.AnythingOfType("[]uint8")).Return(int64(2), nil).Once() batch := []stream.WritableMessage{ stream.NewMessage("foo"), @@ -40,7 +40,7 @@ func TestRedisListOutput_Write(t *testing.T) { func TestRedisListOutput_Write_Chunked(t *testing.T) { output, redisMock := setup(1) - redisMock.On("RPush", mock.AnythingOfType("*context.emptyCtx"), "mcoins-test-analytics-app-my-list", mock.AnythingOfType("[]uint8")).Return(int64(1), nil).Times(2) + redisMock.On("RPush", mock.AnythingOfType("*context.emptyCtx"), "mcoins-test-fam-grp-app-my-list", mock.AnythingOfType("[]uint8")).Return(int64(1), nil).Times(2) batch := []stream.WritableMessage{ stream.NewMessage("foo"), @@ -67,7 +67,8 @@ func getSettings(batchSize int) *stream.RedisListOutputSettings { AppId: cfg.AppId{ Project: "mcoins", Environment: "test", - Family: "analytics", + Family: "fam", + Group: "grp", Application: "app", }, Key: "my-list", diff --git a/pkg/tracing/span.go b/pkg/tracing/span.go index af8b9662d..4cfb6d33a 100644 --- a/pkg/tracing/span.go +++ b/pkg/tracing/span.go @@ -83,8 +83,8 @@ func newSpan(ctx context.Context, seg *xray.Segment, app cfg.AppId) (context.Con segment: seg, } - appFamily := fmt.Sprintf("%v-%v-%v", app.Project, app.Environment, app.Family) - appId := fmt.Sprintf("%v-%v-%v-%v", app.Project, app.Environment, app.Family, app.Application) + appFamily := fmt.Sprintf("%s-%s-%s", app.Project, app.Environment, app.Family) + appId := fmt.Sprintf("%s-%s-%s-%s-%s", app.Project, app.Environment, app.Family, app.Group, app.Application) span.AddAnnotation("appFamily", appFamily) span.AddAnnotation("appId", appId) diff --git a/pkg/tracing/tracer_aws.go b/pkg/tracing/tracer_aws.go index f08048d9c..9687f0447 100644 --- a/pkg/tracing/tracer_aws.go +++ b/pkg/tracing/tracer_aws.go @@ -150,7 +150,7 @@ func (t *awsTracer) HttpHandler(h http.Handler) http.Handler { return h } - name := fmt.Sprintf("%v-%v-%v-%v", t.Project, t.Environment, t.Family, t.Application) + name := fmt.Sprintf("%s-%s-%s-%s-%s", t.Project, t.Environment, t.Family, t.Group, t.Application) handlerFunc := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { ctx := r.Context() seg := xray.GetSegment(r.Context()) diff --git a/pkg/tracing/tracer_aws_test.go b/pkg/tracing/tracer_aws_test.go index 261af158f..f2f1cda41 100644 --- a/pkg/tracing/tracer_aws_test.go +++ b/pkg/tracing/tracer_aws_test.go @@ -64,6 +64,7 @@ func getTracer(t *testing.T) tracing.Tracer { Project: "test_project", Environment: "test_env", Family: "test_family", + Group: "test_group", Application: "test_name", }, &tracing.XRaySettings{ Enabled: true, diff --git a/test/apiserver/config.dist.compressed.yml b/test/apiserver/config.dist.compressed.yml index 03a43dcb2..46ec7c005 100644 --- a/test/apiserver/config.dist.compressed.yml +++ b/test/apiserver/config.dist.compressed.yml @@ -2,6 +2,7 @@ env: test app_project: gosoline app_family: test +app_group: grp app_name: compression-test api: diff --git a/test/apiserver/config.dist.eofBind.yml b/test/apiserver/config.dist.eofBind.yml index 819a9de3b..ce2b21208 100644 --- a/test/apiserver/config.dist.eofBind.yml +++ b/test/apiserver/config.dist.eofBind.yml @@ -2,6 +2,7 @@ env: test app_project: gosoline app_family: test +app_group: grp app_name: eof-bind-test api: diff --git a/test/apiserver/config.dist.validation.yml b/test/apiserver/config.dist.validation.yml index ea61fc6bc..e5760ffa2 100644 --- a/test/apiserver/config.dist.validation.yml +++ b/test/apiserver/config.dist.validation.yml @@ -2,6 +2,7 @@ env: test app_project: gosoline app_family: test +app_group: grp app_name: validation-test api: diff --git a/test/cloud/aws/cloudwatch/client_test_cfg.yml b/test/cloud/aws/cloudwatch/client_test_cfg.yml index 0c440b6e7..660616bbe 100644 --- a/test/cloud/aws/cloudwatch/client_test_cfg.yml +++ b/test/cloud/aws/cloudwatch/client_test_cfg.yml @@ -1,3 +1,10 @@ +env: test + +app_project: prj +app_family: fam +app_group: grp +app_name: app + cloud: aws: credentials: diff --git a/test/cloud/aws/dynamodb/client_test_cfg.yml b/test/cloud/aws/dynamodb/client_test_cfg.yml index 91950c056..d30b68bfe 100644 --- a/test/cloud/aws/dynamodb/client_test_cfg.yml +++ b/test/cloud/aws/dynamodb/client_test_cfg.yml @@ -1,3 +1,10 @@ +env: test + +app_project: prj +app_family: fam +app_group: grp +app_name: app + cloud: aws: credentials: diff --git a/test/cloud/aws/sqs/client_test_cfg.yml b/test/cloud/aws/sqs/client_test_cfg.yml index dfd461d5c..5e7aba188 100644 --- a/test/cloud/aws/sqs/client_test_cfg.yml +++ b/test/cloud/aws/sqs/client_test_cfg.yml @@ -1,3 +1,10 @@ +env: test + +app_project: prj +app_family: fam +app_group: grp +app_name: app + cloud: aws: credentials: diff --git a/test/conc/config.dist.yml b/test/conc/config.dist.yml index 9fb0338d9..411afad14 100644 --- a/test/conc/config.dist.yml +++ b/test/conc/config.dist.yml @@ -2,6 +2,7 @@ env: test app_project: gosoline app_family: test +app_group: grp app_name: ddb-lock-test exec: diff --git a/test/db-repo/change_history/config.test.yml b/test/db-repo/change_history/config.test.yml index 3e5aa9425..1ee4de7f5 100644 --- a/test/db-repo/change_history/config.test.yml +++ b/test/db-repo/change_history/config.test.yml @@ -1,6 +1,7 @@ env: test app_project: gosoline app_family: integration-test +app_group: grp app_name: db-repo-change-history-test db: diff --git a/test/db-repo/query/config.test.yml b/test/db-repo/query/config.test.yml index af8982182..54338ff4f 100644 --- a/test/db-repo/query/config.test.yml +++ b/test/db-repo/query/config.test.yml @@ -1,6 +1,7 @@ env: test app_project: gosoline app_family: integration-test +app_group: grp app_name: db-query-test db: diff --git a/test/ddb/config.dist.yml b/test/ddb/config.dist.yml index f76516988..98735aaa6 100644 --- a/test/ddb/config.dist.yml +++ b/test/ddb/config.dist.yml @@ -2,6 +2,7 @@ env: test app_project: gosoline app_family: test +app_group: grp app_name: ddb-test cloud: diff --git a/test/fixtures/dynamodb/config.test.yml b/test/fixtures/dynamodb/config.test.yml index bb86117fb..5542a7a0c 100644 --- a/test/fixtures/dynamodb/config.test.yml +++ b/test/fixtures/dynamodb/config.test.yml @@ -1,3 +1,10 @@ +env: test + +app_project: prj +app_family: fam +app_group: grp +app_name: app + test: components: ddb: diff --git a/test/fixtures/dynamodb/dynamodb_test.go b/test/fixtures/dynamodb/dynamodb_test.go index 438faf800..fc45e3ccf 100644 --- a/test/fixtures/dynamodb/dynamodb_test.go +++ b/test/fixtures/dynamodb/dynamodb_test.go @@ -54,7 +54,7 @@ func (s *DynamoDbSuite) TestDynamoDb() { Value: "Ash", }, }, - TableName: aws.String("gosoline-test-integration-test-test-application-testModel"), + TableName: aws.String("gosoline-test-integration-test-grp-testModel"), }) // should have created the item @@ -64,7 +64,7 @@ func (s *DynamoDbSuite) TestDynamoDb() { s.Equal("10", gio.Item["Age"].(*types.AttributeValueMemberN).Value) qo, err := ddbClient.Query(context.Background(), &dynamodb.QueryInput{ - TableName: aws.String("gosoline-test-integration-test-test-application-testModel"), + TableName: aws.String("gosoline-test-integration-test-grp-testModel"), IndexName: aws.String("IDX_Age"), KeyConditionExpression: aws.String("Age = :v_age"), ExpressionAttributeValues: map[string]types.AttributeValue{ @@ -78,7 +78,7 @@ func (s *DynamoDbSuite) TestDynamoDb() { s.NoError(err) s.Len(qo.Items, 1, "1 item expected") - _, err = ddbClient.DeleteTable(envContext, &dynamodb.DeleteTableInput{TableName: aws.String("gosoline-test-integration-test-test-application-testModel")}) + _, err = ddbClient.DeleteTable(envContext, &dynamodb.DeleteTableInput{TableName: aws.String("gosoline-test-integration-test-grp-testModel")}) s.NoError(err) } @@ -100,7 +100,7 @@ func (s *DynamoDbSuite) TestDynamoDbWithPurge() { Value: "Ash", }, }, - TableName: aws.String("gosoline-test-integration-test-test-application-testModel"), + TableName: aws.String("gosoline-test-integration-test-grp-testModel"), }) // should have created the first item @@ -118,7 +118,7 @@ func (s *DynamoDbSuite) TestDynamoDbWithPurge() { Value: "Bash", }, }, - TableName: aws.String("gosoline-test-integration-test-test-application-testModel"), + TableName: aws.String("gosoline-test-integration-test-grp-testModel"), }) // should have created the second item @@ -133,7 +133,7 @@ func (s *DynamoDbSuite) TestDynamoDbWithPurge() { Value: "Ash", }, }, - TableName: aws.String("gosoline-test-integration-test-test-application-testModel"), + TableName: aws.String("gosoline-test-integration-test-grp-testModel"), }) // first item should have been purged @@ -141,7 +141,7 @@ func (s *DynamoDbSuite) TestDynamoDbWithPurge() { s.Nil(gio.Item) qo, err := ddbClient.Query(context.Background(), &dynamodb.QueryInput{ - TableName: aws.String("gosoline-test-integration-test-test-application-testModel"), + TableName: aws.String("gosoline-test-integration-test-grp-testModel"), IndexName: aws.String("IDX_Age"), KeyConditionExpression: aws.String("Age = :v_age"), ExpressionAttributeValues: map[string]types.AttributeValue{ @@ -154,7 +154,7 @@ func (s *DynamoDbSuite) TestDynamoDbWithPurge() { s.NoError(err) s.Len(qo.Items, 1, "1 item expected") - _, err = ddbClient.DeleteTable(envContext, &dynamodb.DeleteTableInput{TableName: aws.String("gosoline-test-integration-test-test-application-testModel")}) + _, err = ddbClient.DeleteTable(envContext, &dynamodb.DeleteTableInput{TableName: aws.String("gosoline-test-integration-test-grp-testModel")}) s.NoError(err) } @@ -176,7 +176,7 @@ func (s *DynamoDbSuite) TestDynamoDbKvStore() { Value: "Ash", }, }, - TableName: aws.String("gosoline-test-integration-test-test-application-kvstore-testModel"), + TableName: aws.String("gosoline-test-integration-test-grp-kvstore-testModel"), }) // should have created the item @@ -193,7 +193,7 @@ func (s *DynamoDbSuite) TestDynamoDbKvStore() { } s.Equal(expectedValue, gio.Item["value"].(*types.AttributeValueMemberS)) - _, err = ddbClient.DeleteTable(envContext, &dynamodb.DeleteTableInput{TableName: aws.String("gosoline-test-integration-test-test-application-kvstore-testModel")}) + _, err = ddbClient.DeleteTable(envContext, &dynamodb.DeleteTableInput{TableName: aws.String("gosoline-test-integration-test-grp-kvstore-testModel")}) s.NoError(err) } @@ -215,7 +215,7 @@ func (s *DynamoDbSuite) TestDynamoDbKvStoreWithPurge() { Value: "Ash", }, }, - TableName: aws.String("gosoline-test-integration-test-test-application-kvstore-testModel"), + TableName: aws.String("gosoline-test-integration-test-grp-kvstore-testModel"), }) // should have created the first item @@ -236,7 +236,7 @@ func (s *DynamoDbSuite) TestDynamoDbKvStoreWithPurge() { Value: "Bash", }, }, - TableName: aws.String("gosoline-test-integration-test-test-application-kvstore-testModel"), + TableName: aws.String("gosoline-test-integration-test-grp-kvstore-testModel"), }) // should have created the second item @@ -259,14 +259,14 @@ func (s *DynamoDbSuite) TestDynamoDbKvStoreWithPurge() { Value: "Ash", }, }, - TableName: aws.String("gosoline-test-integration-test-test-application-kvstore-testModel"), + TableName: aws.String("gosoline-test-integration-test-grp-kvstore-testModel"), }) // first item should have been purged s.NoError(err) s.Nil(gio.Item) - _, err = ddbClient.DeleteTable(envContext, &dynamodb.DeleteTableInput{TableName: aws.String("gosoline-test-integration-test-test-application-kvstore-testModel")}) + _, err = ddbClient.DeleteTable(envContext, &dynamodb.DeleteTableInput{TableName: aws.String("gosoline-test-integration-test-grp-kvstore-testModel")}) s.NoError(err) } @@ -279,6 +279,7 @@ var kvStoreSettings = &mdl.ModelId{ Project: "gosoline", Environment: "test", Family: "integration-test", + Group: "grp", Application: "test-application", Name: "testModel", } @@ -319,6 +320,7 @@ var ddbSettings = &ddb.Settings{ Project: "gosoline", Environment: "test", Family: "integration-test", + Group: "grp", Application: "test-application", Name: "testModel", }, diff --git a/test/fixtures/kvstore/config.test.yml b/test/fixtures/kvstore/config.test.yml index 50899b10f..4b8cf8c90 100644 --- a/test/fixtures/kvstore/config.test.yml +++ b/test/fixtures/kvstore/config.test.yml @@ -2,6 +2,7 @@ env: test app_project: prj app_family: fam +app_group: grp app_name: app kvstore: diff --git a/test/fixtures/mysql/config.test.yml b/test/fixtures/mysql/config.test.yml index 8c8f3d658..a10d34455 100644 --- a/test/fixtures/mysql/config.test.yml +++ b/test/fixtures/mysql/config.test.yml @@ -1,6 +1,7 @@ env: test app_project: gosoline app_family: integration-test +app_group: grp app_name: fixture-loader db: diff --git a/test/fixtures/redis/config.test.yml b/test/fixtures/redis/config.test.yml index 44283d1e4..833c68775 100644 --- a/test/fixtures/redis/config.test.yml +++ b/test/fixtures/redis/config.test.yml @@ -2,6 +2,7 @@ env: test app_project: prj app_family: fam +app_group: grp app_name: app fixtures: diff --git a/test/fixtures/redis/redis_test.go b/test/fixtures/redis/redis_test.go index 2c1686b1e..d3d27dab1 100644 --- a/test/fixtures/redis/redis_test.go +++ b/test/fixtures/redis/redis_test.go @@ -115,7 +115,7 @@ func (s *RedisTestSuite) TestRedisKvStore() { s.NoError(err) // should have created the item - res, err := redisClient.Get(context.Background(), "gosoline-integration-test-test-application-kvstore-testModel-kvstore_entry_1").Result() + res, err := redisClient.Get(context.Background(), "gosoline-integration-test-grp-kvstore-testModel-kvstore_entry_1").Result() s.NoError(err) s.JSONEq(`{"name":"foo","age":123}`, res) } @@ -136,7 +136,7 @@ func (s *RedisTestSuite) TestRedisKvStoreWithPurge() { s.NoError(err) // should have created the first item - res, err := redisClient.Get(context.Background(), "gosoline-integration-test-test-application-kvstore-testModel-kvstore_entry_1").Result() + res, err := redisClient.Get(context.Background(), "gosoline-integration-test-grp-kvstore-testModel-kvstore_entry_1").Result() s.NoError(err) s.JSONEq(`{"name":"foo","age":123}`, res) @@ -144,11 +144,11 @@ func (s *RedisTestSuite) TestRedisKvStoreWithPurge() { s.NoError(err) // the first item should have been purged - res, err = redisClient.Get(context.Background(), "gosoline-integration-test-test-application-kvstore-testModel-kvstore_entry_1").Result() + res, err = redisClient.Get(context.Background(), "gosoline-integration-test-grp-kvstore-testModel-kvstore_entry_1").Result() s.Error(err) // should have created the second item - res, err = redisClient.Get(context.Background(), "gosoline-integration-test-test-application-kvstore-testModel-kvstore_entry_2").Result() + res, err = redisClient.Get(context.Background(), "gosoline-integration-test-grp-kvstore-testModel-kvstore_entry_2").Result() s.NoError(err) s.JSONEq(`{"name":"foo","age":123}`, res) } @@ -212,6 +212,7 @@ func kvStoreDisabledPurgeFixtures() []*fixtures.FixtureSet { Project: "gosoline", Environment: "test", Family: "integration-test", + Group: "grp", Application: "test-application", Name: "testModel", }), @@ -237,6 +238,7 @@ func kvStoreEnabledPurgeFixtures() []*fixtures.FixtureSet { Project: "gosoline", Environment: "test", Family: "integration-test", + Group: "grp", Application: "test-application", Name: "testModel", }), diff --git a/test/fixtures/s3/config.test.yml b/test/fixtures/s3/config.test.yml index 7b02e152d..75b252ffd 100644 --- a/test/fixtures/s3/config.test.yml +++ b/test/fixtures/s3/config.test.yml @@ -2,6 +2,7 @@ env: test app_project: prj app_family: fam +app_group: grp app_name: app cloud: diff --git a/test/guard/config.dist.yml b/test/guard/config.dist.yml index 0e2c3d422..d3e48c48f 100644 --- a/test/guard/config.dist.yml +++ b/test/guard/config.dist.yml @@ -2,6 +2,7 @@ env: test app_project: gosoline app_family: test +app_group: grp app_name: guard-test db: diff --git a/test/stream/producer_daemon/config.dist.yml b/test/stream/producer_daemon/config.dist.yml index 48e7132d5..b348b686a 100644 --- a/test/stream/producer_daemon/config.dist.yml +++ b/test/stream/producer_daemon/config.dist.yml @@ -2,6 +2,7 @@ env: test app_project: gosoline app_family: test +app_group: grp app_name: producer-daemon-test stream: diff --git a/test/stream/retry_handler/config.dist.yml b/test/stream/retry_handler/config.dist.yml index d3ad9c568..e9a510796 100644 --- a/test/stream/retry_handler/config.dist.yml +++ b/test/stream/retry_handler/config.dist.yml @@ -2,6 +2,7 @@ env: test app_project: gosoline app_family: test +app_group: grp app_name: retry-handler-test stream: diff --git a/test/suite/config.dist.yml b/test/suite/config.dist.yml new file mode 100644 index 000000000..a7b08c19a --- /dev/null +++ b/test/suite/config.dist.yml @@ -0,0 +1,6 @@ +env: test + +app_project: prj +app_family: fam +app_group: grp +app_name: app diff --git a/test/suite/suite_test.go b/test/suite/suite_test.go index c2e764ced..3bfa065ed 100644 --- a/test/suite/suite_test.go +++ b/test/suite/suite_test.go @@ -17,6 +17,7 @@ type SuiteTestSuite struct { func (s *SuiteTestSuite) SetupSuite() []suite.Option { return []suite.Option{ + suite.WithConfigFile("config.dist.yml"), suite.WithSharedEnvironment(), suite.WithTestCaseWhitelist("TestExpectedToRun"), suite.WithTestCaseRepeatCount(2),