From c58d30b419b300a5af09307878f2d5b4a8937fab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Tue, 19 Nov 2024 11:08:30 +0100 Subject: [PATCH 01/60] PMM-5086-12634 Structure draft. --- managed/models/agent_model.go | 67 ++++++++++++++++++++++------------- 1 file changed, 42 insertions(+), 25 deletions(-) diff --git a/managed/models/agent_model.go b/managed/models/agent_model.go index a7f1822804..4760aa9dbc 100644 --- a/managed/models/agent_model.go +++ b/managed/models/agent_model.go @@ -83,6 +83,15 @@ type MySQLOptions struct { TLSCa string `json:"tls_ca"` TLSCert string `json:"tls_cert"` TLSKey string `json:"tls_key"` + + // TableCount stores last known table count. NULL if unknown. + TableCount *int32 `reform:"table_count"` + + // Tablestats group collectors are disabled if there are more than that number of tables. + // 0 means tablestats group collectors are always enabled (no limit). + // Negative value means tablestats group collectors are always disabled. + // See IsMySQLTablestatsGroupEnabled method. + TableCountTablestatsGroupLimit int32 `reform:"table_count_tablestats_group_limit"` } // Value implements database/sql/driver.Valuer interface. Should be defined on the value. @@ -172,41 +181,49 @@ type Agent struct { TLS bool `reform:"tls"` TLSSkipVerify bool `reform:"tls_skip_verify"` - AWSAccessKey *string `reform:"aws_access_key"` - AWSSecretKey *string `reform:"aws_secret_key"` + LogLevel *string `reform:"log_level"` - AzureOptions *AzureOptions `reform:"azure_options"` + ExporterOptions *ExporterOptions `reform:"exporter_options"` + QANOptions *QANOptions `reform:"qan_options"` - // TableCount stores last known table count. NULL if unknown. - TableCount *int32 `reform:"table_count"` + AWSOptions *AWSOptions `reform:"aws_options"` + RDSOptions *RDSOptions `reform:"rds_options"` + AzureOptions *AzureOptions `reform:"azure_options"` + MySQLOptions *MySQLOptions `reform:"mysql_options"` + MongoDBOptions *MongoDBOptions `reform:"mongo_db_tls_options"` + PostgreSQLOptions *PostgreSQLOptions `reform:"postgresql_options"` +} - // Tablestats group collectors are disabled if there are more than that number of tables. - // 0 means tablestats group collectors are always enabled (no limit). - // Negative value means tablestats group collectors are always disabled. - // See IsMySQLTablestatsGroupEnabled method. - TableCountTablestatsGroupLimit int32 `reform:"table_count_tablestats_group_limit"` +type QANOptions struct { + MaxQueryLength int32 `reform:"max_query_length"` + QueryExamplesDisabled bool `reform:"query_examples_disabled"` + CommentsParsingDisabled bool `reform:"comments_parsing_disabled"` + MaxQueryLogSize int64 `reform:"max_query_log_size"` +} - MaxQueryLength int32 `reform:"max_query_length"` - QueryExamplesDisabled bool `reform:"query_examples_disabled"` - CommentsParsingDisabled bool `reform:"comments_parsing_disabled"` - MaxQueryLogSize int64 `reform:"max_query_log_size"` - MetricsPath *string `reform:"metrics_path"` - MetricsScheme *string `reform:"metrics_scheme"` +type ExporterOptions struct { + MetricsPath *string `reform:"metrics_path"` + MetricsScheme *string `reform:"metrics_scheme"` - RDSBasicMetricsDisabled bool `reform:"rds_basic_metrics_disabled"` - RDSEnhancedMetricsDisabled bool `reform:"rds_enhanced_metrics_disabled"` - PushMetrics bool `reform:"push_metrics"` - DisabledCollectors pq.StringArray `reform:"disabled_collectors"` - MetricsResolutions *MetricsResolutions `reform:"metrics_resolutions"` + PushMetrics bool `reform:"push_metrics"` - MySQLOptions *MySQLOptions `reform:"mysql_options"` - MongoDBOptions *MongoDBOptions `reform:"mongo_db_tls_options"` - PostgreSQLOptions *PostgreSQLOptions `reform:"postgresql_options"` - LogLevel *string `reform:"log_level"` + DisabledCollectors pq.StringArray `reform:"disabled_collectors"` + MetricsResolutions *MetricsResolutions `reform:"metrics_resolutions"` ExposeExporter bool `reform:"expose_exporter"` } +// AWSOptions represents structure for special AWS options. +type AWSOptions struct { + AWSAccessKey *string `reform:"aws_access_key"` + AWSSecretKey *string `reform:"aws_secret_key"` +} + +type RDSOptions struct { + RDSBasicMetricsDisabled bool `reform:"rds_basic_metrics_disabled"` + RDSEnhancedMetricsDisabled bool `reform:"rds_enhanced_metrics_disabled"` +} + // BeforeInsert implements reform.BeforeInserter interface. func (s *Agent) BeforeInsert() error { now := Now() From 3b4bba39a7612592f048f75dcc74ff775eb166f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Wed, 20 Nov 2024 11:19:34 +0100 Subject: [PATCH 02/60] PMM-5086-12634 Some changes. --- managed/models/agent_helpers.go | 163 ++++++++++++------------- managed/models/agent_model.go | 162 ++++++++++++++---------- managed/models/agent_model_test.go | 114 +++++++++-------- managed/models/database.go | 24 ++-- managed/models/encryption_helpers.go | 144 +++++++++++++--------- managed/services/management/mongodb.go | 26 ++-- 6 files changed, 355 insertions(+), 278 deletions(-) diff --git a/managed/models/agent_helpers.go b/managed/models/agent_helpers.go index 4be799dee7..91712efe7e 100644 --- a/managed/models/agent_helpers.go +++ b/managed/models/agent_helpers.go @@ -28,6 +28,7 @@ import ( "google.golang.org/grpc/status" "gopkg.in/reform.v1" + "github.com/percona/pmm/managed/models" "github.com/percona/pmm/version" ) @@ -640,15 +641,17 @@ func CreateNodeExporter(q *reform.Querier, " it doesn't support it, minimum supported version=%q", pointer.GetString(pmmAgent.Version), PMMAgentWithPushMetricsSupport.String()) } row := &Agent{ - AgentID: id, - AgentType: NodeExporterType, - PMMAgentID: &pmmAgentID, - NodeID: pmmAgent.RunsOnNodeID, - PushMetrics: pushMetrics, - DisabledCollectors: disableCollectors, - AgentPassword: agentPassword, - LogLevel: pointer.ToStringOrNil(logLevel), - ExposeExporter: exposeExporter, + AgentID: id, + AgentType: NodeExporterType, + PMMAgentID: &pmmAgentID, + NodeID: pmmAgent.RunsOnNodeID, + AgentPassword: agentPassword, + LogLevel: pointer.ToStringOrNil(logLevel), + ExporterOptions: &models.ExporterOptions{ + ExposeExporter: exposeExporter, + PushMetrics: pushMetrics, + DisabledCollectors: disableCollectors, + }, } if err := row.SetCustomLabels(customLabels); err != nil { return nil, err @@ -725,17 +728,19 @@ func CreateExternalExporter(q *reform.Querier, params *CreateExternalExporterPar metricsPath = "/metrics" } row := &Agent{ - PMMAgentID: pmmAgentID, - AgentID: id, - AgentType: ExternalExporterType, - RunsOnNodeID: runsOnNodeID, - ServiceID: pointer.ToStringOrNil(params.ServiceID), - Username: pointer.ToStringOrNil(params.Username), - Password: pointer.ToStringOrNil(params.Password), - MetricsScheme: &scheme, - MetricsPath: &metricsPath, - ListenPort: pointer.ToUint16(uint16(params.ListenPort)), - PushMetrics: params.PushMetrics, + PMMAgentID: pmmAgentID, + AgentID: id, + AgentType: ExternalExporterType, + RunsOnNodeID: runsOnNodeID, + ServiceID: pointer.ToStringOrNil(params.ServiceID), + Username: pointer.ToStringOrNil(params.Username), + Password: pointer.ToStringOrNil(params.Password), + ListenPort: pointer.ToUint16(uint16(params.ListenPort)), + ExporterOptions: &models.ExporterOptions{ + PushMetrics: params.PushMetrics, + MetricsPath: &metricsPath, + MetricsScheme: &scheme, + }, } if err := row.SetCustomLabels(params.CustomLabels); err != nil { return nil, err @@ -752,33 +757,24 @@ func CreateExternalExporter(q *reform.Querier, params *CreateExternalExporterPar // CreateAgentParams params for add common exporter. type CreateAgentParams struct { - PMMAgentID string - NodeID string - ServiceID string - Username string - Password string - AgentPassword string - CustomLabels map[string]string - TLS bool - TLSSkipVerify bool - MySQLOptions *MySQLOptions - MongoDBOptions *MongoDBOptions - PostgreSQLOptions *PostgreSQLOptions - TableCountTablestatsGroupLimit int32 - MaxQueryLength int32 - QueryExamplesDisabled bool - CommentsParsingDisabled bool - MaxQueryLogSize int64 - AWSAccessKey string - AWSSecretKey string - RDSBasicMetricsDisabled bool - RDSEnhancedMetricsDisabled bool - AzureOptions *AzureOptions - PushMetrics bool - ExposeExporter bool - DisableCollectors []string - LogLevel string - MetricsResolutions *MetricsResolutions + PMMAgentID string + NodeID string + ServiceID string + Username string + Password string + AgentPassword string + CustomLabels map[string]string + TLS bool + TLSSkipVerify bool + LogLevel string + ExporterOptions *ExporterOptions + QANOptions *QANOptions + AWSOptions *AWSOptions + AzureOptions *AzureOptions + MongoDBOptions *MongoDBOptions + MySQLOptions *MySQLOptions + PostgreSQLOptions *PostgreSQLOptions + RDSOptions *RDSOptions } func compatibleNodeAndAgent(nodeType NodeType, agentType AgentType) bool { @@ -871,7 +867,7 @@ func CreateAgent(q *reform.Querier, agentType AgentType, params *CreateAgentPara return nil, err } // check version for agent, if it exists. - if params.PushMetrics { + if params.ExporterOptions != nil && params.ExporterOptions.PushMetrics { // special case for vmAgent, it always supports push metrics. if agentType != VMAgentType && !IsPushMetricsSupported(pmmAgent.Version) { return nil, status.Errorf(codes.FailedPrecondition, "cannot use push_metrics_enabled with pmm_agent version=%q,"+ @@ -902,33 +898,25 @@ func CreateAgent(q *reform.Querier, agentType AgentType, params *CreateAgentPara } row := &Agent{ - AgentID: id, - AgentType: agentType, - PMMAgentID: ¶ms.PMMAgentID, - ServiceID: pointer.ToStringOrNil(params.ServiceID), - NodeID: pointer.ToStringOrNil(params.NodeID), - Username: pointer.ToStringOrNil(params.Username), - Password: pointer.ToStringOrNil(params.Password), - AgentPassword: pointer.ToStringOrNil(params.AgentPassword), - TLS: params.TLS, - TLSSkipVerify: params.TLSSkipVerify, - MySQLOptions: params.MySQLOptions, - MongoDBOptions: params.MongoDBOptions, - PostgreSQLOptions: params.PostgreSQLOptions, - TableCountTablestatsGroupLimit: params.TableCountTablestatsGroupLimit, - MaxQueryLength: params.MaxQueryLength, - QueryExamplesDisabled: params.QueryExamplesDisabled, - CommentsParsingDisabled: params.CommentsParsingDisabled, - MaxQueryLogSize: params.MaxQueryLogSize, - AWSAccessKey: pointer.ToStringOrNil(params.AWSAccessKey), - AWSSecretKey: pointer.ToStringOrNil(params.AWSSecretKey), - RDSBasicMetricsDisabled: params.RDSBasicMetricsDisabled, - RDSEnhancedMetricsDisabled: params.RDSEnhancedMetricsDisabled, - AzureOptions: params.AzureOptions, - PushMetrics: params.PushMetrics, - ExposeExporter: params.ExposeExporter, - DisabledCollectors: params.DisableCollectors, - LogLevel: pointer.ToStringOrNil(params.LogLevel), + AgentID: id, + AgentType: agentType, + PMMAgentID: ¶ms.PMMAgentID, + ServiceID: pointer.ToStringOrNil(params.ServiceID), + NodeID: pointer.ToStringOrNil(params.NodeID), + Username: pointer.ToStringOrNil(params.Username), + Password: pointer.ToStringOrNil(params.Password), + AgentPassword: pointer.ToStringOrNil(params.AgentPassword), + TLS: params.TLS, + TLSSkipVerify: params.TLSSkipVerify, + LogLevel: pointer.ToStringOrNil(params.LogLevel), + ExporterOptions: params.ExporterOptions, + QANOptions: params.QANOptions, + AWSOptions: params.AWSOptions, + AzureOptions: params.AzureOptions, + MongoDBOptions: params.MongoDBOptions, + MySQLOptions: params.MySQLOptions, + PostgreSQLOptions: params.PostgreSQLOptions, + RDSOptions: params.RDSOptions, } if err := row.SetCustomLabels(params.CustomLabels); err != nil { return nil, err @@ -970,7 +958,9 @@ func ChangeAgent(q *reform.Querier, agentID string, params *ChangeCommonAgentPar } if params.EnablePushMetrics != nil { - row.PushMetrics = *params.EnablePushMetrics + row.ExporterOptions = &models.ExporterOptions{ + PushMetrics: *params.EnablePushMetrics, + } if row.AgentType == ExternalExporterType { if err := updateExternalExporterParams(q, row); err != nil { return nil, errors.Wrap(err, "failed to update External exporterParams for PushMetrics") @@ -990,22 +980,25 @@ func ChangeAgent(q *reform.Querier, agentID string, params *ChangeCommonAgentPar } } - if row.MetricsResolutions == nil { - row.MetricsResolutions = &MetricsResolutions{} + if row.ExporterOptions == nil { + row.ExporterOptions = &ExporterOptions{} + } + if row.ExporterOptions.MetricsResolutions == nil { + row.ExporterOptions.MetricsResolutions = &MetricsResolutions{} } if params.MetricsResolutions.LR != nil { - row.MetricsResolutions.LR = *params.MetricsResolutions.LR + row.ExporterOptions.MetricsResolutions.LR = *params.MetricsResolutions.LR } if params.MetricsResolutions.MR != nil { - row.MetricsResolutions.MR = *params.MetricsResolutions.MR + row.ExporterOptions.MetricsResolutions.MR = *params.MetricsResolutions.MR } if params.MetricsResolutions.HR != nil { - row.MetricsResolutions.HR = *params.MetricsResolutions.HR + row.ExporterOptions.MetricsResolutions.HR = *params.MetricsResolutions.HR } // If all resolutions are empty, then drop whole MetricsResolution field. - if row.MetricsResolutions.HR == 0 && row.MetricsResolutions.MR == 0 && row.MetricsResolutions.LR == 0 { - row.MetricsResolutions = nil + if row.ExporterOptions.MetricsResolutions.HR == 0 && row.ExporterOptions.MetricsResolutions.MR == 0 && row.ExporterOptions.MetricsResolutions.LR == 0 { + row.ExporterOptions.MetricsResolutions = nil } if err = q.Update(row); err != nil { @@ -1057,7 +1050,7 @@ func RemoveAgent(q *reform.Querier, id string, mode RemoveMode) (*Agent, error) // for external exporter, is needed for push_metrics mode. func updateExternalExporterParams(q *reform.Querier, row *Agent) error { // with push metrics, external exporter must have PMMAgent id without RunsOnNodeID - if row.PushMetrics && row.PMMAgentID == nil { + if row.ExporterOptions != nil && row.ExporterOptions.PushMetrics && row.PMMAgentID == nil { pmmAgent, err := FindPMMAgentsRunningOnNode(q, pointer.GetString(row.RunsOnNodeID)) if err != nil { return err @@ -1074,7 +1067,7 @@ func updateExternalExporterParams(q *reform.Querier, row *Agent) error { row.PMMAgentID = pointer.ToString(pmmAgent[0].AgentID) } // without push metrics, external exporter must have RunsOnNodeID without PMMAgentID - if !row.PushMetrics && row.RunsOnNodeID == nil { + if row.ExporterOptions != nil && !row.ExporterOptions.PushMetrics && row.RunsOnNodeID == nil { pmmAgent, err := FindAgentByID(q, pointer.GetString(row.PMMAgentID)) if err != nil { return err diff --git a/managed/models/agent_model.go b/managed/models/agent_model.go index 4760aa9dbc..2e34099f3a 100644 --- a/managed/models/agent_model.go +++ b/managed/models/agent_model.go @@ -78,27 +78,62 @@ var v2_42 = version.MustParse("2.42.0-0") // PMMServerAgentID is a special Agent ID representing pmm-agent on PMM Server. const PMMServerAgentID = string("pmm-server") // a special ID, reserved for PMM Server -// MySQLOptions represents structure for special MySQL options. -type MySQLOptions struct { - TLSCa string `json:"tls_ca"` - TLSCert string `json:"tls_cert"` - TLSKey string `json:"tls_key"` +// ExporterOptions represents structure for special Exporter options. +type ExporterOptions struct { + ExposeExporter bool `reform:"expose_exporter"` + PushMetrics bool `reform:"push_metrics"` + DisabledCollectors pq.StringArray `reform:"disabled_collectors"` + MetricsResolutions *MetricsResolutions `reform:"metrics_resolutions"` + MetricsPath string `reform:"metrics_path"` + MetricsScheme string `reform:"metrics_scheme"` +} - // TableCount stores last known table count. NULL if unknown. - TableCount *int32 `reform:"table_count"` +// Value implements database/sql/driver.Valuer interface. Should be defined on the value. +func (c ExporterOptions) Value() (driver.Value, error) { return jsonValue(c) } - // Tablestats group collectors are disabled if there are more than that number of tables. - // 0 means tablestats group collectors are always enabled (no limit). - // Negative value means tablestats group collectors are always disabled. - // See IsMySQLTablestatsGroupEnabled method. - TableCountTablestatsGroupLimit int32 `reform:"table_count_tablestats_group_limit"` +// Scan implements database/sql.Scanner interface. Should be defined on the pointer. +func (c *ExporterOptions) Scan(src interface{}) error { return jsonScan(c, src) } + +// QANOptions represents structure for special QAN options. +type QANOptions struct { + MaxQueryLength int32 `reform:"max_query_length"` + MaxQueryLogSize int64 `reform:"max_query_log_size"` + QueryExamplesDisabled bool `reform:"query_examples_disabled"` + CommentsParsingDisabled bool `reform:"comments_parsing_disabled"` } // Value implements database/sql/driver.Valuer interface. Should be defined on the value. -func (c MySQLOptions) Value() (driver.Value, error) { return jsonValue(c) } +func (c QANOptions) Value() (driver.Value, error) { return jsonValue(c) } // Scan implements database/sql.Scanner interface. Should be defined on the pointer. -func (c *MySQLOptions) Scan(src interface{}) error { return jsonScan(c, src) } +func (c *QANOptions) Scan(src interface{}) error { return jsonScan(c, src) } + +// AWSOptions represents structure for special AWS options. +type AWSOptions struct { + AWSAccessKey string `reform:"aws_access_key"` + AWSSecretKey string `reform:"aws_secret_key"` +} + +// Value implements database/sql/driver.Valuer interface. Should be defined on the value. +func (c AWSOptions) Value() (driver.Value, error) { return jsonValue(c) } + +// Scan implements database/sql.Scanner interface. Should be defined on the pointer. +func (c *AWSOptions) Scan(src interface{}) error { return jsonScan(c, src) } + +// AzureOptions represents structure for special Azure options. +type AzureOptions struct { + SubscriptionID string `json:"subscription_id"` + ClientID string `json:"client_id"` + ClientSecret string `json:"client_secret"` + TenantID string `json:"tenant_id"` + ResourceGroup string `json:"resource_group"` +} + +// Value implements database/sql/driver.Valuer interface. Should be defined on the value. +func (c AzureOptions) Value() (driver.Value, error) { return jsonValue(c) } + +// Scan implements database/sql.Scanner interface. Should be defined on the pointer. +func (c *AzureOptions) Scan(src interface{}) error { return jsonScan(c, src) } // MongoDBOptions represents structure for special MongoDB options. type MongoDBOptions struct { @@ -118,20 +153,27 @@ func (c MongoDBOptions) Value() (driver.Value, error) { return jsonValue(c) } // Scan implements database/sql.Scanner interface. Should be defined on the pointer. func (c *MongoDBOptions) Scan(src interface{}) error { return jsonScan(c, src) } -// AzureOptions represents structure for special Azure options. -type AzureOptions struct { - SubscriptionID string `json:"subscription_id"` - ClientID string `json:"client_id"` - ClientSecret string `json:"client_secret"` - TenantID string `json:"tenant_id"` - ResourceGroup string `json:"resource_group"` +// MySQLOptions represents structure for special MySQL options. +type MySQLOptions struct { + TLSCa string `json:"tls_ca"` + TLSCert string `json:"tls_cert"` + TLSKey string `json:"tls_key"` + + // TableCount stores last known table count. NULL if unknown. + TableCount *int32 `reform:"table_count"` + + // Tablestats group collectors are disabled if there are more than that number of tables. + // 0 means tablestats group collectors are always enabled (no limit). + // Negative value means tablestats group collectors are always disabled. + // See IsMySQLTablestatsGroupEnabled method. + TableCountTablestatsGroupLimit int32 `reform:"table_count_tablestats_group_limit"` } // Value implements database/sql/driver.Valuer interface. Should be defined on the value. -func (c AzureOptions) Value() (driver.Value, error) { return jsonValue(c) } +func (c MySQLOptions) Value() (driver.Value, error) { return jsonValue(c) } // Scan implements database/sql.Scanner interface. Should be defined on the pointer. -func (c *AzureOptions) Scan(src interface{}) error { return jsonScan(c, src) } +func (c *MySQLOptions) Scan(src interface{}) error { return jsonScan(c, src) } // PostgreSQLOptions represents structure for special PostgreSQL options. type PostgreSQLOptions struct { @@ -150,6 +192,18 @@ func (c PostgreSQLOptions) Value() (driver.Value, error) { return jsonValue(c) } // Scan implements database/sql.Scanner interface. Should be defined on the pointer. func (c *PostgreSQLOptions) Scan(src interface{}) error { return jsonScan(c, src) } +// RDSOptions represents structure for special RDSOptions options. +type RDSOptions struct { + RDSBasicMetricsDisabled bool `reform:"rds_basic_metrics_disabled"` + RDSEnhancedMetricsDisabled bool `reform:"rds_enhanced_metrics_disabled"` +} + +// Value implements database/sql/driver.Valuer interface. Should be defined on the value. +func (c RDSOptions) Value() (driver.Value, error) { return jsonValue(c) } + +// Scan implements database/sql.Scanner interface. Should be defined on the pointer. +func (c *RDSOptions) Scan(src interface{}) error { return jsonScan(c, src) } + // PMMAgentWithPushMetricsSupport - version of pmmAgent, // that support vmagent and push metrics mode // will be released with PMM Agent v2.12. @@ -187,41 +241,11 @@ type Agent struct { QANOptions *QANOptions `reform:"qan_options"` AWSOptions *AWSOptions `reform:"aws_options"` - RDSOptions *RDSOptions `reform:"rds_options"` AzureOptions *AzureOptions `reform:"azure_options"` - MySQLOptions *MySQLOptions `reform:"mysql_options"` MongoDBOptions *MongoDBOptions `reform:"mongo_db_tls_options"` + MySQLOptions *MySQLOptions `reform:"mysql_options"` PostgreSQLOptions *PostgreSQLOptions `reform:"postgresql_options"` -} - -type QANOptions struct { - MaxQueryLength int32 `reform:"max_query_length"` - QueryExamplesDisabled bool `reform:"query_examples_disabled"` - CommentsParsingDisabled bool `reform:"comments_parsing_disabled"` - MaxQueryLogSize int64 `reform:"max_query_log_size"` -} - -type ExporterOptions struct { - MetricsPath *string `reform:"metrics_path"` - MetricsScheme *string `reform:"metrics_scheme"` - - PushMetrics bool `reform:"push_metrics"` - - DisabledCollectors pq.StringArray `reform:"disabled_collectors"` - MetricsResolutions *MetricsResolutions `reform:"metrics_resolutions"` - - ExposeExporter bool `reform:"expose_exporter"` -} - -// AWSOptions represents structure for special AWS options. -type AWSOptions struct { - AWSAccessKey *string `reform:"aws_access_key"` - AWSSecretKey *string `reform:"aws_secret_key"` -} - -type RDSOptions struct { - RDSBasicMetricsDisabled bool `reform:"rds_basic_metrics_disabled"` - RDSEnhancedMetricsDisabled bool `reform:"rds_enhanced_metrics_disabled"` + RDSOptions *RDSOptions `reform:"rds_options"` } // BeforeInsert implements reform.BeforeInserter interface. @@ -580,14 +604,18 @@ func (s *Agent) DSN(service *Service, dsnParams DSNParams, tdp *DelimiterPair, p // ExporterURL composes URL to an external exporter. func (s *Agent) ExporterURL(q *reform.Querier) (string, error) { - scheme := pointer.GetString(s.MetricsScheme) - path := pointer.GetString(s.MetricsPath) + if s.ExporterOptions == nil { + s.ExporterOptions = &ExporterOptions{} + } + + scheme := s.ExporterOptions.MetricsScheme + path := s.ExporterOptions.MetricsPath listenPort := int(pointer.GetUint16(s.ListenPort)) username := pointer.GetString(s.Username) password := pointer.GetString(s.Password) host := "127.0.0.1" - if !s.PushMetrics { + if !s.ExporterOptions.PushMetrics { node, err := FindNodeByID(q, *s.RunsOnNodeID) if err != nil { return "", err @@ -625,15 +653,19 @@ func (s *Agent) IsMySQLTablestatsGroupEnabled() bool { panic(fmt.Errorf("unhandled AgentType %q", s.AgentType)) } + if s.MySQLOptions == nil { + s.MySQLOptions = &MySQLOptions{} + } + switch { - case s.TableCountTablestatsGroupLimit == 0: // server defined + case s.MySQLOptions.TableCountTablestatsGroupLimit == 0: // server defined return true - case s.TableCountTablestatsGroupLimit < 0: // always disabled + case s.MySQLOptions.TableCountTablestatsGroupLimit < 0: // always disabled return false - case s.TableCount == nil: // for compatibility with 2.0 + case s.MySQLOptions.TableCount == nil: // for compatibility with 2.0 return true default: - return *s.TableCount <= s.TableCountTablestatsGroupLimit + return *s.MySQLOptions.TableCount <= s.MySQLOptions.TableCountTablestatsGroupLimit } } @@ -692,11 +724,15 @@ func (s Agent) Files() map[string]string { // TemplateDelimiters returns a pair of safe template delimiters that are not present in agent parameters. func (s Agent) TemplateDelimiters(svc *Service) *DelimiterPair { + if s.ExporterOptions == nil { + s.ExporterOptions = &ExporterOptions{} + } + templateParams := []string{ pointer.GetString(svc.Address), pointer.GetString(s.Username), pointer.GetString(s.Password), - pointer.GetString(s.MetricsPath), + s.ExporterOptions.MetricsPath, } switch svc.ServiceType { diff --git a/managed/models/agent_model_test.go b/managed/models/agent_model_test.go index 4fb68a0a9f..c06c5e13a8 100644 --- a/managed/models/agent_model_test.go +++ b/managed/models/agent_model_test.go @@ -364,9 +364,11 @@ func TestIsMySQLTablestatsGroupEnabled(t *testing.T) { } t.Run(fmt.Sprintf("Count:%s/Limit:%d", c, testCase.limit), func(t *testing.T) { agent := &models.Agent{ - AgentType: models.MySQLdExporterType, - TableCount: testCase.count, - TableCountTablestatsGroupLimit: testCase.limit, + AgentType: models.MySQLdExporterType, + MySQLOptions: &models.MySQLOptions{ + TableCount: testCase.count, + TableCountTablestatsGroupLimit: testCase.limit, + }, } assert.Equal(t, testCase.expected, agent.IsMySQLTablestatsGroupEnabled()) }) @@ -439,66 +441,76 @@ func TestExporterURL(t *testing.T) { }, &models.Agent{ - AgentID: "ExporterAgentPush", - AgentType: models.ExternalExporterType, - ServiceID: pointer.ToString("external"), - RunsOnNodeID: pointer.ToString("ExporterNodeID"), - MetricsScheme: pointer.ToString("http"), - PushMetrics: true, - ListenPort: pointer.ToUint16(9121), - MetricsPath: pointer.ToString("/metrics"), + AgentID: "ExporterAgentPush", + AgentType: models.ExternalExporterType, + ServiceID: pointer.ToString("external"), + RunsOnNodeID: pointer.ToString("ExporterNodeID"), + ListenPort: pointer.ToUint16(9121), + ExporterOptions: &models.ExporterOptions{ + PushMetrics: true, + MetricsPath: "/metrics", + MetricsScheme: "http", + }, }, &models.Agent{ - AgentID: "ExporterAgentPull", - AgentType: models.ExternalExporterType, - ServiceID: pointer.ToString("external"), - RunsOnNodeID: pointer.ToString("ExporterNodeID"), - MetricsScheme: pointer.ToString("http"), - PushMetrics: false, - ListenPort: pointer.ToUint16(9121), - MetricsPath: pointer.ToString("/metrics"), - Username: pointer.ToString("user"), - Password: pointer.ToString("secret"), + AgentID: "ExporterAgentPull", + AgentType: models.ExternalExporterType, + ServiceID: pointer.ToString("external"), + RunsOnNodeID: pointer.ToString("ExporterNodeID"), + ListenPort: pointer.ToUint16(9121), + Username: pointer.ToString("user"), + Password: pointer.ToString("secret"), + ExporterOptions: &models.ExporterOptions{ + PushMetrics: false, + MetricsPath: "/metrics", + MetricsScheme: "http", + }, }, &models.Agent{ - AgentID: "ExporterServerless", - AgentType: models.ExternalExporterType, - RunsOnNodeID: pointer.ToString("ExporterServerlessNodeID"), - ServiceID: pointer.ToString("redis_exporter-external"), - MetricsScheme: pointer.ToString("http"), - PushMetrics: false, - ListenPort: pointer.ToUint16(9121), - MetricsPath: pointer.ToString("/metrics"), - Username: pointer.ToString("user"), - Password: pointer.ToString("secret"), + AgentID: "ExporterServerless", + AgentType: models.ExternalExporterType, + RunsOnNodeID: pointer.ToString("ExporterServerlessNodeID"), + ServiceID: pointer.ToString("redis_exporter-external"), + ListenPort: pointer.ToUint16(9121), + Username: pointer.ToString("user"), + Password: pointer.ToString("secret"), + ExporterOptions: &models.ExporterOptions{ + PushMetrics: false, + MetricsPath: "/metrics", + MetricsScheme: "http", + }, }, &models.Agent{ - AgentID: "ExporterServerlessWithQueryParams", - AgentType: models.ExternalExporterType, - RunsOnNodeID: pointer.ToString("ExporterServerlessNodeID2"), - ServiceID: pointer.ToString("nomad_exporter-external"), - MetricsScheme: pointer.ToString("http"), - PushMetrics: false, - ListenPort: pointer.ToUint16(9121), - MetricsPath: pointer.ToString("/metrics?format=prometheus&output=json"), - Username: pointer.ToString("user"), - Password: pointer.ToString("secret"), + AgentID: "ExporterServerlessWithQueryParams", + AgentType: models.ExternalExporterType, + RunsOnNodeID: pointer.ToString("ExporterServerlessNodeID2"), + ServiceID: pointer.ToString("nomad_exporter-external"), + ListenPort: pointer.ToUint16(9121), + Username: pointer.ToString("user"), + Password: pointer.ToString("secret"), + ExporterOptions: &models.ExporterOptions{ + PushMetrics: false, + MetricsPath: "/metrics?format=prometheus&output=json", + MetricsScheme: "http", + }, }, &models.Agent{ - AgentID: "ExporterServerlessWithEmptyMetricsPath", - AgentType: models.ExternalExporterType, - RunsOnNodeID: pointer.ToString("ExporterServerlessNodeID2"), - ServiceID: pointer.ToString("nomad_exporter-external"), - MetricsScheme: pointer.ToString("http"), - PushMetrics: false, - ListenPort: pointer.ToUint16(9121), - MetricsPath: pointer.ToString("/"), - Username: pointer.ToString("user"), - Password: pointer.ToString("secret"), + AgentID: "ExporterServerlessWithEmptyMetricsPath", + AgentType: models.ExternalExporterType, + RunsOnNodeID: pointer.ToString("ExporterServerlessNodeID2"), + ServiceID: pointer.ToString("nomad_exporter-external"), + ListenPort: pointer.ToUint16(9121), + Username: pointer.ToString("user"), + Password: pointer.ToString("secret"), + ExporterOptions: &models.ExporterOptions{ + PushMetrics: false, + MetricsPath: "/", + MetricsScheme: "http", + }, }, } { require.NoError(t, q.Insert(str), "failed to INSERT %+v", str) diff --git a/managed/models/database.go b/managed/models/database.go index d7a9778721..d3608da193 100644 --- a/managed/models/database.go +++ b/managed/models/database.go @@ -37,6 +37,7 @@ import ( "gopkg.in/reform.v1" "gopkg.in/reform.v1/dialects/postgresql" + "github.com/percona/pmm/managed/models" "github.com/percona/pmm/managed/utils/encryption" ) @@ -68,13 +69,12 @@ var DefaultAgentEncryptionColumns = []encryption.Table{ Columns: []encryption.Column{ {Name: "username"}, {Name: "password"}, - {Name: "aws_access_key"}, - {Name: "aws_secret_key"}, - {Name: "mongo_db_tls_options", CustomHandler: EncryptMongoDBOptionsHandler}, + {Name: "agent_password"}, + {Name: "aws_options", CustomHandler: EncryptAWSOptionsHandler}, {Name: "azure_options", CustomHandler: EncryptAzureOptionsHandler}, + {Name: "mongo_db_tls_options", CustomHandler: EncryptMongoDBOptionsHandler}, {Name: "mysql_options", CustomHandler: EncryptMySQLOptionsHandler}, {Name: "postgresql_options", CustomHandler: EncryptPostgreSQLOptionsHandler}, - {Name: "agent_password"}, }, }, } @@ -1414,13 +1414,15 @@ func setupPMMServerAgents(q *reform.Querier, params SetupDBParams) error { } ap := &CreateAgentParams{ - PMMAgentID: PMMServerAgentID, - ServiceID: service.ServiceID, - TLS: params.SSLMode != DisableSSLMode, - TLSSkipVerify: params.SSLMode == DisableSSLMode || params.SSLMode == VerifyCaSSLMode, - CommentsParsingDisabled: true, - Username: params.Username, - Password: params.Password, + PMMAgentID: PMMServerAgentID, + ServiceID: service.ServiceID, + TLS: params.SSLMode != DisableSSLMode, + TLSSkipVerify: params.SSLMode == DisableSSLMode || params.SSLMode == VerifyCaSSLMode, + Username: params.Username, + Password: params.Password, + QANOptions: &models.QANOptions{ + CommentsParsingDisabled: true, + }, } if ap.TLS { ap.PostgreSQLOptions = &PostgreSQLOptions{} diff --git a/managed/models/encryption_helpers.go b/managed/models/encryption_helpers.go index f3ae2e01a8..ccda8b75a0 100644 --- a/managed/models/encryption_helpers.go +++ b/managed/models/encryption_helpers.go @@ -59,40 +59,33 @@ func agentEncryption(agent Agent, handler func(string) (string, error)) Agent { agent.AgentPassword = &agentPassword } - if agent.AWSAccessKey != nil { - awsAccessKey, err := handler(*agent.AWSAccessKey) + var err error + if agent.AWSOptions != nil { + agent.AWSOptions.AWSAccessKey, err = handler(agent.AWSOptions.AWSAccessKey) if err != nil { logrus.Warning(err) } - agent.AWSAccessKey = &awsAccessKey - } - if agent.AWSSecretKey != nil { - awsSecretKey, err := handler(*agent.AWSSecretKey) + agent.AWSOptions.AWSSecretKey, err = handler(agent.AWSOptions.AWSSecretKey) if err != nil { logrus.Warning(err) } - agent.AWSSecretKey = &awsSecretKey } - var err error - if agent.MySQLOptions != nil { - agent.MySQLOptions.TLSCert, err = handler(agent.MySQLOptions.TLSCert) + if agent.AzureOptions != nil { + agent.AzureOptions.ClientID, err = handler(agent.AzureOptions.ClientID) if err != nil { logrus.Warning(err) } - agent.MySQLOptions.TLSKey, err = handler(agent.MySQLOptions.TLSKey) + agent.AzureOptions.ClientSecret, err = handler(agent.AzureOptions.ClientSecret) if err != nil { logrus.Warning(err) } - } - - if agent.PostgreSQLOptions != nil { - agent.PostgreSQLOptions.SSLCert, err = handler(agent.PostgreSQLOptions.SSLCert) + agent.AzureOptions.SubscriptionID, err = handler(agent.AzureOptions.SubscriptionID) if err != nil { logrus.Warning(err) } - agent.PostgreSQLOptions.SSLKey, err = handler(agent.PostgreSQLOptions.SSLKey) + agent.AzureOptions.TenantID, err = handler(agent.AzureOptions.TenantID) if err != nil { logrus.Warning(err) } @@ -109,20 +102,23 @@ func agentEncryption(agent Agent, handler func(string) (string, error)) Agent { } } - if agent.AzureOptions != nil { - agent.AzureOptions.ClientID, err = handler(agent.AzureOptions.ClientID) + if agent.MySQLOptions != nil { + agent.MySQLOptions.TLSCert, err = handler(agent.MySQLOptions.TLSCert) if err != nil { logrus.Warning(err) } - agent.AzureOptions.ClientSecret, err = handler(agent.AzureOptions.ClientSecret) + agent.MySQLOptions.TLSKey, err = handler(agent.MySQLOptions.TLSKey) if err != nil { logrus.Warning(err) } - agent.AzureOptions.SubscriptionID, err = handler(agent.AzureOptions.SubscriptionID) + } + + if agent.PostgreSQLOptions != nil { + agent.PostgreSQLOptions.SSLCert, err = handler(agent.PostgreSQLOptions.SSLCert) if err != nil { logrus.Warning(err) } - agent.AzureOptions.TenantID, err = handler(agent.AzureOptions.TenantID) + agent.PostgreSQLOptions.SSLKey, err = handler(agent.PostgreSQLOptions.SSLKey) if err != nil { logrus.Warning(err) } @@ -131,18 +127,18 @@ func agentEncryption(agent Agent, handler func(string) (string, error)) Agent { return agent } -// EncryptMySQLOptionsHandler returns encrypted MySQL Options. -func EncryptMySQLOptionsHandler(e *encryption.Encryption, val any) (any, error) { - return mySQLOptionsHandler(val, e.Encrypt) +// EncryptAWSOptionsHandler returns encrypted AWS Options. +func EncryptAWSOptionsHandler(e *encryption.Encryption, val any) (any, error) { + return azureOptionsHandler(val, e.Encrypt) } -// DecryptMySQLOptionsHandler returns decrypted MySQL Options. -func DecryptMySQLOptionsHandler(e *encryption.Encryption, val any) (any, error) { - return mySQLOptionsHandler(val, e.Decrypt) +// DecryptAWSOptionsHandler returns decrypted AWS Options. +func DecryptAWSOptionsHandler(e *encryption.Encryption, val any) (any, error) { + return azureOptionsHandler(val, e.Decrypt) } -func mySQLOptionsHandler(val any, handler func(string) (string, error)) (any, error) { - o := MySQLOptions{} +func awsOptionsHandler(val any, handler func(string) (string, error)) (any, error) { + o := AWSOptions{} value := val.(*sql.NullString) //nolint:forcetypeassert if !value.Valid { return sql.NullString{}, nil @@ -153,14 +149,11 @@ func mySQLOptionsHandler(val any, handler func(string) (string, error)) (any, er return nil, err } - o.TLSCert, err = handler(o.TLSCert) - if err != nil { - return nil, err - } - o.TLSKey, err = handler(o.TLSKey) + o.AWSAccessKey, err = handler(o.AWSAccessKey) if err != nil { return nil, err } + o.AWSSecretKey, err = handler(o.AWSSecretKey) res, err := json.Marshal(o) if err != nil { @@ -170,18 +163,18 @@ func mySQLOptionsHandler(val any, handler func(string) (string, error)) (any, er return res, nil } -// EncryptPostgreSQLOptionsHandler returns encrypted PostgreSQL Options. -func EncryptPostgreSQLOptionsHandler(e *encryption.Encryption, val any) (any, error) { - return postgreSQLOptionsHandler(val, e.Encrypt) +// EncryptAzureOptionsHandler returns encrypted Azure Options. +func EncryptAzureOptionsHandler(e *encryption.Encryption, val any) (any, error) { + return azureOptionsHandler(val, e.Encrypt) } -// DecryptPostgreSQLOptionsHandler returns decrypted PostgreSQL Options. -func DecryptPostgreSQLOptionsHandler(e *encryption.Encryption, val any) (any, error) { - return postgreSQLOptionsHandler(val, e.Decrypt) +// DecryptAzureOptionsHandler returns decrypted Azure Options. +func DecryptAzureOptionsHandler(e *encryption.Encryption, val any) (any, error) { + return azureOptionsHandler(val, e.Decrypt) } -func postgreSQLOptionsHandler(val any, handler func(string) (string, error)) (any, error) { - o := PostgreSQLOptions{} +func azureOptionsHandler(val any, handler func(string) (string, error)) (any, error) { + o := AzureOptions{} value := val.(*sql.NullString) //nolint:forcetypeassert if !value.Valid { return sql.NullString{}, nil @@ -192,11 +185,19 @@ func postgreSQLOptionsHandler(val any, handler func(string) (string, error)) (an return nil, err } - o.SSLCert, err = handler(o.SSLCert) + o.ClientID, err = handler(o.ClientID) if err != nil { return nil, err } - o.SSLKey, err = handler(o.SSLKey) + o.ClientSecret, err = handler(o.ClientSecret) + if err != nil { + return nil, err + } + o.SubscriptionID, err = handler(o.SubscriptionID) + if err != nil { + return nil, err + } + o.TenantID, err = handler(o.TenantID) if err != nil { return nil, err } @@ -248,18 +249,18 @@ func mongoDBOptionsHandler(val any, handler func(string) (string, error)) (any, return res, nil } -// EncryptAzureOptionsHandler returns encrypted Azure Options. -func EncryptAzureOptionsHandler(e *encryption.Encryption, val any) (any, error) { - return azureOptionsHandler(val, e.Encrypt) +// EncryptMySQLOptionsHandler returns encrypted MySQL Options. +func EncryptMySQLOptionsHandler(e *encryption.Encryption, val any) (any, error) { + return mySQLOptionsHandler(val, e.Encrypt) } -// DecryptAzureOptionsHandler returns decrypted Azure Options. -func DecryptAzureOptionsHandler(e *encryption.Encryption, val any) (any, error) { - return azureOptionsHandler(val, e.Decrypt) +// DecryptMySQLOptionsHandler returns decrypted MySQL Options. +func DecryptMySQLOptionsHandler(e *encryption.Encryption, val any) (any, error) { + return mySQLOptionsHandler(val, e.Decrypt) } -func azureOptionsHandler(val any, handler func(string) (string, error)) (any, error) { - o := AzureOptions{} +func mySQLOptionsHandler(val any, handler func(string) (string, error)) (any, error) { + o := MySQLOptions{} value := val.(*sql.NullString) //nolint:forcetypeassert if !value.Valid { return sql.NullString{}, nil @@ -270,19 +271,50 @@ func azureOptionsHandler(val any, handler func(string) (string, error)) (any, er return nil, err } - o.ClientID, err = handler(o.ClientID) + o.TLSCert, err = handler(o.TLSCert) if err != nil { return nil, err } - o.ClientSecret, err = handler(o.ClientSecret) + o.TLSKey, err = handler(o.TLSKey) if err != nil { return nil, err } - o.SubscriptionID, err = handler(o.SubscriptionID) + + res, err := json.Marshal(o) if err != nil { return nil, err } - o.TenantID, err = handler(o.TenantID) + + return res, nil +} + +// EncryptPostgreSQLOptionsHandler returns encrypted PostgreSQL Options. +func EncryptPostgreSQLOptionsHandler(e *encryption.Encryption, val any) (any, error) { + return postgreSQLOptionsHandler(val, e.Encrypt) +} + +// DecryptPostgreSQLOptionsHandler returns decrypted PostgreSQL Options. +func DecryptPostgreSQLOptionsHandler(e *encryption.Encryption, val any) (any, error) { + return postgreSQLOptionsHandler(val, e.Decrypt) +} + +func postgreSQLOptionsHandler(val any, handler func(string) (string, error)) (any, error) { + o := PostgreSQLOptions{} + value := val.(*sql.NullString) //nolint:forcetypeassert + if !value.Valid { + return sql.NullString{}, nil + } + + err := json.Unmarshal([]byte(value.String), &o) + if err != nil { + return nil, err + } + + o.SSLCert, err = handler(o.SSLCert) + if err != nil { + return nil, err + } + o.SSLKey, err = handler(o.SSLKey) if err != nil { return nil, err } diff --git a/managed/services/management/mongodb.go b/managed/services/management/mongodb.go index fe7782c194..5498ba2c51 100644 --- a/managed/services/management/mongodb.go +++ b/managed/services/management/mongodb.go @@ -63,18 +63,20 @@ func (s *ManagementService) addMongoDB(ctx context.Context, req *managementv1.Ad } row, err := models.CreateAgent(tx.Querier, models.MongoDBExporterType, &models.CreateAgentParams{ - PMMAgentID: req.PmmAgentId, - ServiceID: service.ServiceID, - Username: req.Username, - Password: req.Password, - AgentPassword: req.AgentPassword, - TLS: req.Tls, - TLSSkipVerify: req.TlsSkipVerify, - MongoDBOptions: models.MongoDBOptionsFromRequest(req), - PushMetrics: isPushMode(req.MetricsMode), - ExposeExporter: req.ExposeExporter, - DisableCollectors: req.DisableCollectors, - LogLevel: services.SpecifyLogLevel(req.LogLevel, inventoryv1.LogLevel_LOG_LEVEL_FATAL), + PMMAgentID: req.PmmAgentId, + ServiceID: service.ServiceID, + Username: req.Username, + Password: req.Password, + AgentPassword: req.AgentPassword, + TLS: req.Tls, + TLSSkipVerify: req.TlsSkipVerify, + MongoDBOptions: models.MongoDBOptionsFromRequest(req), + LogLevel: services.SpecifyLogLevel(req.LogLevel, inventoryv1.LogLevel_LOG_LEVEL_FATAL), + ExporterOptions: &models.ExporterOptions{ + ExposeExporter: req.ExposeExporter, + PushMetrics: isPushMode(req.MetricsMode), + DisabledCollectors: req.DisableCollectors, + }, }) if err != nil { return err From 53511076e871253a9af655f0a0de682995d7392b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Wed, 20 Nov 2024 11:55:18 +0100 Subject: [PATCH 03/60] PMM-5086-12634 Another changes. --- managed/services/agents/agents.go | 31 +++++----- managed/services/agents/agents_test.go | 20 +++++-- managed/services/agents/mongodb.go | 6 +- managed/services/agents/mongodb_test.go | 8 ++- managed/services/management/mongodb.go | 18 +++--- managed/services/management/postgresql.go | 70 ++++++++++++----------- 6 files changed, 85 insertions(+), 68 deletions(-) diff --git a/managed/services/agents/agents.go b/managed/services/agents/agents.go index ea5ea813ad..bd06495740 100644 --- a/managed/services/agents/agents.go +++ b/managed/services/agents/agents.go @@ -101,29 +101,31 @@ func redactWords(agent *models.Agent) []string { if s := pointer.GetString(agent.AgentPassword); s != "" { words = append(words, s) } - if s := pointer.GetString(agent.AWSSecretKey); s != "" { - words = append(words, s) + if agent.AWSOptions != nil { + if s := agent.AWSOptions.AWSSecretKey; s != "" { + words = append(words, s) + } } if agent.AzureOptions != nil { if s := agent.AzureOptions.ClientSecret; s != "" { words = append(words, s) } } - if agent.MySQLOptions != nil { - if s := agent.MySQLOptions.TLSKey; s != "" { + if agent.MongoDBOptions != nil { + if s := agent.MongoDBOptions.TLSCertificateKey; s != "" { words = append(words, s) } - } - if agent.PostgreSQLOptions != nil { - if s := agent.PostgreSQLOptions.SSLKey; s != "" { + if s := agent.MongoDBOptions.TLSCertificateKeyFilePassword; s != "" { words = append(words, s) } } - if agent.MongoDBOptions != nil { - if s := agent.MongoDBOptions.TLSCertificateKey; s != "" { + if agent.MySQLOptions != nil { + if s := agent.MySQLOptions.TLSKey; s != "" { words = append(words, s) } - if s := agent.MongoDBOptions.TLSCertificateKeyFilePassword; s != "" { + } + if agent.PostgreSQLOptions != nil { + if s := agent.PostgreSQLOptions.SSLKey; s != "" { words = append(words, s) } } @@ -169,12 +171,9 @@ func ensureAuthParams(exporter *models.Agent, params *agentv1.SetStateRequest_Ag // getExporterListenAddress returns the appropriate listen address to use for a given exporter. func getExporterListenAddress(_ *models.Node, exporter *models.Agent) string { - switch { - case exporter.ExposeExporter: - return "0.0.0.0" - case exporter.PushMetrics: + if exporter.ExporterOptions != nil && exporter.ExporterOptions.PushMetrics { return "127.0.0.1" - default: - return "0.0.0.0" } + + return "0.0.0.0" } diff --git a/managed/services/agents/agents_test.go b/managed/services/agents/agents_test.go index 4c3115a551..846700ac39 100644 --- a/managed/services/agents/agents_test.go +++ b/managed/services/agents/agents_test.go @@ -56,7 +56,9 @@ func TestGetExporterListenAddress(t *testing.T) { Address: "1.2.3.4", } exporter := &models.Agent{ - PushMetrics: true, + ExporterOptions: &models.ExporterOptions{ + PushMetrics: true, + }, } assert.Equal(t, "127.0.0.1", getExporterListenAddress(node, exporter)) @@ -66,8 +68,10 @@ func TestGetExporterListenAddress(t *testing.T) { Address: "1.2.3.4", } exporter := &models.Agent{ - PushMetrics: true, - ExposeExporter: true, + ExporterOptions: &models.ExporterOptions{ + ExposeExporter: true, + PushMetrics: true, + }, } assert.Equal(t, "0.0.0.0", getExporterListenAddress(node, exporter)) @@ -77,15 +81,19 @@ func TestGetExporterListenAddress(t *testing.T) { Address: "1.2.3.4", } exporter := &models.Agent{ - PushMetrics: false, - ExposeExporter: true, + ExporterOptions: &models.ExporterOptions{ + ExposeExporter: true, + PushMetrics: false, + }, } assert.Equal(t, "0.0.0.0", getExporterListenAddress(node, exporter)) }) t.Run("exposes exporter address if node IP is unavailable in pull mode", func(t *testing.T) { exporter := &models.Agent{ - PushMetrics: false, + ExporterOptions: &models.ExporterOptions{ + PushMetrics: false, + }, } assert.Equal(t, "0.0.0.0", getExporterListenAddress(nil, exporter)) diff --git a/managed/services/agents/mongodb.go b/managed/services/agents/mongodb.go index a61d6af06d..37a81e4a0b 100644 --- a/managed/services/agents/mongodb.go +++ b/managed/services/agents/mongodb.go @@ -21,8 +21,6 @@ import ( "strings" "time" - "github.com/AlekSi/pointer" - agentv1 "github.com/percona/pmm/api/agent/v1" inventoryv1 "github.com/percona/pmm/api/inventory/v1" "github.com/percona/pmm/managed/models" @@ -49,8 +47,8 @@ func mongodbExporterConfig(node *models.Node, service *models.Service, exporter args := getArgs(exporter, tdp, listenAddress, pmmAgentVersion) - if pointer.GetString(exporter.MetricsPath) != "" { - args = append(args, "--web.telemetry-path="+*exporter.MetricsPath) //nolint:goconst + if exporter.ExporterOptions != nil && exporter.ExporterOptions.MetricsPath != "" { + args = append(args, "--web.telemetry-path="+exporter.ExporterOptions.MetricsPath) //nolint:goconst } args = withLogLevel(args, exporter.LogLevel, pmmAgentVersion, true) diff --git a/managed/services/agents/mongodb_test.go b/managed/services/agents/mongodb_test.go index c0357e7464..1c1a5ead65 100644 --- a/managed/services/agents/mongodb_test.go +++ b/managed/services/agents/mongodb_test.go @@ -387,11 +387,13 @@ func TestMongodbExporterConfig2411(t *testing.T) { }) t.Run("Enable all collectors and disable some", func(t *testing.T) { + exporter.ExporterOptions = &models.ExporterOptions{ + DisabledCollectors: []string{"dbstats", "topmetrics"}, + } exporter.MongoDBOptions = &models.MongoDBOptions{ EnableAllCollectors: true, StatsCollections: []string{"db1.col1.one", "db2.col2", "db3"}, } - exporter.DisabledCollectors = []string{"dbstats", "topmetrics"} expected.Args = []string{ "--collector.collstats", @@ -597,7 +599,9 @@ func TestMongodbExporterConfig(t *testing.T) { }) t.Run("DisabledCollectors", func(t *testing.T) { - exporter.DisabledCollectors = []string{"topmetrics"} + exporter.ExporterOptions = &models.ExporterOptions{ + DisabledCollectors: []string{"topmetrics"}, + } actual, err := mongodbExporterConfig(node, mongodb, exporter, exposeSecrets, pmmAgentVersion) expected := &agentv1.SetStateRequest_AgentProcess{ Type: inventoryv1.AgentType_AGENT_TYPE_MONGODB_EXPORTER, diff --git a/managed/services/management/mongodb.go b/managed/services/management/mongodb.go index 5498ba2c51..34b7e9d7ca 100644 --- a/managed/services/management/mongodb.go +++ b/managed/services/management/mongodb.go @@ -100,16 +100,18 @@ func (s *ManagementService) addMongoDB(ctx context.Context, req *managementv1.Ad if req.QanMongodbProfiler { row, err = models.CreateAgent(tx.Querier, models.QANMongoDBProfilerAgentType, &models.CreateAgentParams{ - PMMAgentID: req.PmmAgentId, - ServiceID: service.ServiceID, - Username: req.Username, - Password: req.Password, - TLS: req.Tls, - TLSSkipVerify: req.TlsSkipVerify, + PMMAgentID: req.PmmAgentId, + ServiceID: service.ServiceID, + Username: req.Username, + Password: req.Password, + TLS: req.Tls, + TLSSkipVerify: req.TlsSkipVerify, + QANOptions: &models.QANOptions{ + MaxQueryLength: req.MaxQueryLength, + // TODO QueryExamplesDisabled https://jira.percona.com/browse/PMM-7860 + }, MongoDBOptions: models.MongoDBOptionsFromRequest(req), - MaxQueryLength: req.MaxQueryLength, LogLevel: services.SpecifyLogLevel(req.LogLevel, inventoryv1.LogLevel_LOG_LEVEL_FATAL), - // TODO QueryExamplesDisabled https://jira.percona.com/browse/PMM-7860 }) if err != nil { return err diff --git a/managed/services/management/postgresql.go b/managed/services/management/postgresql.go index e1f1a473da..63bb85171d 100644 --- a/managed/services/management/postgresql.go +++ b/managed/services/management/postgresql.go @@ -65,16 +65,18 @@ func (s *ManagementService) addPostgreSQL(ctx context.Context, req *managementv1 } row, err := models.CreateAgent(tx.Querier, models.PostgresExporterType, &models.CreateAgentParams{ - PMMAgentID: req.PmmAgentId, - ServiceID: service.ServiceID, - Username: req.Username, - Password: req.Password, - AgentPassword: req.AgentPassword, - TLS: req.Tls, - TLSSkipVerify: req.TlsSkipVerify, - PushMetrics: isPushMode(req.MetricsMode), - ExposeExporter: req.ExposeExporter, - DisableCollectors: req.DisableCollectors, + PMMAgentID: req.PmmAgentId, + ServiceID: service.ServiceID, + Username: req.Username, + Password: req.Password, + AgentPassword: req.AgentPassword, + TLS: req.Tls, + TLSSkipVerify: req.TlsSkipVerify, + ExporterOptions: &models.ExporterOptions{ + ExposeExporter: req.ExposeExporter, + PushMetrics: isPushMode(req.MetricsMode), + DisabledCollectors: req.DisableCollectors, + }, PostgreSQLOptions: models.PostgreSQLOptionsFromRequest(req), LogLevel: services.SpecifyLogLevel(req.LogLevel, inventoryv1.LogLevel_LOG_LEVEL_ERROR), }) @@ -107,17 +109,19 @@ func (s *ManagementService) addPostgreSQL(ctx context.Context, req *managementv1 if req.QanPostgresqlPgstatementsAgent { row, err = models.CreateAgent(tx.Querier, models.QANPostgreSQLPgStatementsAgentType, &models.CreateAgentParams{ - PMMAgentID: req.PmmAgentId, - ServiceID: service.ServiceID, - Username: req.Username, - Password: req.Password, - MaxQueryLength: req.MaxQueryLength, - QueryExamplesDisabled: req.DisableQueryExamples, - CommentsParsingDisabled: req.DisableCommentsParsing, - TLS: req.Tls, - TLSSkipVerify: req.TlsSkipVerify, - PostgreSQLOptions: models.PostgreSQLOptionsFromRequest(req), - LogLevel: services.SpecifyLogLevel(req.LogLevel, inventoryv1.LogLevel_LOG_LEVEL_FATAL), + PMMAgentID: req.PmmAgentId, + ServiceID: service.ServiceID, + Username: req.Username, + Password: req.Password, + QANOptions: &models.QANOptions{ + MaxQueryLength: req.MaxQueryLength, + QueryExamplesDisabled: req.DisableQueryExamples, + CommentsParsingDisabled: req.DisableCommentsParsing, + }, + TLS: req.Tls, + TLSSkipVerify: req.TlsSkipVerify, + PostgreSQLOptions: models.PostgreSQLOptionsFromRequest(req), + LogLevel: services.SpecifyLogLevel(req.LogLevel, inventoryv1.LogLevel_LOG_LEVEL_FATAL), }) if err != nil { return err @@ -132,17 +136,19 @@ func (s *ManagementService) addPostgreSQL(ctx context.Context, req *managementv1 if req.QanPostgresqlPgstatmonitorAgent { row, err = models.CreateAgent(tx.Querier, models.QANPostgreSQLPgStatMonitorAgentType, &models.CreateAgentParams{ - PMMAgentID: req.PmmAgentId, - ServiceID: service.ServiceID, - Username: req.Username, - Password: req.Password, - MaxQueryLength: req.MaxQueryLength, - QueryExamplesDisabled: req.DisableQueryExamples, - CommentsParsingDisabled: req.DisableCommentsParsing, - TLS: req.Tls, - TLSSkipVerify: req.TlsSkipVerify, - PostgreSQLOptions: models.PostgreSQLOptionsFromRequest(req), - LogLevel: services.SpecifyLogLevel(req.LogLevel, inventoryv1.LogLevel_LOG_LEVEL_FATAL), + PMMAgentID: req.PmmAgentId, + ServiceID: service.ServiceID, + Username: req.Username, + Password: req.Password, + TLS: req.Tls, + TLSSkipVerify: req.TlsSkipVerify, + QANOptions: &models.QANOptions{ + MaxQueryLength: req.MaxQueryLength, + QueryExamplesDisabled: req.DisableQueryExamples, + CommentsParsingDisabled: req.DisableCommentsParsing, + }, + PostgreSQLOptions: models.PostgreSQLOptionsFromRequest(req), + LogLevel: services.SpecifyLogLevel(req.LogLevel, inventoryv1.LogLevel_LOG_LEVEL_FATAL), }) if err != nil { return err From b6fd35afa9138e84b4d491409518e2a308b509e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Thu, 21 Nov 2024 11:48:45 +0100 Subject: [PATCH 04/60] PMM-5086-12634 Another changes. --- managed/models/agent_helpers.go | 2 - managed/models/agent_model.go | 19 +---- managed/services/agents/connection_checker.go | 2 +- managed/services/agents/mongodb.go | 10 +-- managed/services/agents/mysql.go | 22 +++-- managed/services/agents/mysql_test.go | 30 +++---- managed/services/agents/node.go | 14 +--- managed/services/agents/node_test.go | 8 +- managed/services/agents/postgresql.go | 20 ++--- managed/services/agents/postgresql_test.go | 2 +- managed/services/agents/proxysql.go | 8 +- managed/services/agents/proxysql_test.go | 2 +- managed/services/agents/rds_test.go | 26 +++--- managed/services/management/mysql.go | 82 ++++++++++--------- managed/services/management/proxysql.go | 24 +++--- managed/services/management/rds.go | 77 ++++++++++------- 16 files changed, 178 insertions(+), 170 deletions(-) diff --git a/managed/models/agent_helpers.go b/managed/models/agent_helpers.go index 91712efe7e..6a64e87d16 100644 --- a/managed/models/agent_helpers.go +++ b/managed/models/agent_helpers.go @@ -774,7 +774,6 @@ type CreateAgentParams struct { MongoDBOptions *MongoDBOptions MySQLOptions *MySQLOptions PostgreSQLOptions *PostgreSQLOptions - RDSOptions *RDSOptions } func compatibleNodeAndAgent(nodeType NodeType, agentType AgentType) bool { @@ -916,7 +915,6 @@ func CreateAgent(q *reform.Querier, agentType AgentType, params *CreateAgentPara MongoDBOptions: params.MongoDBOptions, MySQLOptions: params.MySQLOptions, PostgreSQLOptions: params.PostgreSQLOptions, - RDSOptions: params.RDSOptions, } if err := row.SetCustomLabels(params.CustomLabels); err != nil { return nil, err diff --git a/managed/models/agent_model.go b/managed/models/agent_model.go index 2e34099f3a..829b360e12 100644 --- a/managed/models/agent_model.go +++ b/managed/models/agent_model.go @@ -110,8 +110,10 @@ func (c *QANOptions) Scan(src interface{}) error { return jsonScan(c, src) } // AWSOptions represents structure for special AWS options. type AWSOptions struct { - AWSAccessKey string `reform:"aws_access_key"` - AWSSecretKey string `reform:"aws_secret_key"` + AWSAccessKey string `reform:"aws_access_key"` + AWSSecretKey string `reform:"aws_secret_key"` + RDSBasicMetricsDisabled bool `reform:"rds_basic_metrics_disabled"` + RDSEnhancedMetricsDisabled bool `reform:"rds_enhanced_metrics_disabled"` } // Value implements database/sql/driver.Valuer interface. Should be defined on the value. @@ -192,18 +194,6 @@ func (c PostgreSQLOptions) Value() (driver.Value, error) { return jsonValue(c) } // Scan implements database/sql.Scanner interface. Should be defined on the pointer. func (c *PostgreSQLOptions) Scan(src interface{}) error { return jsonScan(c, src) } -// RDSOptions represents structure for special RDSOptions options. -type RDSOptions struct { - RDSBasicMetricsDisabled bool `reform:"rds_basic_metrics_disabled"` - RDSEnhancedMetricsDisabled bool `reform:"rds_enhanced_metrics_disabled"` -} - -// Value implements database/sql/driver.Valuer interface. Should be defined on the value. -func (c RDSOptions) Value() (driver.Value, error) { return jsonValue(c) } - -// Scan implements database/sql.Scanner interface. Should be defined on the pointer. -func (c *RDSOptions) Scan(src interface{}) error { return jsonScan(c, src) } - // PMMAgentWithPushMetricsSupport - version of pmmAgent, // that support vmagent and push metrics mode // will be released with PMM Agent v2.12. @@ -245,7 +235,6 @@ type Agent struct { MongoDBOptions *MongoDBOptions `reform:"mongo_db_tls_options"` MySQLOptions *MySQLOptions `reform:"mysql_options"` PostgreSQLOptions *PostgreSQLOptions `reform:"postgresql_options"` - RDSOptions *RDSOptions `reform:"rds_options"` } // BeforeInsert implements reform.BeforeInserter interface. diff --git a/managed/services/agents/connection_checker.go b/managed/services/agents/connection_checker.go index 93e5c92a73..388af0e8f5 100644 --- a/managed/services/agents/connection_checker.go +++ b/managed/services/agents/connection_checker.go @@ -60,7 +60,7 @@ func (c *ConnectionChecker) CheckConnectionToService(ctx context.Context, q *ref }() pmmAgentID := pointer.GetString(agent.PMMAgentID) - if !agent.PushMetrics && (service.ServiceType == models.ExternalServiceType || service.ServiceType == models.HAProxyServiceType) { + if !agent.ExporterOptions.PushMetrics && (service.ServiceType == models.ExternalServiceType || service.ServiceType == models.HAProxyServiceType) { pmmAgentID = models.PMMServerAgentID } diff --git a/managed/services/agents/mongodb.go b/managed/services/agents/mongodb.go index 37a81e4a0b..647559df30 100644 --- a/managed/services/agents/mongodb.go +++ b/managed/services/agents/mongodb.go @@ -115,8 +115,8 @@ func getArgs(exporter *models.Agent, tdp *models.DelimiterPair, listenAddress st args = append(args, "--collector.pbm") } - args = collectors.FilterOutCollectors("--collector.", args, exporter.DisabledCollectors) - args = append(args, collectors.DisableDefaultEnabledCollectors("--no-collector.", defaultEnabledCollectors, exporter.DisabledCollectors)...) + args = collectors.FilterOutCollectors("--collector.", args, exporter.ExporterOptions.DisabledCollectors) + args = append(args, collectors.DisableDefaultEnabledCollectors("--no-collector.", defaultEnabledCollectors, exporter.ExporterOptions.DisabledCollectors)...) if exporter.MongoDBOptions != nil && len(exporter.MongoDBOptions.StatsCollections) != 0 { args = append(args, "--mongodb.collstats-colls="+strings.Join(exporter.MongoDBOptions.StatsCollections, ",")) @@ -146,7 +146,7 @@ func getArgs(exporter *models.Agent, tdp *models.DelimiterPair, listenAddress st "--web.listen-address=" + listenAddress + ":" + tdp.Left + " .listen_port " + tdp.Right, //nolint:goconst } - args = collectors.FilterOutCollectors("--collect.", args, exporter.DisabledCollectors) + args = collectors.FilterOutCollectors("--collect.", args, exporter.ExporterOptions.DisabledCollectors) } return args @@ -166,8 +166,8 @@ func qanMongoDBProfilerAgentConfig(service *models.Service, agent *models.Agent, return &agentv1.SetStateRequest_BuiltinAgent{ Type: inventoryv1.AgentType_AGENT_TYPE_QAN_MONGODB_PROFILER_AGENT, Dsn: agent.DSN(service, models.DSNParams{DialTimeout: time.Second, Database: ""}, nil, pmmAgentVersion), - DisableQueryExamples: agent.QueryExamplesDisabled, - MaxQueryLength: agent.MaxQueryLength, + DisableQueryExamples: agent.QANOptions.QueryExamplesDisabled, + MaxQueryLength: agent.QANOptions.MaxQueryLength, TextFiles: &agentv1.TextFiles{ Files: agent.Files(), TemplateLeftDelim: tdp.Left, diff --git a/managed/services/agents/mysql.go b/managed/services/agents/mysql.go index 54f13f9dfc..34140b0f89 100644 --- a/managed/services/agents/mysql.go +++ b/managed/services/agents/mysql.go @@ -20,8 +20,6 @@ import ( "sort" "time" - "github.com/AlekSi/pointer" - agentv1 "github.com/percona/pmm/api/agent/v1" inventoryv1 "github.com/percona/pmm/api/inventory/v1" "github.com/percona/pmm/managed/models" @@ -106,10 +104,10 @@ func mysqldExporterConfig( args = append(args, tablestatsGroup...) } - args = collectors.FilterOutCollectors("--collect.", args, exporter.DisabledCollectors) + args = collectors.FilterOutCollectors("--collect.", args, exporter.ExporterOptions.DisabledCollectors) - if pointer.GetString(exporter.MetricsPath) != "" { - args = append(args, "--web.telemetry-path="+*exporter.MetricsPath) + if exporter.ExporterOptions.MetricsPath != "" { + args = append(args, "--web.telemetry-path="+exporter.ExporterOptions.MetricsPath) } files := exporter.Files() @@ -159,9 +157,9 @@ func qanMySQLPerfSchemaAgentConfig(service *models.Service, agent *models.Agent, return &agentv1.SetStateRequest_BuiltinAgent{ Type: inventoryv1.AgentType_AGENT_TYPE_QAN_MYSQL_PERFSCHEMA_AGENT, Dsn: agent.DSN(service, models.DSNParams{DialTimeout: time.Second, Database: ""}, nil, pmmAgentVersion), - MaxQueryLength: agent.MaxQueryLength, - DisableQueryExamples: agent.QueryExamplesDisabled, - DisableCommentsParsing: agent.CommentsParsingDisabled, + MaxQueryLength: agent.QANOptions.MaxQueryLength, + DisableQueryExamples: agent.QANOptions.QueryExamplesDisabled, + DisableCommentsParsing: agent.QANOptions.CommentsParsingDisabled, TextFiles: &agentv1.TextFiles{ Files: agent.Files(), TemplateLeftDelim: tdp.Left, @@ -177,10 +175,10 @@ func qanMySQLSlowlogAgentConfig(service *models.Service, agent *models.Agent, pm return &agentv1.SetStateRequest_BuiltinAgent{ Type: inventoryv1.AgentType_AGENT_TYPE_QAN_MYSQL_SLOWLOG_AGENT, Dsn: agent.DSN(service, models.DSNParams{DialTimeout: time.Second, Database: ""}, nil, pmmAgentVersion), - MaxQueryLength: agent.MaxQueryLength, - DisableQueryExamples: agent.QueryExamplesDisabled, - DisableCommentsParsing: agent.CommentsParsingDisabled, - MaxQueryLogSize: agent.MaxQueryLogSize, + MaxQueryLength: agent.QANOptions.MaxQueryLength, + DisableQueryExamples: agent.QANOptions.QueryExamplesDisabled, + DisableCommentsParsing: agent.QANOptions.CommentsParsingDisabled, + MaxQueryLogSize: agent.QANOptions.MaxQueryLogSize, TextFiles: &agentv1.TextFiles{ Files: agent.Files(), TemplateLeftDelim: tdp.Left, diff --git a/managed/services/agents/mysql_test.go b/managed/services/agents/mysql_test.go index 6cee95abeb..943db762aa 100644 --- a/managed/services/agents/mysql_test.go +++ b/managed/services/agents/mysql_test.go @@ -141,16 +141,16 @@ func TestMySQLdExporterConfigTablestatsGroupDisabled(t *testing.T) { Port: pointer.ToUint16(3306), } exporter := &models.Agent{ - AgentID: "agent-id", - AgentType: models.MySQLdExporterType, - Username: pointer.ToString("username"), - Password: pointer.ToString("s3cur3 p@$$w0r4."), - TableCountTablestatsGroupLimit: -1, - TLS: true, + AgentID: "agent-id", + AgentType: models.MySQLdExporterType, + Username: pointer.ToString("username"), + Password: pointer.ToString("s3cur3 p@$$w0r4."), + TLS: true, MySQLOptions: &models.MySQLOptions{ - TLSCa: "content-of-tls-ca", - TLSCert: "content-of-tls-cert", - TLSKey: "content-of-tls-key", + TableCountTablestatsGroupLimit: -1, + TLSCa: "content-of-tls-ca", + TLSCert: "content-of-tls-cert", + TLSKey: "content-of-tls-key", }, } pmmAgentVersion := version.MustParse("2.24.0") @@ -245,11 +245,13 @@ func TestMySQLdExporterConfigDisabledCollectors(t *testing.T) { Port: pointer.ToUint16(3306), } exporter := &models.Agent{ - AgentID: "agent-id", - AgentType: models.MySQLdExporterType, - Username: pointer.ToString("username"), - Password: pointer.ToString("s3cur3 p@$$w0r4."), - DisabledCollectors: []string{"heartbeat", "info_schema.clientstats", "perf_schema.eventsstatements", "custom_query.hr"}, + AgentID: "agent-id", + AgentType: models.MySQLdExporterType, + Username: pointer.ToString("username"), + Password: pointer.ToString("s3cur3 p@$$w0r4."), + ExporterOptions: &models.ExporterOptions{ + DisabledCollectors: []string{"heartbeat", "info_schema.clientstats", "perf_schema.eventsstatements", "custom_query.hr"}, + }, } pmmAgentVersion := version.MustParse("2.24.0") diff --git a/managed/services/agents/node.go b/managed/services/agents/node.go index 4aac0b2422..b81c763ad2 100644 --- a/managed/services/agents/node.go +++ b/managed/services/agents/node.go @@ -18,8 +18,6 @@ package agents import ( "sort" - "github.com/AlekSi/pointer" - agentv1 "github.com/percona/pmm/api/agent/v1" inventoryv1 "github.com/percona/pmm/api/inventory/v1" "github.com/percona/pmm/managed/models" @@ -36,11 +34,7 @@ var ( func nodeExporterConfig(node *models.Node, exporter *models.Agent, agentVersion *version.Parsed) (*agentv1.SetStateRequest_AgentProcess, error) { listenAddress := getExporterListenAddress(node, exporter) - - tdp := models.TemplateDelimsPair( - pointer.GetString(exporter.MetricsPath), - ) - + tdp := models.TemplateDelimsPair(exporter.ExporterOptions.MetricsPath) args := []string{ "--collector.textfile.directory.lr=" + pathsBase(agentVersion, tdp.Left, tdp.Right) + "/collectors/textfile-collector/low-resolution", "--collector.textfile.directory.mr=" + pathsBase(agentVersion, tdp.Left, tdp.Right) + "/collectors/textfile-collector/medium-resolution", @@ -127,10 +121,10 @@ func nodeExporterConfig(node *models.Node, exporter *models.Agent, agentVersion ) } - args = collectors.FilterOutCollectors("--collector.", args, exporter.DisabledCollectors) + args = collectors.FilterOutCollectors("--collector.", args, exporter.ExporterOptions.DisabledCollectors) - if pointer.GetString(exporter.MetricsPath) != "" { - args = append(args, "--web.telemetry-path="+*exporter.MetricsPath) + if exporter.ExporterOptions.MetricsPath != "" { + args = append(args, "--web.telemetry-path="+exporter.ExporterOptions.MetricsPath) } args = withLogLevel(args, exporter.LogLevel, agentVersion, false) diff --git a/managed/services/agents/node_test.go b/managed/services/agents/node_test.go index 01bbdaf279..3de1512e52 100644 --- a/managed/services/agents/node_test.go +++ b/managed/services/agents/node_test.go @@ -206,9 +206,11 @@ func TestNodeExporterConfig(t *testing.T) { t.Parallel() node := &models.Node{} exporter := &models.Agent{ - AgentID: "agent-id", - AgentType: models.NodeExporterType, - DisabledCollectors: []string{"cpu", "netstat", "netstat.fields", "vmstat", "meminfo"}, + AgentID: "agent-id", + AgentType: models.NodeExporterType, + ExporterOptions: &models.ExporterOptions{ + DisabledCollectors: []string{"cpu", "netstat", "netstat.fields", "vmstat", "meminfo"}, + }, } agentVersion := version.MustParse("2.15.1") diff --git a/managed/services/agents/postgresql.go b/managed/services/agents/postgresql.go index 7f2c035f90..cca54ba767 100644 --- a/managed/services/agents/postgresql.go +++ b/managed/services/agents/postgresql.go @@ -22,8 +22,6 @@ import ( "strings" "time" - "github.com/AlekSi/pointer" - agentv1 "github.com/percona/pmm/api/agent/v1" inventoryv1 "github.com/percona/pmm/api/inventory/v1" "github.com/percona/pmm/managed/models" @@ -110,14 +108,14 @@ func postgresExporterConfig(node *models.Node, service *models.Service, exporter args = append(args, "--max-connections="+strconv.Itoa(int(exporter.PostgreSQLOptions.MaxExporterConnections))) } - if pointer.GetString(exporter.MetricsPath) != "" { - args = append(args, "--web.telemetry-path="+*exporter.MetricsPath) + if exporter.ExporterOptions.MetricsPath != "" { + args = append(args, "--web.telemetry-path="+exporter.ExporterOptions.MetricsPath) } - args = collectors.FilterOutCollectors("--collect.", args, exporter.DisabledCollectors) + args = collectors.FilterOutCollectors("--collect.", args, exporter.ExporterOptions.DisabledCollectors) if !pmmAgentVersion.Less(postgresExporterCollectorsVersion) { - disableCollectorArgs := collectors.DisableDefaultEnabledCollectors("--no-collector.", defaultPostgresExporterCollectors, exporter.DisabledCollectors) + disableCollectorArgs := collectors.DisableDefaultEnabledCollectors("--no-collector.", defaultPostgresExporterCollectors, exporter.ExporterOptions.DisabledCollectors) args = append(args, disableCollectorArgs...) } @@ -168,8 +166,8 @@ func qanPostgreSQLPgStatementsAgentConfig(service *models.Service, agent *models return &agentv1.SetStateRequest_BuiltinAgent{ Type: inventoryv1.AgentType_AGENT_TYPE_QAN_POSTGRESQL_PGSTATEMENTS_AGENT, Dsn: agent.DSN(service, dnsParams, nil, pmmAgentVersion), - MaxQueryLength: agent.MaxQueryLength, - DisableCommentsParsing: agent.CommentsParsingDisabled, + MaxQueryLength: agent.QANOptions.MaxQueryLength, + DisableCommentsParsing: agent.QANOptions.CommentsParsingDisabled, TextFiles: &agentv1.TextFiles{ Files: agent.Files(), TemplateLeftDelim: tdp.Left, @@ -189,9 +187,9 @@ func qanPostgreSQLPgStatMonitorAgentConfig(service *models.Service, agent *model return &agentv1.SetStateRequest_BuiltinAgent{ Type: inventoryv1.AgentType_AGENT_TYPE_QAN_POSTGRESQL_PGSTATMONITOR_AGENT, Dsn: agent.DSN(service, dnsParams, nil, pmmAgentVersion), - DisableQueryExamples: agent.QueryExamplesDisabled, - MaxQueryLength: agent.MaxQueryLength, - DisableCommentsParsing: agent.CommentsParsingDisabled, + DisableQueryExamples: agent.QANOptions.QueryExamplesDisabled, + MaxQueryLength: agent.QANOptions.MaxQueryLength, + DisableCommentsParsing: agent.QANOptions.CommentsParsingDisabled, TextFiles: &agentv1.TextFiles{ Files: agent.Files(), TemplateLeftDelim: tdp.Left, diff --git a/managed/services/agents/postgresql_test.go b/managed/services/agents/postgresql_test.go index f51b858fb7..444d370c43 100644 --- a/managed/services/agents/postgresql_test.go +++ b/managed/services/agents/postgresql_test.go @@ -154,7 +154,7 @@ func (s *PostgresExporterConfigTestSuite) TestDisabledCollectors() { s.postgresql.Address = nil s.postgresql.Port = nil s.postgresql.Socket = pointer.ToString("/var/run/postgres") - s.exporter.DisabledCollectors = []string{"custom_query.hr", "custom_query.hr.directory", "locks"} + s.exporter.ExporterOptions.DisabledCollectors = []string{"custom_query.hr", "custom_query.hr.directory", "locks"} actual, err := postgresExporterConfig(s.node, s.postgresql, s.exporter, exposeSecrets, s.pmmAgentVersion) s.NoError(err, "Failed to create exporter config") diff --git a/managed/services/agents/proxysql.go b/managed/services/agents/proxysql.go index 2de7a16fdc..8f786c25e1 100644 --- a/managed/services/agents/proxysql.go +++ b/managed/services/agents/proxysql.go @@ -20,8 +20,6 @@ import ( "sort" "time" - "github.com/AlekSi/pointer" - agentv1 "github.com/percona/pmm/api/agent/v1" inventoryv1 "github.com/percona/pmm/api/inventory/v1" "github.com/percona/pmm/managed/models" @@ -57,11 +55,11 @@ func proxysqlExporterConfig(node *models.Node, service *models.Service, exporter args = append(args, "-collect.runtime_mysql_servers") } - if pointer.GetString(exporter.MetricsPath) != "" { - args = append(args, "-web.telemetry-path="+*exporter.MetricsPath) + if exporter.ExporterOptions.MetricsPath != "" { + args = append(args, "-web.telemetry-path="+exporter.ExporterOptions.MetricsPath) } - args = collectors.FilterOutCollectors("-collect.", args, exporter.DisabledCollectors) + args = collectors.FilterOutCollectors("-collect.", args, exporter.ExporterOptions.DisabledCollectors) args = withLogLevel(args, exporter.LogLevel, pmmAgentVersion, true) diff --git a/managed/services/agents/proxysql_test.go b/managed/services/agents/proxysql_test.go index e3e2b46ddb..bd870e76d5 100644 --- a/managed/services/agents/proxysql_test.go +++ b/managed/services/agents/proxysql_test.go @@ -80,7 +80,7 @@ func TestProxySQLExporterConfig(t *testing.T) { }) t.Run("DisabledCollector", func(t *testing.T) { - exporter.DisabledCollectors = []string{"mysql_connection_list", "stats_memory_metrics"} + exporter.ExporterOptions.DisabledCollectors = []string{"mysql_connection_list", "stats_memory_metrics"} actual := proxysqlExporterConfig(node, proxysql, exporter, exposeSecrets, pmmAgentVersion) expected := &agentv1.SetStateRequest_AgentProcess{ Type: inventoryv1.AgentType_AGENT_TYPE_PROXYSQL_EXPORTER, diff --git a/managed/services/agents/rds_test.go b/managed/services/agents/rds_test.go index 5a398d30a4..3cecdef1c6 100644 --- a/managed/services/agents/rds_test.go +++ b/managed/services/agents/rds_test.go @@ -45,12 +45,14 @@ func TestRDSExporterConfig(t *testing.T) { }) require.NoError(t, err) agent1 := &models.Agent{ - AgentID: "agent1", - AgentType: models.RDSExporterType, - NodeID: &node1.NodeID, - AWSAccessKey: pointer.ToString("access_key1"), - AWSSecretKey: pointer.ToString("secret_key1"), - RDSBasicMetricsDisabled: true, + AgentID: "agent1", + AgentType: models.RDSExporterType, + NodeID: &node1.NodeID, + AWSOptions: &models.AWSOptions{ + AWSAccessKey: "access_key1", + AWSSecretKey: "secret_key1", + RDSBasicMetricsDisabled: true, + }, } node2 := &models.Node{ @@ -67,11 +69,13 @@ func TestRDSExporterConfig(t *testing.T) { }) require.NoError(t, err) agent2 := &models.Agent{ - AgentID: "agent2", - AgentType: models.RDSExporterType, - NodeID: &node2.NodeID, - AWSAccessKey: pointer.ToString("access_key2"), - AWSSecretKey: pointer.ToString("secret_key2"), + AgentID: "agent2", + AgentType: models.RDSExporterType, + NodeID: &node2.NodeID, + AWSOptions: &models.AWSOptions{ + AWSAccessKey: "access_key2", + AWSSecretKey: "secret_key2", + }, } pairs := map[*models.Node]*models.Agent{ diff --git a/managed/services/management/mysql.go b/managed/services/management/mysql.go index 8ef37eb4d0..112de34e5a 100644 --- a/managed/services/management/mysql.go +++ b/managed/services/management/mysql.go @@ -86,20 +86,24 @@ func (s *ManagementService) addMySQL(ctx context.Context, req *managementv1.AddM return err } + mysqlOptions := models.MySQLOptionsFromRequest(req) + mysqlOptions.TableCountTablestatsGroupLimit = tablestatsGroupTableLimit + row, err := models.CreateAgent(tx.Querier, models.MySQLdExporterType, &models.CreateAgentParams{ - PMMAgentID: req.PmmAgentId, - ServiceID: service.ServiceID, - Username: req.Username, - Password: req.Password, - AgentPassword: req.AgentPassword, - TLS: req.Tls, - TLSSkipVerify: req.TlsSkipVerify, - MySQLOptions: models.MySQLOptionsFromRequest(req), - TableCountTablestatsGroupLimit: tablestatsGroupTableLimit, - PushMetrics: isPushMode(req.MetricsMode), - ExposeExporter: req.ExposeExporter, - DisableCollectors: req.DisableCollectors, - LogLevel: services.SpecifyLogLevel(req.LogLevel, inventoryv1.LogLevel_LOG_LEVEL_ERROR), + PMMAgentID: req.PmmAgentId, + ServiceID: service.ServiceID, + Username: req.Username, + Password: req.Password, + AgentPassword: req.AgentPassword, + TLS: req.Tls, + TLSSkipVerify: req.TlsSkipVerify, + MySQLOptions: mysqlOptions, + ExporterOptions: &models.ExporterOptions{ + ExposeExporter: req.ExposeExporter, + PushMetrics: isPushMode(req.MetricsMode), + DisabledCollectors: req.DisableCollectors, + }, + LogLevel: services.SpecifyLogLevel(req.LogLevel, inventoryv1.LogLevel_LOG_LEVEL_ERROR), }) if err != nil { return err @@ -113,7 +117,7 @@ func (s *ManagementService) addMySQL(ctx context.Context, req *managementv1.AddM return err } // GetInfoFromService updates the table count in row so, let's also update the response - mysql.TableCount = *row.TableCount + mysql.TableCount = *row.MySQLOptions.TableCount } agent, err := services.ToAPIAgent(tx.Querier, row) @@ -124,17 +128,19 @@ func (s *ManagementService) addMySQL(ctx context.Context, req *managementv1.AddM if req.QanMysqlPerfschema { row, err = models.CreateAgent(tx.Querier, models.QANMySQLPerfSchemaAgentType, &models.CreateAgentParams{ - PMMAgentID: req.PmmAgentId, - ServiceID: service.ServiceID, - Username: req.Username, - Password: req.Password, - TLS: req.Tls, - TLSSkipVerify: req.TlsSkipVerify, - MySQLOptions: models.MySQLOptionsFromRequest(req), - MaxQueryLength: req.MaxQueryLength, - QueryExamplesDisabled: req.DisableQueryExamples, - CommentsParsingDisabled: req.DisableCommentsParsing, - LogLevel: services.SpecifyLogLevel(req.LogLevel, inventoryv1.LogLevel_LOG_LEVEL_FATAL), + PMMAgentID: req.PmmAgentId, + ServiceID: service.ServiceID, + Username: req.Username, + Password: req.Password, + TLS: req.Tls, + TLSSkipVerify: req.TlsSkipVerify, + QANOptions: &models.QANOptions{ + MaxQueryLength: req.MaxQueryLength, + QueryExamplesDisabled: req.DisableQueryExamples, + CommentsParsingDisabled: req.DisableCommentsParsing, + }, + MySQLOptions: models.MySQLOptionsFromRequest(req), + LogLevel: services.SpecifyLogLevel(req.LogLevel, inventoryv1.LogLevel_LOG_LEVEL_FATAL), }) if err != nil { return err @@ -149,18 +155,20 @@ func (s *ManagementService) addMySQL(ctx context.Context, req *managementv1.AddM if req.QanMysqlSlowlog { row, err = models.CreateAgent(tx.Querier, models.QANMySQLSlowlogAgentType, &models.CreateAgentParams{ - PMMAgentID: req.PmmAgentId, - ServiceID: service.ServiceID, - Username: req.Username, - Password: req.Password, - TLS: req.Tls, - TLSSkipVerify: req.TlsSkipVerify, - MySQLOptions: models.MySQLOptionsFromRequest(req), - MaxQueryLength: req.MaxQueryLength, - QueryExamplesDisabled: req.DisableQueryExamples, - CommentsParsingDisabled: req.DisableCommentsParsing, - MaxQueryLogSize: maxSlowlogFileSize, - LogLevel: services.SpecifyLogLevel(req.LogLevel, inventoryv1.LogLevel_LOG_LEVEL_FATAL), + PMMAgentID: req.PmmAgentId, + ServiceID: service.ServiceID, + Username: req.Username, + Password: req.Password, + TLS: req.Tls, + TLSSkipVerify: req.TlsSkipVerify, + MySQLOptions: models.MySQLOptionsFromRequest(req), + QANOptions: &models.QANOptions{ + MaxQueryLength: req.MaxQueryLength, + QueryExamplesDisabled: req.DisableQueryExamples, + CommentsParsingDisabled: req.DisableCommentsParsing, + MaxQueryLogSize: maxSlowlogFileSize, + }, + LogLevel: services.SpecifyLogLevel(req.LogLevel, inventoryv1.LogLevel_LOG_LEVEL_FATAL), }) if err != nil { return err diff --git a/managed/services/management/proxysql.go b/managed/services/management/proxysql.go index cf8ac0d5da..d1f5eb6bce 100644 --- a/managed/services/management/proxysql.go +++ b/managed/services/management/proxysql.go @@ -63,17 +63,19 @@ func (s *ManagementService) addProxySQL(ctx context.Context, req *managementv1.A } row, err := models.CreateAgent(tx.Querier, models.ProxySQLExporterType, &models.CreateAgentParams{ - PMMAgentID: req.PmmAgentId, - ServiceID: service.ServiceID, - Username: req.Username, - Password: req.Password, - AgentPassword: req.AgentPassword, - TLS: req.Tls, - TLSSkipVerify: req.TlsSkipVerify, - PushMetrics: isPushMode(req.MetricsMode), - ExposeExporter: req.ExposeExporter, - DisableCollectors: req.DisableCollectors, - LogLevel: services.SpecifyLogLevel(req.LogLevel, inventoryv1.LogLevel_LOG_LEVEL_FATAL), + PMMAgentID: req.PmmAgentId, + ServiceID: service.ServiceID, + Username: req.Username, + Password: req.Password, + AgentPassword: req.AgentPassword, + TLS: req.Tls, + TLSSkipVerify: req.TlsSkipVerify, + ExporterOptions: &models.ExporterOptions{ + ExposeExporter: req.ExposeExporter, + PushMetrics: isPushMode(req.MetricsMode), + DisabledCollectors: req.DisableCollectors, + }, + LogLevel: services.SpecifyLogLevel(req.LogLevel, inventoryv1.LogLevel_LOG_LEVEL_FATAL), }) if err != nil { return err diff --git a/managed/services/management/rds.go b/managed/services/management/rds.go index a7ce3035e8..de5f774c08 100644 --- a/managed/services/management/rds.go +++ b/managed/services/management/rds.go @@ -274,13 +274,17 @@ func (s *ManagementService) addRDS(ctx context.Context, req *managementv1.AddRDS // add RDSExporter Agent if req.RdsExporter { rdsExporter, err := models.CreateAgent(tx.Querier, models.RDSExporterType, &models.CreateAgentParams{ - PMMAgentID: pmmAgentID, - NodeID: node.NodeID, - AWSAccessKey: req.AwsAccessKey, - AWSSecretKey: req.AwsSecretKey, - RDSBasicMetricsDisabled: req.DisableBasicMetrics, - RDSEnhancedMetricsDisabled: req.DisableEnhancedMetrics, - PushMetrics: isPushMode(metricsMode), + PMMAgentID: pmmAgentID, + NodeID: node.NodeID, + AWSOptions: &models.AWSOptions{ + AWSAccessKey: req.AwsAccessKey, + AWSSecretKey: req.AwsSecretKey, + RDSBasicMetricsDisabled: req.DisableBasicMetrics, + RDSEnhancedMetricsDisabled: req.DisableEnhancedMetrics, + }, + ExporterOptions: &models.ExporterOptions{ + PushMetrics: isPushMode(metricsMode), + }, }) if err != nil { return err @@ -316,14 +320,18 @@ func (s *ManagementService) addRDS(ctx context.Context, req *managementv1.AddRDS // add MySQL Exporter mysqldExporter, err := models.CreateAgent(tx.Querier, models.MySQLdExporterType, &models.CreateAgentParams{ - PMMAgentID: pmmAgentID, - ServiceID: service.ServiceID, - Username: req.Username, - Password: req.Password, - TLS: req.Tls, - TLSSkipVerify: req.TlsSkipVerify, - TableCountTablestatsGroupLimit: tablestatsGroupTableLimit, - PushMetrics: isPushMode(metricsMode), + PMMAgentID: pmmAgentID, + ServiceID: service.ServiceID, + Username: req.Username, + Password: req.Password, + TLS: req.Tls, + TLSSkipVerify: req.TlsSkipVerify, + ExporterOptions: &models.ExporterOptions{ + PushMetrics: isPushMode(metricsMode), + }, + MySQLOptions: &models.MySQLOptions{ + TableCountTablestatsGroupLimit: tablestatsGroupTableLimit, + }, }) if err != nil { return err @@ -346,14 +354,16 @@ func (s *ManagementService) addRDS(ctx context.Context, req *managementv1.AddRDS // add MySQL PerfSchema QAN Agent if req.QanMysqlPerfschema { qanAgent, err := models.CreateAgent(tx.Querier, models.QANMySQLPerfSchemaAgentType, &models.CreateAgentParams{ - PMMAgentID: pmmAgentID, - ServiceID: service.ServiceID, - Username: req.Username, - Password: req.Password, - TLS: req.Tls, - TLSSkipVerify: req.TlsSkipVerify, - QueryExamplesDisabled: req.DisableQueryExamples, - CommentsParsingDisabled: req.DisableCommentsParsing, + PMMAgentID: pmmAgentID, + ServiceID: service.ServiceID, + Username: req.Username, + Password: req.Password, + TLS: req.Tls, + TLSSkipVerify: req.TlsSkipVerify, + QANOptions: &models.QANOptions{ + QueryExamplesDisabled: req.DisableQueryExamples, + CommentsParsingDisabled: req.DisableCommentsParsing, + }, }) if err != nil { return err @@ -391,18 +401,23 @@ func (s *ManagementService) addRDS(ctx context.Context, req *managementv1.AddRDS // add PostgreSQL Exporter postgresExporter, err := models.CreateAgent(tx.Querier, models.PostgresExporterType, &models.CreateAgentParams{ - PMMAgentID: pmmAgentID, - ServiceID: service.ServiceID, - Username: req.Username, - Password: req.Password, - TLS: req.Tls, - TLSSkipVerify: req.TlsSkipVerify, - TableCountTablestatsGroupLimit: tablestatsGroupTableLimit, + PMMAgentID: pmmAgentID, + ServiceID: service.ServiceID, + Username: req.Username, + Password: req.Password, + TLS: req.Tls, + TLSSkipVerify: req.TlsSkipVerify, + ExporterOptions: &models.ExporterOptions{ + PushMetrics: isPushMode(metricsMode), + }, + AWSOptions: &models.AWSOptions{ + TableCountTablestatsGroupLimit: tablestatsGroupTableLimit, + }, + PostgreSQLOptions: &models.PostgreSQLOptions{ AutoDiscoveryLimit: req.AutoDiscoveryLimit, MaxExporterConnections: req.MaxPostgresqlExporterConnections, }, - PushMetrics: isPushMode(metricsMode), }) if err != nil { return err From a38d44daf25aa05abc0369df167fdc21e12e5511 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Mon, 25 Nov 2024 11:37:42 +0100 Subject: [PATCH 05/60] PMM-5086-12634 Another changes. --- Makefile | 4 +- managed/models/agent_helpers.go | 4 +- managed/models/agent_helpers_test.go | 112 ++++---- managed/models/agent_model_reform.go | 123 ++------- managed/services/converters.go | 88 +++--- managed/services/inventory/agents.go | 254 ++++++++++-------- managed/services/management/agent.go | 30 +-- managed/services/management/azure_database.go | 32 ++- managed/services/management/rds.go | 21 +- .../victoriametrics/scrape_configs_test.go | 12 +- 10 files changed, 334 insertions(+), 346 deletions(-) diff --git a/Makefile b/Makefile index 6dbf9abe63..b57f52798f 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ endif env-up: ## Start devcontainer COMPOSE_PROFILES=$(PROFILES) \ - docker compose up -d --wait --wait-timeout 100 + docker compose up -d --wait env-up-rebuild: env-update-image ## Rebuild and start devcontainer. Useful for custom $PMM_SERVER_IMAGE COMPOSE_PROFILES=$(PROFILES) \ @@ -20,7 +20,7 @@ env-update-image: ## Pull latest dev image env-compose-up: env-update-image COMPOSE_PROFILES=$(PROFILES) \ - docker compose up --detach --renew-anon-volumes --remove-orphans --wait --wait-timeout 100 + docker compose up --detach --renew-anon-volumes --remove-orphans --wait env-devcontainer: docker exec -it --workdir=/root/go/src/github.com/percona/pmm pmm-server .devcontainer/setup.py diff --git a/managed/models/agent_helpers.go b/managed/models/agent_helpers.go index 6a64e87d16..080cb6b6da 100644 --- a/managed/models/agent_helpers.go +++ b/managed/models/agent_helpers.go @@ -646,12 +646,12 @@ func CreateNodeExporter(q *reform.Querier, PMMAgentID: &pmmAgentID, NodeID: pmmAgent.RunsOnNodeID, AgentPassword: agentPassword, - LogLevel: pointer.ToStringOrNil(logLevel), ExporterOptions: &models.ExporterOptions{ ExposeExporter: exposeExporter, PushMetrics: pushMetrics, DisabledCollectors: disableCollectors, }, + LogLevel: pointer.ToStringOrNil(logLevel), } if err := row.SetCustomLabels(customLabels); err != nil { return nil, err @@ -907,7 +907,6 @@ func CreateAgent(q *reform.Querier, agentType AgentType, params *CreateAgentPara AgentPassword: pointer.ToStringOrNil(params.AgentPassword), TLS: params.TLS, TLSSkipVerify: params.TLSSkipVerify, - LogLevel: pointer.ToStringOrNil(params.LogLevel), ExporterOptions: params.ExporterOptions, QANOptions: params.QANOptions, AWSOptions: params.AWSOptions, @@ -915,6 +914,7 @@ func CreateAgent(q *reform.Querier, agentType AgentType, params *CreateAgentPara MongoDBOptions: params.MongoDBOptions, MySQLOptions: params.MySQLOptions, PostgreSQLOptions: params.PostgreSQLOptions, + LogLevel: pointer.ToStringOrNil(params.LogLevel), } if err := row.SetCustomLabels(params.CustomLabels); err != nil { return nil, err diff --git a/managed/models/agent_helpers_test.go b/managed/models/agent_helpers_test.go index fb9f0e6645..32516f1c14 100644 --- a/managed/models/agent_helpers_test.go +++ b/managed/models/agent_helpers_test.go @@ -103,8 +103,10 @@ func TestAgentHelpers(t *testing.T) { PMMAgentID: pointer.ToString("A4"), RunsOnNodeID: nil, NodeID: pointer.ToString("N2"), - PushMetrics: true, - ListenPort: pointer.ToUint16(8200), + ExporterOptions: &models.ExporterOptions{ + PushMetrics: true, + }, + ListenPort: pointer.ToUint16(8200), }, &models.Agent{ AgentID: "A6", @@ -112,8 +114,10 @@ func TestAgentHelpers(t *testing.T) { PMMAgentID: pointer.ToString("A4"), RunsOnNodeID: nil, NodeID: pointer.ToString("N2"), - PushMetrics: false, - ListenPort: pointer.ToUint16(8200), + ExporterOptions: &models.ExporterOptions{ + PushMetrics: false, + }, + ListenPort: pointer.ToUint16(8200), }, &models.Agent{ AgentID: "A7", @@ -121,20 +125,22 @@ func TestAgentHelpers(t *testing.T) { PMMAgentID: pointer.ToString("A4"), RunsOnNodeID: nil, NodeID: pointer.ToString("N1"), - PushMetrics: false, ListenPort: pointer.ToUint16(8200), TLS: true, TLSSkipVerify: true, + ExporterOptions: &models.ExporterOptions{ + PushMetrics: false, + MetricsResolutions: &models.MetricsResolutions{ + HR: 1 * time.Minute, + MR: 5 * time.Minute, + LR: 15 * time.Minute, + }, + }, PostgreSQLOptions: &models.PostgreSQLOptions{ SSLCa: "ssl_ca", SSLCert: "ssl_cert", SSLKey: "ssl_key", }, - MetricsResolutions: &models.MetricsResolutions{ - HR: 1 * time.Minute, - MR: 5 * time.Minute, - LR: 15 * time.Minute, - }, }, &models.Agent{ AgentID: "A8", @@ -142,10 +148,12 @@ func TestAgentHelpers(t *testing.T) { PMMAgentID: pointer.ToString("A8"), RunsOnNodeID: nil, NodeID: pointer.ToString("N1"), - PushMetrics: false, ListenPort: pointer.ToUint16(8200), TLS: true, TLSSkipVerify: true, + ExporterOptions: &models.ExporterOptions{ + PushMetrics: false, + }, MongoDBOptions: &models.MongoDBOptions{ TLSCertificateKey: "tls_certificate_key", TLSCertificateKeyFilePassword: "tls_certificate_key_file_password", @@ -162,10 +170,12 @@ func TestAgentHelpers(t *testing.T) { PMMAgentID: pointer.ToString("A9"), RunsOnNodeID: nil, NodeID: pointer.ToString("N1"), - PushMetrics: false, ListenPort: pointer.ToUint16(8200), TLS: true, TLSSkipVerify: true, + ExporterOptions: &models.ExporterOptions{ + PushMetrics: false, + }, MongoDBOptions: &models.MongoDBOptions{ TLSCertificateKey: "tls_certificate_key", TLSCertificateKeyFilePassword: "tls_certificate_key_file_password", @@ -178,15 +188,17 @@ func TestAgentHelpers(t *testing.T) { }, }, &models.Agent{ - AgentID: "A10", - AgentType: models.MongoDBExporterType, - PMMAgentID: pointer.ToString("A10"), - RunsOnNodeID: nil, - NodeID: pointer.ToString("N1"), - PushMetrics: false, - ListenPort: pointer.ToUint16(8200), - TLS: true, - TLSSkipVerify: true, + AgentID: "A10", + AgentType: models.MongoDBExporterType, + PMMAgentID: pointer.ToString("A10"), + RunsOnNodeID: nil, + NodeID: pointer.ToString("N1"), + ListenPort: pointer.ToUint16(8200), + TLS: true, + TLSSkipVerify: true, + ExporterOptions: &models.ExporterOptions{ + PushMetrics: false, + }, MongoDBOptions: nil, // this test is specific for nil MongoDBOptions }, } { @@ -208,18 +220,20 @@ func TestAgentHelpers(t *testing.T) { require.NoError(t, err) expected := []*models.Agent{ { - CreatedAt: now, - UpdatedAt: now, - Status: models.AgentStatusUnknown, - AgentID: "A10", - AgentType: models.MongoDBExporterType, - PMMAgentID: pointer.ToString("A10"), - RunsOnNodeID: nil, - NodeID: pointer.ToString("N1"), - PushMetrics: false, - ListenPort: pointer.ToUint16(8200), - TLS: true, - TLSSkipVerify: true, + CreatedAt: now, + UpdatedAt: now, + Status: models.AgentStatusUnknown, + AgentID: "A10", + AgentType: models.MongoDBExporterType, + PMMAgentID: pointer.ToString("A10"), + RunsOnNodeID: nil, + NodeID: pointer.ToString("N1"), + ListenPort: pointer.ToUint16(8200), + TLS: true, + TLSSkipVerify: true, + ExporterOptions: &models.ExporterOptions{ + PushMetrics: false, + }, MongoDBOptions: nil, // this test is specific for nil MongoDBOptions }, { @@ -243,16 +257,18 @@ func TestAgentHelpers(t *testing.T) { ListenPort: pointer.ToUint16OrNil(8200), TLS: true, TLSSkipVerify: true, + ExporterOptions: &models.ExporterOptions{ + MetricsResolutions: &models.MetricsResolutions{ + HR: 1 * time.Minute, + MR: 5 * time.Minute, + LR: 15 * time.Minute, + }, + }, PostgreSQLOptions: &models.PostgreSQLOptions{ SSLCa: "ssl_ca", SSLCert: "ssl_cert", SSLKey: "ssl_key", }, - MetricsResolutions: &models.MetricsResolutions{ - HR: 1 * time.Minute, - MR: 5 * time.Minute, - LR: 15 * time.Minute, - }, }, { AgentID: "A8", @@ -468,15 +484,17 @@ func TestAgentHelpers(t *testing.T) { }) require.NoError(t, err) assert.Equal(t, &models.Agent{ - AgentID: agent.AgentID, - AgentType: models.ExternalExporterType, - RunsOnNodeID: pointer.ToString("N1"), - ServiceID: pointer.ToString("S1"), - ListenPort: pointer.ToUint16(9104), - MetricsPath: pointer.ToString("/metrics"), - MetricsScheme: pointer.ToString("http"), - CreatedAt: now, - UpdatedAt: now, + AgentID: agent.AgentID, + AgentType: models.ExternalExporterType, + RunsOnNodeID: pointer.ToString("N1"), + ServiceID: pointer.ToString("S1"), + ListenPort: pointer.ToUint16(9104), + ExporterOptions: &models.ExporterOptions{ + MetricsPath: "/metrics", + MetricsScheme: "http", + }, + CreatedAt: now, + UpdatedAt: now, }, agent) }) t.Run("Invalid listen port", func(t *testing.T) { diff --git a/managed/models/agent_model_reform.go b/managed/models/agent_model_reform.go index 0f6c88c440..f916bb2d3b 100644 --- a/managed/models/agent_model_reform.go +++ b/managed/models/agent_model_reform.go @@ -47,27 +47,14 @@ func (v *agentTableType) Columns() []string { "agent_password", "tls", "tls_skip_verify", - "aws_access_key", - "aws_secret_key", + "log_level", + "exporter_options", + "qan_options", + "aws_options", "azure_options", - "table_count", - "table_count_tablestats_group_limit", - "max_query_length", - "query_examples_disabled", - "comments_parsing_disabled", - "max_query_log_size", - "metrics_path", - "metrics_scheme", - "rds_basic_metrics_disabled", - "rds_enhanced_metrics_disabled", - "push_metrics", - "disabled_collectors", - "metrics_resolutions", - "mysql_options", "mongo_db_tls_options", + "mysql_options", "postgresql_options", - "log_level", - "expose_exporter", } } @@ -111,27 +98,14 @@ var AgentTable = &agentTableType{ {Name: "AgentPassword", Type: "*string", Column: "agent_password"}, {Name: "TLS", Type: "bool", Column: "tls"}, {Name: "TLSSkipVerify", Type: "bool", Column: "tls_skip_verify"}, - {Name: "AWSAccessKey", Type: "*string", Column: "aws_access_key"}, - {Name: "AWSSecretKey", Type: "*string", Column: "aws_secret_key"}, + {Name: "LogLevel", Type: "*string", Column: "log_level"}, + {Name: "ExporterOptions", Type: "*ExporterOptions", Column: "exporter_options"}, + {Name: "QANOptions", Type: "*QANOptions", Column: "qan_options"}, + {Name: "AWSOptions", Type: "*AWSOptions", Column: "aws_options"}, {Name: "AzureOptions", Type: "*AzureOptions", Column: "azure_options"}, - {Name: "TableCount", Type: "*int32", Column: "table_count"}, - {Name: "TableCountTablestatsGroupLimit", Type: "int32", Column: "table_count_tablestats_group_limit"}, - {Name: "MaxQueryLength", Type: "int32", Column: "max_query_length"}, - {Name: "QueryExamplesDisabled", Type: "bool", Column: "query_examples_disabled"}, - {Name: "CommentsParsingDisabled", Type: "bool", Column: "comments_parsing_disabled"}, - {Name: "MaxQueryLogSize", Type: "int64", Column: "max_query_log_size"}, - {Name: "MetricsPath", Type: "*string", Column: "metrics_path"}, - {Name: "MetricsScheme", Type: "*string", Column: "metrics_scheme"}, - {Name: "RDSBasicMetricsDisabled", Type: "bool", Column: "rds_basic_metrics_disabled"}, - {Name: "RDSEnhancedMetricsDisabled", Type: "bool", Column: "rds_enhanced_metrics_disabled"}, - {Name: "PushMetrics", Type: "bool", Column: "push_metrics"}, - {Name: "DisabledCollectors", Type: "pq.StringArray", Column: "disabled_collectors"}, - {Name: "MetricsResolutions", Type: "*MetricsResolutions", Column: "metrics_resolutions"}, - {Name: "MySQLOptions", Type: "*MySQLOptions", Column: "mysql_options"}, {Name: "MongoDBOptions", Type: "*MongoDBOptions", Column: "mongo_db_tls_options"}, + {Name: "MySQLOptions", Type: "*MySQLOptions", Column: "mysql_options"}, {Name: "PostgreSQLOptions", Type: "*PostgreSQLOptions", Column: "postgresql_options"}, - {Name: "LogLevel", Type: "*string", Column: "log_level"}, - {Name: "ExposeExporter", Type: "bool", Column: "expose_exporter"}, }, PKFieldIndex: 0, }, @@ -140,7 +114,7 @@ var AgentTable = &agentTableType{ // String returns a string representation of this struct or record. func (s Agent) String() string { - res := make([]string, 40) + res := make([]string, 27) res[0] = "AgentID: " + reform.Inspect(s.AgentID, true) res[1] = "AgentType: " + reform.Inspect(s.AgentType, true) res[2] = "RunsOnNodeID: " + reform.Inspect(s.RunsOnNodeID, true) @@ -160,27 +134,14 @@ func (s Agent) String() string { res[16] = "AgentPassword: " + reform.Inspect(s.AgentPassword, true) res[17] = "TLS: " + reform.Inspect(s.TLS, true) res[18] = "TLSSkipVerify: " + reform.Inspect(s.TLSSkipVerify, true) - res[19] = "AWSAccessKey: " + reform.Inspect(s.AWSAccessKey, true) - res[20] = "AWSSecretKey: " + reform.Inspect(s.AWSSecretKey, true) - res[21] = "AzureOptions: " + reform.Inspect(s.AzureOptions, true) - res[22] = "TableCount: " + reform.Inspect(s.TableCount, true) - res[23] = "TableCountTablestatsGroupLimit: " + reform.Inspect(s.TableCountTablestatsGroupLimit, true) - res[24] = "MaxQueryLength: " + reform.Inspect(s.MaxQueryLength, true) - res[25] = "QueryExamplesDisabled: " + reform.Inspect(s.QueryExamplesDisabled, true) - res[26] = "CommentsParsingDisabled: " + reform.Inspect(s.CommentsParsingDisabled, true) - res[27] = "MaxQueryLogSize: " + reform.Inspect(s.MaxQueryLogSize, true) - res[28] = "MetricsPath: " + reform.Inspect(s.MetricsPath, true) - res[29] = "MetricsScheme: " + reform.Inspect(s.MetricsScheme, true) - res[30] = "RDSBasicMetricsDisabled: " + reform.Inspect(s.RDSBasicMetricsDisabled, true) - res[31] = "RDSEnhancedMetricsDisabled: " + reform.Inspect(s.RDSEnhancedMetricsDisabled, true) - res[32] = "PushMetrics: " + reform.Inspect(s.PushMetrics, true) - res[33] = "DisabledCollectors: " + reform.Inspect(s.DisabledCollectors, true) - res[34] = "MetricsResolutions: " + reform.Inspect(s.MetricsResolutions, true) - res[35] = "MySQLOptions: " + reform.Inspect(s.MySQLOptions, true) - res[36] = "MongoDBOptions: " + reform.Inspect(s.MongoDBOptions, true) - res[37] = "PostgreSQLOptions: " + reform.Inspect(s.PostgreSQLOptions, true) - res[38] = "LogLevel: " + reform.Inspect(s.LogLevel, true) - res[39] = "ExposeExporter: " + reform.Inspect(s.ExposeExporter, true) + res[19] = "LogLevel: " + reform.Inspect(s.LogLevel, true) + res[20] = "ExporterOptions: " + reform.Inspect(s.ExporterOptions, true) + res[21] = "QANOptions: " + reform.Inspect(s.QANOptions, true) + res[22] = "AWSOptions: " + reform.Inspect(s.AWSOptions, true) + res[23] = "AzureOptions: " + reform.Inspect(s.AzureOptions, true) + res[24] = "MongoDBOptions: " + reform.Inspect(s.MongoDBOptions, true) + res[25] = "MySQLOptions: " + reform.Inspect(s.MySQLOptions, true) + res[26] = "PostgreSQLOptions: " + reform.Inspect(s.PostgreSQLOptions, true) return strings.Join(res, ", ") } @@ -207,27 +168,14 @@ func (s *Agent) Values() []interface{} { s.AgentPassword, s.TLS, s.TLSSkipVerify, - s.AWSAccessKey, - s.AWSSecretKey, + s.LogLevel, + s.ExporterOptions, + s.QANOptions, + s.AWSOptions, s.AzureOptions, - s.TableCount, - s.TableCountTablestatsGroupLimit, - s.MaxQueryLength, - s.QueryExamplesDisabled, - s.CommentsParsingDisabled, - s.MaxQueryLogSize, - s.MetricsPath, - s.MetricsScheme, - s.RDSBasicMetricsDisabled, - s.RDSEnhancedMetricsDisabled, - s.PushMetrics, - s.DisabledCollectors, - s.MetricsResolutions, - s.MySQLOptions, s.MongoDBOptions, + s.MySQLOptions, s.PostgreSQLOptions, - s.LogLevel, - s.ExposeExporter, } } @@ -254,27 +202,14 @@ func (s *Agent) Pointers() []interface{} { &s.AgentPassword, &s.TLS, &s.TLSSkipVerify, - &s.AWSAccessKey, - &s.AWSSecretKey, + &s.LogLevel, + &s.ExporterOptions, + &s.QANOptions, + &s.AWSOptions, &s.AzureOptions, - &s.TableCount, - &s.TableCountTablestatsGroupLimit, - &s.MaxQueryLength, - &s.QueryExamplesDisabled, - &s.CommentsParsingDisabled, - &s.MaxQueryLogSize, - &s.MetricsPath, - &s.MetricsScheme, - &s.RDSBasicMetricsDisabled, - &s.RDSEnhancedMetricsDisabled, - &s.PushMetrics, - &s.DisabledCollectors, - &s.MetricsResolutions, - &s.MySQLOptions, &s.MongoDBOptions, + &s.MySQLOptions, &s.PostgreSQLOptions, - &s.LogLevel, - &s.ExposeExporter, } } diff --git a/managed/services/converters.go b/managed/services/converters.go index 10589cc07f..280e274746 100644 --- a/managed/services/converters.go +++ b/managed/services/converters.go @@ -236,12 +236,12 @@ func ToAPIAgent(q *reform.Querier, agent *models.Agent) (inventoryv1.Agent, erro Status: inventoryv1.AgentStatus(inventoryv1.AgentStatus_value[agent.Status]), ListenPort: uint32(pointer.GetUint16(agent.ListenPort)), CustomLabels: labels, - PushMetricsEnabled: agent.PushMetrics, - DisabledCollectors: agent.DisabledCollectors, + PushMetricsEnabled: agent.ExporterOptions.PushMetrics, + DisabledCollectors: agent.ExporterOptions.DisabledCollectors, ProcessExecPath: processExecPath, LogLevel: inventoryv1.LogLevelAPIValue(agent.LogLevel), - ExposeExporter: agent.ExposeExporter, - MetricsResolutions: ConvertMetricsResolutions(agent.MetricsResolutions), + ExposeExporter: agent.ExporterOptions.ExposeExporter, + MetricsResolutions: ConvertMetricsResolutions(agent.ExporterOptions.MetricsResolutions), }, nil case models.MySQLdExporterType: @@ -256,15 +256,15 @@ func ToAPIAgent(q *reform.Querier, agent *models.Agent) (inventoryv1.Agent, erro CustomLabels: labels, Tls: agent.TLS, TlsSkipVerify: agent.TLSSkipVerify, - TablestatsGroupTableLimit: agent.TableCountTablestatsGroupLimit, + TablestatsGroupTableLimit: agent.MySQLOptions.TableCountTablestatsGroupLimit, TablestatsGroupDisabled: !agent.IsMySQLTablestatsGroupEnabled(), - TableCount: pointer.GetInt32(agent.TableCount), - PushMetricsEnabled: agent.PushMetrics, - DisabledCollectors: agent.DisabledCollectors, + TableCount: pointer.GetInt32(agent.MySQLOptions.TableCount), + PushMetricsEnabled: agent.ExporterOptions.PushMetrics, + DisabledCollectors: agent.ExporterOptions.DisabledCollectors, ProcessExecPath: processExecPath, LogLevel: inventoryv1.LogLevelAPIValue(agent.LogLevel), - ExposeExporter: agent.ExposeExporter, - MetricsResolutions: ConvertMetricsResolutions(agent.MetricsResolutions), + ExposeExporter: agent.ExporterOptions.ExposeExporter, + MetricsResolutions: ConvertMetricsResolutions(agent.ExporterOptions.MetricsResolutions), }, nil case models.MongoDBExporterType: @@ -279,12 +279,12 @@ func ToAPIAgent(q *reform.Querier, agent *models.Agent) (inventoryv1.Agent, erro CustomLabels: labels, Tls: agent.TLS, TlsSkipVerify: agent.TLSSkipVerify, - PushMetricsEnabled: agent.PushMetrics, - DisabledCollectors: agent.DisabledCollectors, + PushMetricsEnabled: agent.ExporterOptions.PushMetrics, + DisabledCollectors: agent.ExporterOptions.DisabledCollectors, ProcessExecPath: processExecPath, LogLevel: inventoryv1.LogLevelAPIValue(agent.LogLevel), - ExposeExporter: agent.ExposeExporter, - MetricsResolutions: ConvertMetricsResolutions(agent.MetricsResolutions), + ExposeExporter: agent.ExporterOptions.ExposeExporter, + MetricsResolutions: ConvertMetricsResolutions(agent.ExporterOptions.MetricsResolutions), } if agent.MongoDBOptions != nil { exporter.StatsCollections = agent.MongoDBOptions.StatsCollections @@ -305,12 +305,12 @@ func ToAPIAgent(q *reform.Querier, agent *models.Agent) (inventoryv1.Agent, erro CustomLabels: labels, Tls: agent.TLS, TlsSkipVerify: agent.TLSSkipVerify, - PushMetricsEnabled: agent.PushMetrics, - DisabledCollectors: agent.DisabledCollectors, + PushMetricsEnabled: agent.ExporterOptions.PushMetrics, + DisabledCollectors: agent.ExporterOptions.DisabledCollectors, ProcessExecPath: processExecPath, LogLevel: inventoryv1.LogLevelAPIValue(agent.LogLevel), - ExposeExporter: agent.ExposeExporter, - MetricsResolutions: ConvertMetricsResolutions(agent.MetricsResolutions), + ExposeExporter: agent.ExporterOptions.ExposeExporter, + MetricsResolutions: ConvertMetricsResolutions(agent.ExporterOptions.MetricsResolutions), } if agent.PostgreSQLOptions != nil { exporter.AutoDiscoveryLimit = agent.PostgreSQLOptions.AutoDiscoveryLimit @@ -328,9 +328,9 @@ func ToAPIAgent(q *reform.Querier, agent *models.Agent) (inventoryv1.Agent, erro CustomLabels: labels, Tls: agent.TLS, TlsSkipVerify: agent.TLSSkipVerify, - MaxQueryLength: agent.MaxQueryLength, - QueryExamplesDisabled: agent.QueryExamplesDisabled, - DisableCommentsParsing: agent.CommentsParsingDisabled, + MaxQueryLength: agent.QANOptions.MaxQueryLength, + QueryExamplesDisabled: agent.QANOptions.QueryExamplesDisabled, + DisableCommentsParsing: agent.QANOptions.CommentsParsingDisabled, ProcessExecPath: processExecPath, LogLevel: inventoryv1.LogLevelAPIValue(agent.LogLevel), }, nil @@ -346,9 +346,9 @@ func ToAPIAgent(q *reform.Querier, agent *models.Agent) (inventoryv1.Agent, erro CustomLabels: labels, Tls: agent.TLS, TlsSkipVerify: agent.TLSSkipVerify, - QueryExamplesDisabled: agent.QueryExamplesDisabled, - DisableCommentsParsing: agent.CommentsParsingDisabled, - MaxSlowlogFileSize: agent.MaxQueryLogSize, + QueryExamplesDisabled: agent.QANOptions.QueryExamplesDisabled, + DisableCommentsParsing: agent.QANOptions.CommentsParsingDisabled, + MaxSlowlogFileSize: agent.QANOptions.MaxQueryLogSize, ProcessExecPath: processExecPath, LogLevel: inventoryv1.LogLevelAPIValue(agent.LogLevel), }, nil @@ -364,7 +364,7 @@ func ToAPIAgent(q *reform.Querier, agent *models.Agent) (inventoryv1.Agent, erro CustomLabels: labels, Tls: agent.TLS, TlsSkipVerify: agent.TLSSkipVerify, - MaxQueryLength: agent.MaxQueryLength, + MaxQueryLength: agent.QANOptions.MaxQueryLength, ProcessExecPath: processExecPath, LogLevel: inventoryv1.LogLevelAPIValue(agent.LogLevel), // TODO QueryExamplesDisabled https://jira.percona.com/browse/PMM-4650 @@ -382,12 +382,12 @@ func ToAPIAgent(q *reform.Querier, agent *models.Agent) (inventoryv1.Agent, erro CustomLabels: labels, Tls: agent.TLS, TlsSkipVerify: agent.TLSSkipVerify, - PushMetricsEnabled: agent.PushMetrics, - DisabledCollectors: agent.DisabledCollectors, + PushMetricsEnabled: agent.ExporterOptions.PushMetrics, + DisabledCollectors: agent.ExporterOptions.DisabledCollectors, ProcessExecPath: processExecPath, LogLevel: inventoryv1.LogLevelAPIValue(agent.LogLevel), - ExposeExporter: agent.ExposeExporter, - MetricsResolutions: ConvertMetricsResolutions(agent.MetricsResolutions), + ExposeExporter: agent.ExporterOptions.ExposeExporter, + MetricsResolutions: ConvertMetricsResolutions(agent.ExporterOptions.MetricsResolutions), }, nil case models.QANPostgreSQLPgStatementsAgentType: @@ -399,8 +399,8 @@ func ToAPIAgent(q *reform.Querier, agent *models.Agent) (inventoryv1.Agent, erro Disabled: agent.Disabled, Status: inventoryv1.AgentStatus(inventoryv1.AgentStatus_value[agent.Status]), CustomLabels: labels, - MaxQueryLength: agent.MaxQueryLength, - DisableCommentsParsing: agent.CommentsParsingDisabled, + MaxQueryLength: agent.QANOptions.MaxQueryLength, + DisableCommentsParsing: agent.QANOptions.CommentsParsingDisabled, Tls: agent.TLS, TlsSkipVerify: agent.TLSSkipVerify, ProcessExecPath: processExecPath, @@ -416,11 +416,11 @@ func ToAPIAgent(q *reform.Querier, agent *models.Agent) (inventoryv1.Agent, erro Disabled: agent.Disabled, Status: inventoryv1.AgentStatus(inventoryv1.AgentStatus_value[agent.Status]), CustomLabels: labels, - MaxQueryLength: agent.MaxQueryLength, + MaxQueryLength: agent.QANOptions.MaxQueryLength, Tls: agent.TLS, TlsSkipVerify: agent.TLSSkipVerify, - QueryExamplesDisabled: agent.QueryExamplesDisabled, - DisableCommentsParsing: agent.CommentsParsingDisabled, + QueryExamplesDisabled: agent.QANOptions.QueryExamplesDisabled, + DisableCommentsParsing: agent.QANOptions.CommentsParsingDisabled, ProcessExecPath: processExecPath, LogLevel: inventoryv1.LogLevelAPIValue(agent.LogLevel), }, nil @@ -431,16 +431,16 @@ func ToAPIAgent(q *reform.Querier, agent *models.Agent) (inventoryv1.Agent, erro PmmAgentId: pointer.GetString(agent.PMMAgentID), NodeId: nodeID, Disabled: agent.Disabled, - AwsAccessKey: pointer.GetString(agent.AWSAccessKey), + AwsAccessKey: agent.AWSOptions.AWSAccessKey, Status: inventoryv1.AgentStatus(inventoryv1.AgentStatus_value[agent.Status]), ListenPort: uint32(pointer.GetUint16(agent.ListenPort)), CustomLabels: labels, - BasicMetricsDisabled: agent.RDSBasicMetricsDisabled, - EnhancedMetricsDisabled: agent.RDSEnhancedMetricsDisabled, - PushMetricsEnabled: agent.PushMetrics, + BasicMetricsDisabled: agent.AWSOptions.RDSBasicMetricsDisabled, + EnhancedMetricsDisabled: agent.AWSOptions.RDSEnhancedMetricsDisabled, + PushMetricsEnabled: agent.ExporterOptions.PushMetrics, ProcessExecPath: processExecPath, LogLevel: inventoryv1.LogLevelAPIValue(agent.LogLevel), - MetricsResolutions: ConvertMetricsResolutions(agent.MetricsResolutions), + MetricsResolutions: ConvertMetricsResolutions(agent.ExporterOptions.MetricsResolutions), }, nil case models.ExternalExporterType: @@ -457,13 +457,13 @@ func ToAPIAgent(q *reform.Querier, agent *models.Agent) (inventoryv1.Agent, erro ServiceId: pointer.GetString(agent.ServiceID), Username: pointer.GetString(agent.Username), Disabled: agent.Disabled, - Scheme: pointer.GetString(agent.MetricsScheme), - MetricsPath: pointer.GetString(agent.MetricsPath), + Scheme: agent.ExporterOptions.MetricsScheme, + MetricsPath: agent.ExporterOptions.MetricsPath, ListenPort: uint32(pointer.GetUint16(agent.ListenPort)), CustomLabels: labels, - PushMetricsEnabled: agent.PushMetrics, + PushMetricsEnabled: agent.ExporterOptions.PushMetrics, ProcessExecPath: processExecPath, - MetricsResolutions: ConvertMetricsResolutions(agent.MetricsResolutions), + MetricsResolutions: ConvertMetricsResolutions(agent.ExporterOptions.MetricsResolutions), }, nil case models.AzureDatabaseExporterType: @@ -478,7 +478,7 @@ func ToAPIAgent(q *reform.Querier, agent *models.Agent) (inventoryv1.Agent, erro CustomLabels: labels, ProcessExecPath: processExecPath, LogLevel: inventoryv1.LogLevelAPIValue(agent.LogLevel), - MetricsResolutions: ConvertMetricsResolutions(agent.MetricsResolutions), + MetricsResolutions: ConvertMetricsResolutions(agent.ExporterOptions.MetricsResolutions), }, nil case models.VMAgentType: diff --git a/managed/services/inventory/agents.go b/managed/services/inventory/agents.go index 1b96b51081..9f6a39baaf 100644 --- a/managed/services/inventory/agents.go +++ b/managed/services/inventory/agents.go @@ -262,21 +262,25 @@ func (as *AgentsService) ChangeNodeExporter(ctx context.Context, agentID string, func (as *AgentsService) AddMySQLdExporter(ctx context.Context, p *inventoryv1.AddMySQLdExporterParams) (*inventoryv1.AddAgentResponse, error) { var row *models.Agent var agent *inventoryv1.MySQLdExporter + + mysqlOptions := models.MySQLOptionsFromRequest(p) + mysqlOptions.TableCountTablestatsGroupLimit = p.TablestatsGroupTableLimit e := as.db.InTransactionContext(ctx, nil, func(tx *reform.TX) error { params := &models.CreateAgentParams{ - PMMAgentID: p.PmmAgentId, - ServiceID: p.ServiceId, - Username: p.Username, - Password: p.Password, - AgentPassword: p.AgentPassword, - CustomLabels: p.CustomLabels, - TLS: p.Tls, - TLSSkipVerify: p.TlsSkipVerify, - MySQLOptions: models.MySQLOptionsFromRequest(p), - TableCountTablestatsGroupLimit: p.TablestatsGroupTableLimit, - PushMetrics: p.PushMetrics, - DisableCollectors: p.DisableCollectors, - LogLevel: services.SpecifyLogLevel(p.LogLevel, inventoryv1.LogLevel_LOG_LEVEL_ERROR), + PMMAgentID: p.PmmAgentId, + ServiceID: p.ServiceId, + Username: p.Username, + Password: p.Password, + AgentPassword: p.AgentPassword, + CustomLabels: p.CustomLabels, + TLS: p.Tls, + TLSSkipVerify: p.TlsSkipVerify, + ExporterOptions: &models.ExporterOptions{ + PushMetrics: p.PushMetrics, + DisabledCollectors: p.DisableCollectors, + }, + MySQLOptions: mysqlOptions, + LogLevel: services.SpecifyLogLevel(p.LogLevel, inventoryv1.LogLevel_LOG_LEVEL_ERROR), } var err error row, err = models.CreateAgent(tx.Querier, models.MySQLdExporterType, params) @@ -350,18 +354,20 @@ func (as *AgentsService) AddMongoDBExporter(ctx context.Context, p *inventoryv1. var agent *inventoryv1.MongoDBExporter e := as.db.InTransactionContext(ctx, nil, func(tx *reform.TX) error { params := &models.CreateAgentParams{ - PMMAgentID: p.PmmAgentId, - ServiceID: p.ServiceId, - Username: p.Username, - Password: p.Password, - AgentPassword: p.AgentPassword, - CustomLabels: p.CustomLabels, - TLS: p.Tls, - TLSSkipVerify: p.TlsSkipVerify, - MongoDBOptions: models.MongoDBOptionsFromRequest(p), - PushMetrics: p.PushMetrics, - DisableCollectors: p.DisableCollectors, - LogLevel: services.SpecifyLogLevel(p.LogLevel, inventoryv1.LogLevel_LOG_LEVEL_FATAL), + PMMAgentID: p.PmmAgentId, + ServiceID: p.ServiceId, + Username: p.Username, + Password: p.Password, + AgentPassword: p.AgentPassword, + CustomLabels: p.CustomLabels, + TLS: p.Tls, + TLSSkipVerify: p.TlsSkipVerify, + MongoDBOptions: models.MongoDBOptionsFromRequest(p), + ExporterOptions: &models.ExporterOptions{ + PushMetrics: p.PushMetrics, + DisabledCollectors: p.DisableCollectors, + }, + LogLevel: services.SpecifyLogLevel(p.LogLevel, inventoryv1.LogLevel_LOG_LEVEL_FATAL), } row, err := models.CreateAgent(tx.Querier, models.MongoDBExporterType, params) if err != nil { @@ -434,18 +440,20 @@ func (as *AgentsService) AddQANMySQLPerfSchemaAgent(ctx context.Context, p *inve var agent *inventoryv1.QANMySQLPerfSchemaAgent e := as.db.InTransactionContext(ctx, nil, func(tx *reform.TX) error { params := &models.CreateAgentParams{ - PMMAgentID: p.PmmAgentId, - ServiceID: p.ServiceId, - Username: p.Username, - Password: p.Password, - CustomLabels: p.CustomLabels, - TLS: p.Tls, - TLSSkipVerify: p.TlsSkipVerify, - MySQLOptions: models.MySQLOptionsFromRequest(p), - MaxQueryLength: p.MaxQueryLength, - QueryExamplesDisabled: p.DisableQueryExamples, - CommentsParsingDisabled: p.DisableCommentsParsing, - LogLevel: services.SpecifyLogLevel(p.LogLevel, inventoryv1.LogLevel_LOG_LEVEL_FATAL), + PMMAgentID: p.PmmAgentId, + ServiceID: p.ServiceId, + Username: p.Username, + Password: p.Password, + CustomLabels: p.CustomLabels, + TLS: p.Tls, + TLSSkipVerify: p.TlsSkipVerify, + QANOptions: &models.QANOptions{ + MaxQueryLength: p.MaxQueryLength, + QueryExamplesDisabled: p.DisableQueryExamples, + CommentsParsingDisabled: p.DisableCommentsParsing, + }, + MySQLOptions: models.MySQLOptionsFromRequest(p), + LogLevel: services.SpecifyLogLevel(p.LogLevel, inventoryv1.LogLevel_LOG_LEVEL_FATAL), } row, err := models.CreateAgent(tx.Querier, models.QANMySQLPerfSchemaAgentType, params) if err != nil { @@ -518,19 +526,21 @@ func (as *AgentsService) AddQANMySQLSlowlogAgent(ctx context.Context, p *invento } params := &models.CreateAgentParams{ - PMMAgentID: p.PmmAgentId, - ServiceID: p.ServiceId, - Username: p.Username, - Password: p.Password, - CustomLabels: p.CustomLabels, - TLS: p.Tls, - TLSSkipVerify: p.TlsSkipVerify, - MySQLOptions: models.MySQLOptionsFromRequest(p), - MaxQueryLength: p.MaxQueryLength, - QueryExamplesDisabled: p.DisableQueryExamples, - CommentsParsingDisabled: p.DisableCommentsParsing, - MaxQueryLogSize: maxSlowlogFileSize, - LogLevel: services.SpecifyLogLevel(p.LogLevel, inventoryv1.LogLevel_LOG_LEVEL_FATAL), + PMMAgentID: p.PmmAgentId, + ServiceID: p.ServiceId, + Username: p.Username, + Password: p.Password, + CustomLabels: p.CustomLabels, + TLS: p.Tls, + TLSSkipVerify: p.TlsSkipVerify, + QANOptions: &models.QANOptions{ + MaxQueryLength: p.MaxQueryLength, + QueryExamplesDisabled: p.DisableQueryExamples, + CommentsParsingDisabled: p.DisableCommentsParsing, + MaxQueryLogSize: maxSlowlogFileSize, + }, + MySQLOptions: models.MySQLOptionsFromRequest(p), + LogLevel: services.SpecifyLogLevel(p.LogLevel, inventoryv1.LogLevel_LOG_LEVEL_FATAL), } row, err := models.CreateAgent(tx.Querier, models.QANMySQLSlowlogAgentType, params) if err != nil { @@ -597,16 +607,18 @@ func (as *AgentsService) AddPostgresExporter(ctx context.Context, p *inventoryv1 var agent *inventoryv1.PostgresExporter e := as.db.InTransactionContext(ctx, nil, func(tx *reform.TX) error { params := &models.CreateAgentParams{ - PMMAgentID: p.PmmAgentId, - ServiceID: p.ServiceId, - Username: p.Username, - Password: p.Password, - AgentPassword: p.AgentPassword, - CustomLabels: p.CustomLabels, - TLS: p.Tls, - TLSSkipVerify: p.TlsSkipVerify, - PushMetrics: p.PushMetrics, - DisableCollectors: p.DisableCollectors, + PMMAgentID: p.PmmAgentId, + ServiceID: p.ServiceId, + Username: p.Username, + Password: p.Password, + AgentPassword: p.AgentPassword, + CustomLabels: p.CustomLabels, + TLS: p.Tls, + TLSSkipVerify: p.TlsSkipVerify, + ExporterOptions: &models.ExporterOptions{ + PushMetrics: p.PushMetrics, + DisabledCollectors: p.DisableCollectors, + }, PostgreSQLOptions: models.PostgreSQLOptionsFromRequest(p), LogLevel: services.SpecifyLogLevel(p.LogLevel, inventoryv1.LogLevel_LOG_LEVEL_ERROR), } @@ -681,17 +693,19 @@ func (as *AgentsService) AddQANMongoDBProfilerAgent(ctx context.Context, p *inve e := as.db.InTransactionContext(ctx, nil, func(tx *reform.TX) error { params := &models.CreateAgentParams{ - PMMAgentID: p.PmmAgentId, - ServiceID: p.ServiceId, - Username: p.Username, - Password: p.Password, - CustomLabels: p.CustomLabels, - TLS: p.Tls, - TLSSkipVerify: p.TlsSkipVerify, + PMMAgentID: p.PmmAgentId, + ServiceID: p.ServiceId, + Username: p.Username, + Password: p.Password, + CustomLabels: p.CustomLabels, + TLS: p.Tls, + TLSSkipVerify: p.TlsSkipVerify, + QANOptions: &models.QANOptions{ + MaxQueryLength: p.MaxQueryLength, + // TODO QueryExamplesDisabled https://jira.percona.com/browse/PMM-4650 - done, but not included in params. + }, MongoDBOptions: models.MongoDBOptionsFromRequest(p), - MaxQueryLength: p.MaxQueryLength, LogLevel: services.SpecifyLogLevel(p.LogLevel, inventoryv1.LogLevel_LOG_LEVEL_FATAL), - // TODO QueryExamplesDisabled https://jira.percona.com/browse/PMM-4650 } row, err := models.CreateAgent(tx.Querier, models.QANMongoDBProfilerAgentType, params) if err != nil { @@ -760,17 +774,19 @@ func (as *AgentsService) AddProxySQLExporter(ctx context.Context, p *inventoryv1 var agent *inventoryv1.ProxySQLExporter e := as.db.InTransactionContext(ctx, nil, func(tx *reform.TX) error { params := &models.CreateAgentParams{ - PMMAgentID: p.PmmAgentId, - ServiceID: p.ServiceId, - Username: p.Username, - Password: p.Password, - AgentPassword: p.AgentPassword, - CustomLabels: p.CustomLabels, - TLS: p.Tls, - TLSSkipVerify: p.TlsSkipVerify, - PushMetrics: p.PushMetrics, - DisableCollectors: p.DisableCollectors, - LogLevel: services.SpecifyLogLevel(p.LogLevel, inventoryv1.LogLevel_LOG_LEVEL_FATAL), + PMMAgentID: p.PmmAgentId, + ServiceID: p.ServiceId, + Username: p.Username, + Password: p.Password, + AgentPassword: p.AgentPassword, + CustomLabels: p.CustomLabels, + TLS: p.Tls, + TLSSkipVerify: p.TlsSkipVerify, + ExporterOptions: &models.ExporterOptions{ + PushMetrics: p.PushMetrics, + DisabledCollectors: p.DisableCollectors, + }, + LogLevel: services.SpecifyLogLevel(p.LogLevel, inventoryv1.LogLevel_LOG_LEVEL_FATAL), } row, err := models.CreateAgent(tx.Querier, models.ProxySQLExporterType, params) if err != nil { @@ -842,17 +858,19 @@ func (as *AgentsService) AddQANPostgreSQLPgStatementsAgent(ctx context.Context, var agent *inventoryv1.QANPostgreSQLPgStatementsAgent e := as.db.InTransactionContext(ctx, nil, func(tx *reform.TX) error { params := &models.CreateAgentParams{ - PMMAgentID: p.PmmAgentId, - ServiceID: p.ServiceId, - Username: p.Username, - Password: p.Password, - CustomLabels: p.CustomLabels, - MaxQueryLength: p.MaxQueryLength, - CommentsParsingDisabled: p.DisableCommentsParsing, - TLS: p.Tls, - TLSSkipVerify: p.TlsSkipVerify, - PostgreSQLOptions: models.PostgreSQLOptionsFromRequest(p), - LogLevel: services.SpecifyLogLevel(p.LogLevel, inventoryv1.LogLevel_LOG_LEVEL_FATAL), + PMMAgentID: p.PmmAgentId, + ServiceID: p.ServiceId, + Username: p.Username, + Password: p.Password, + CustomLabels: p.CustomLabels, + TLS: p.Tls, + TLSSkipVerify: p.TlsSkipVerify, + QANOptions: &models.QANOptions{ + MaxQueryLength: p.MaxQueryLength, + CommentsParsingDisabled: p.DisableCommentsParsing, + }, + PostgreSQLOptions: models.PostgreSQLOptionsFromRequest(p), + LogLevel: services.SpecifyLogLevel(p.LogLevel, inventoryv1.LogLevel_LOG_LEVEL_FATAL), } row, err := models.CreateAgent(tx.Querier, models.QANPostgreSQLPgStatementsAgentType, params) if err != nil { @@ -919,18 +937,20 @@ func (as *AgentsService) AddQANPostgreSQLPgStatMonitorAgent(ctx context.Context, var agent *inventoryv1.QANPostgreSQLPgStatMonitorAgent e := as.db.InTransactionContext(ctx, nil, func(tx *reform.TX) error { params := &models.CreateAgentParams{ - PMMAgentID: p.PmmAgentId, - ServiceID: p.ServiceId, - Username: p.Username, - Password: p.Password, - MaxQueryLength: p.MaxQueryLength, - QueryExamplesDisabled: p.DisableQueryExamples, - CommentsParsingDisabled: p.DisableCommentsParsing, - CustomLabels: p.CustomLabels, - TLS: p.Tls, - TLSSkipVerify: p.TlsSkipVerify, - PostgreSQLOptions: models.PostgreSQLOptionsFromRequest(p), - LogLevel: services.SpecifyLogLevel(p.LogLevel, inventoryv1.LogLevel_LOG_LEVEL_FATAL), + PMMAgentID: p.PmmAgentId, + ServiceID: p.ServiceId, + Username: p.Username, + Password: p.Password, + CustomLabels: p.CustomLabels, + TLS: p.Tls, + TLSSkipVerify: p.TlsSkipVerify, + QANOptions: &models.QANOptions{ + MaxQueryLength: p.MaxQueryLength, + QueryExamplesDisabled: p.DisableQueryExamples, + CommentsParsingDisabled: p.DisableCommentsParsing, + }, + PostgreSQLOptions: models.PostgreSQLOptionsFromRequest(p), + LogLevel: services.SpecifyLogLevel(p.LogLevel, inventoryv1.LogLevel_LOG_LEVEL_FATAL), } row, err := models.CreateAgent(tx.Querier, models.QANPostgreSQLPgStatMonitorAgentType, params) if err != nil { @@ -997,15 +1017,19 @@ func (as *AgentsService) AddRDSExporter(ctx context.Context, p *inventoryv1.AddR var agent *inventoryv1.RDSExporter e := as.db.InTransactionContext(ctx, nil, func(tx *reform.TX) error { params := &models.CreateAgentParams{ - PMMAgentID: p.PmmAgentId, - NodeID: p.NodeId, - AWSAccessKey: p.AwsAccessKey, - AWSSecretKey: p.AwsSecretKey, - CustomLabels: p.CustomLabels, - RDSBasicMetricsDisabled: p.DisableBasicMetrics, - RDSEnhancedMetricsDisabled: p.DisableEnhancedMetrics, - PushMetrics: p.PushMetrics, - LogLevel: services.SpecifyLogLevel(p.LogLevel, inventoryv1.LogLevel_LOG_LEVEL_FATAL), + PMMAgentID: p.PmmAgentId, + NodeID: p.NodeId, + CustomLabels: p.CustomLabels, + ExporterOptions: &models.ExporterOptions{ + PushMetrics: p.PushMetrics, + }, + AWSOptions: &models.AWSOptions{ + AWSAccessKey: p.AwsAccessKey, + AWSSecretKey: p.AwsSecretKey, + RDSBasicMetricsDisabled: p.DisableBasicMetrics, + RDSEnhancedMetricsDisabled: p.DisableEnhancedMetrics, + }, + LogLevel: services.SpecifyLogLevel(p.LogLevel, inventoryv1.LogLevel_LOG_LEVEL_FATAL), } row, err := models.CreateAgent(tx.Querier, models.RDSExporterType, params) if err != nil { @@ -1148,9 +1172,11 @@ func (as *AgentsService) AddAzureDatabaseExporter(ctx context.Context, p *invent params := &models.CreateAgentParams{ PMMAgentID: p.PmmAgentId, NodeID: p.NodeId, - AzureOptions: models.AzureOptionsFromRequest(p), CustomLabels: p.CustomLabels, - PushMetrics: p.PushMetrics, + ExporterOptions: &models.ExporterOptions{ + PushMetrics: p.PushMetrics, + }, + AzureOptions: models.AzureOptionsFromRequest(p), LogLevel: services.SpecifyLogLevel(p.LogLevel, inventoryv1.LogLevel_LOG_LEVEL_FATAL), } row, err := models.CreateAgent(tx.Querier, models.AzureDatabaseExporterType, params) diff --git a/managed/services/management/agent.go b/managed/services/management/agent.go index 6d4bf61963..565bc3f385 100644 --- a/managed/services/management/agent.go +++ b/managed/services/management/agent.go @@ -125,35 +125,35 @@ func (s *ManagementService) agentToAPI(agent *models.Agent) (*managementv1.Unive ua := &managementv1.UniversalAgent{ AgentId: agent.AgentID, AgentType: string(agent.AgentType), - AwsAccessKey: pointer.GetString(agent.AWSAccessKey), + AwsAccessKey: agent.AWSOptions.AWSAccessKey, CreatedAt: timestamppb.New(agent.CreatedAt), CustomLabels: labels, Disabled: agent.Disabled, - DisabledCollectors: agent.DisabledCollectors, + DisabledCollectors: agent.ExporterOptions.DisabledCollectors, IsConnected: s.r.IsConnected(agent.AgentID), IsAgentPasswordSet: pointer.GetString(agent.AgentPassword) != "", - IsAwsSecretKeySet: pointer.GetString(agent.AWSSecretKey) != "", + IsAwsSecretKeySet: agent.AWSOptions.AWSAccessKey != "", IsPasswordSet: pointer.GetString(agent.Password) != "", ListenPort: uint32(pointer.GetUint16(agent.ListenPort)), LogLevel: inventoryv1.LogLevelAPIValue(agent.LogLevel), - MaxQueryLength: agent.MaxQueryLength, - MaxQueryLogSize: agent.MaxQueryLogSize, - MetricsPath: pointer.GetString(agent.MetricsPath), - MetricsScheme: pointer.GetString(agent.MetricsScheme), + MaxQueryLength: agent.QANOptions.MaxQueryLength, + MaxQueryLogSize: agent.QANOptions.MaxQueryLogSize, + MetricsPath: agent.ExporterOptions.MetricsPath, + MetricsScheme: agent.ExporterOptions.MetricsScheme, NodeId: pointer.GetString(agent.NodeID), PmmAgentId: pointer.GetString(agent.PMMAgentID), ProcessExecPath: pointer.GetString(agent.ProcessExecPath), - PushMetrics: agent.PushMetrics, - ExposeExporter: agent.ExposeExporter, - QueryExamplesDisabled: agent.QueryExamplesDisabled, - CommentsParsingDisabled: agent.CommentsParsingDisabled, - RdsBasicMetricsDisabled: agent.RDSBasicMetricsDisabled, - RdsEnhancedMetricsDisabled: agent.RDSEnhancedMetricsDisabled, + PushMetrics: agent.ExporterOptions.PushMetrics, + ExposeExporter: agent.ExporterOptions.ExposeExporter, + QueryExamplesDisabled: agent.QANOptions.QueryExamplesDisabled, + CommentsParsingDisabled: agent.QANOptions.CommentsParsingDisabled, + RdsBasicMetricsDisabled: agent.AWSOptions.RDSBasicMetricsDisabled, + RdsEnhancedMetricsDisabled: agent.AWSOptions.RDSEnhancedMetricsDisabled, RunsOnNodeId: pointer.GetString(agent.RunsOnNodeID), ServiceId: pointer.GetString(agent.ServiceID), Status: agent.Status, - TableCount: pointer.GetInt32(agent.TableCount), - TableCountTablestatsGroupLimit: agent.TableCountTablestatsGroupLimit, + TableCount: pointer.GetInt32(agent.MySQLOptions.TableCount), + TableCountTablestatsGroupLimit: agent.MySQLOptions.TableCountTablestatsGroupLimit, Tls: agent.TLS, TlsSkipVerify: agent.TLSSkipVerify, Username: pointer.GetString(agent.Username), diff --git a/managed/services/management/azure_database.go b/managed/services/management/azure_database.go index 0fea413810..d98af5d1aa 100644 --- a/managed/services/management/azure_database.go +++ b/managed/services/management/azure_database.go @@ -271,13 +271,15 @@ func (s *ManagementService) AddAzureDatabase(ctx context.Context, req *managemen } metricsExporter, err := models.CreateAgent(tx.Querier, exporterType, &models.CreateAgentParams{ - PMMAgentID: models.PMMServerAgentID, - ServiceID: service.ServiceID, - Username: req.Username, - Password: req.Password, - TLS: req.Tls, - TLSSkipVerify: req.TlsSkipVerify, - TableCountTablestatsGroupLimit: tablestatsGroupTableLimit, + PMMAgentID: models.PMMServerAgentID, + ServiceID: service.ServiceID, + Username: req.Username, + Password: req.Password, + TLS: req.Tls, + TLSSkipVerify: req.TlsSkipVerify, + MySQLOptions: &models.MySQLOptions{ + TableCountTablestatsGroupLimit: tablestatsGroupTableLimit, + }, }) if err != nil { return err @@ -295,13 +297,15 @@ func (s *ManagementService) AddAzureDatabase(ctx context.Context, req *managemen if req.Qan { qanAgent, err := models.CreateAgent(tx.Querier, qanAgentType, &models.CreateAgentParams{ - PMMAgentID: models.PMMServerAgentID, - ServiceID: service.ServiceID, - Username: req.Username, - Password: req.Password, - TLS: req.Tls, - TLSSkipVerify: req.TlsSkipVerify, - QueryExamplesDisabled: req.DisableQueryExamples, + PMMAgentID: models.PMMServerAgentID, + ServiceID: service.ServiceID, + Username: req.Username, + Password: req.Password, + TLS: req.Tls, + TLSSkipVerify: req.TlsSkipVerify, + QANOptions: &models.QANOptions{ + QueryExamplesDisabled: req.DisableQueryExamples, + }, }) if err != nil { return err diff --git a/managed/services/management/rds.go b/managed/services/management/rds.go index de5f774c08..27f2c96a55 100644 --- a/managed/services/management/rds.go +++ b/managed/services/management/rds.go @@ -410,7 +410,8 @@ func (s *ManagementService) addRDS(ctx context.Context, req *managementv1.AddRDS ExporterOptions: &models.ExporterOptions{ PushMetrics: isPushMode(metricsMode), }, - AWSOptions: &models.AWSOptions{ + // TODO? PostgresExporter but MySQL Option? + MySQLOptions: &models.MySQLOptions{ TableCountTablestatsGroupLimit: tablestatsGroupTableLimit, }, @@ -440,14 +441,16 @@ func (s *ManagementService) addRDS(ctx context.Context, req *managementv1.AddRDS // add PostgreSQL Pgstatements QAN Agent if req.QanPostgresqlPgstatements { qanAgent, err := models.CreateAgent(tx.Querier, models.QANPostgreSQLPgStatementsAgentType, &models.CreateAgentParams{ - PMMAgentID: pmmAgentID, - ServiceID: service.ServiceID, - Username: req.Username, - Password: req.Password, - TLS: req.Tls, - TLSSkipVerify: req.TlsSkipVerify, - QueryExamplesDisabled: req.DisableQueryExamples, - CommentsParsingDisabled: req.DisableCommentsParsing, + PMMAgentID: pmmAgentID, + ServiceID: service.ServiceID, + Username: req.Username, + Password: req.Password, + TLS: req.Tls, + TLSSkipVerify: req.TlsSkipVerify, + QANOptions: &models.QANOptions{ + QueryExamplesDisabled: req.DisableQueryExamples, + CommentsParsingDisabled: req.DisableCommentsParsing, + }, }) if err != nil { return err diff --git a/managed/services/victoriametrics/scrape_configs_test.go b/managed/services/victoriametrics/scrape_configs_test.go index 73b98560e3..63d77db19c 100644 --- a/managed/services/victoriametrics/scrape_configs_test.go +++ b/managed/services/victoriametrics/scrape_configs_test.go @@ -555,11 +555,13 @@ func TestScrapeConfig(t *testing.T) { Address: pointer.ToString("5.6.7.8"), } agent := &models.Agent{ - AgentID: "75bb30d3-ef4a-4147-97a8-621a996611dd", - AgentType: models.MySQLdExporterType, - ListenPort: pointer.ToUint16(12345), - TableCount: pointer.ToInt32(100500), - TableCountTablestatsGroupLimit: 1000, + AgentID: "75bb30d3-ef4a-4147-97a8-621a996611dd", + AgentType: models.MySQLdExporterType, + ListenPort: pointer.ToUint16(12345), + MySQLOptions: &models.MySQLOptions{ + TableCount: pointer.ToInt32(100500), + TableCountTablestatsGroupLimit: 1000, + }, } expected := []*config.ScrapeConfig{{ From 8e733b479542d873d0ec720fd7ca3b507cb78d82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Mon, 25 Nov 2024 11:44:54 +0100 Subject: [PATCH 06/60] PMM-5086-12634 Revert changed makefile. --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index b57f52798f..6dbf9abe63 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ endif env-up: ## Start devcontainer COMPOSE_PROFILES=$(PROFILES) \ - docker compose up -d --wait + docker compose up -d --wait --wait-timeout 100 env-up-rebuild: env-update-image ## Rebuild and start devcontainer. Useful for custom $PMM_SERVER_IMAGE COMPOSE_PROFILES=$(PROFILES) \ @@ -20,7 +20,7 @@ env-update-image: ## Pull latest dev image env-compose-up: env-update-image COMPOSE_PROFILES=$(PROFILES) \ - docker compose up --detach --renew-anon-volumes --remove-orphans --wait + docker compose up --detach --renew-anon-volumes --remove-orphans --wait --wait-timeout 100 env-devcontainer: docker exec -it --workdir=/root/go/src/github.com/percona/pmm pmm-server .devcontainer/setup.py From 18fac541e66a13acd8ba4bdc0ed102b0e87ec30d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Mon, 25 Nov 2024 11:58:02 +0100 Subject: [PATCH 07/60] PMM-5086-12634 Changes. --- managed/models/agent_helpers.go | 11 ++-- managed/models/database.go | 3 +- managed/services/agents/rds.go | 8 +-- managed/services/agents/registry.go | 12 ++-- managed/services/agents/roster_test.go | 17 +++-- .../services/agents/service_info_broker.go | 2 +- managed/services/agents/state.go | 2 +- .../services/victoriametrics/prometheus.go | 14 ++-- .../victoriametrics/scrape_configs.go | 30 ++++----- .../victoriametrics/scrape_configs_test.go | 66 +++++++++++-------- .../victoriametrics/victoriametrics_test.go | 8 ++- 11 files changed, 95 insertions(+), 78 deletions(-) diff --git a/managed/models/agent_helpers.go b/managed/models/agent_helpers.go index 080cb6b6da..1c96e164a3 100644 --- a/managed/models/agent_helpers.go +++ b/managed/models/agent_helpers.go @@ -28,7 +28,6 @@ import ( "google.golang.org/grpc/status" "gopkg.in/reform.v1" - "github.com/percona/pmm/managed/models" "github.com/percona/pmm/version" ) @@ -646,7 +645,7 @@ func CreateNodeExporter(q *reform.Querier, PMMAgentID: &pmmAgentID, NodeID: pmmAgent.RunsOnNodeID, AgentPassword: agentPassword, - ExporterOptions: &models.ExporterOptions{ + ExporterOptions: &ExporterOptions{ ExposeExporter: exposeExporter, PushMetrics: pushMetrics, DisabledCollectors: disableCollectors, @@ -736,10 +735,10 @@ func CreateExternalExporter(q *reform.Querier, params *CreateExternalExporterPar Username: pointer.ToStringOrNil(params.Username), Password: pointer.ToStringOrNil(params.Password), ListenPort: pointer.ToUint16(uint16(params.ListenPort)), - ExporterOptions: &models.ExporterOptions{ + ExporterOptions: &ExporterOptions{ PushMetrics: params.PushMetrics, - MetricsPath: &metricsPath, - MetricsScheme: &scheme, + MetricsPath: metricsPath, + MetricsScheme: scheme, }, } if err := row.SetCustomLabels(params.CustomLabels); err != nil { @@ -956,7 +955,7 @@ func ChangeAgent(q *reform.Querier, agentID string, params *ChangeCommonAgentPar } if params.EnablePushMetrics != nil { - row.ExporterOptions = &models.ExporterOptions{ + row.ExporterOptions = &ExporterOptions{ PushMetrics: *params.EnablePushMetrics, } if row.AgentType == ExternalExporterType { diff --git a/managed/models/database.go b/managed/models/database.go index d3608da193..47f7757c57 100644 --- a/managed/models/database.go +++ b/managed/models/database.go @@ -37,7 +37,6 @@ import ( "gopkg.in/reform.v1" "gopkg.in/reform.v1/dialects/postgresql" - "github.com/percona/pmm/managed/models" "github.com/percona/pmm/managed/utils/encryption" ) @@ -1420,7 +1419,7 @@ func setupPMMServerAgents(q *reform.Querier, params SetupDBParams) error { TLSSkipVerify: params.SSLMode == DisableSSLMode || params.SSLMode == VerifyCaSSLMode, Username: params.Username, Password: params.Password, - QANOptions: &models.QANOptions{ + QANOptions: &QANOptions{ CommentsParsingDisabled: true, }, } diff --git a/managed/services/agents/rds.go b/managed/services/agents/rds.go index 158925f07c..1be5fe70ee 100644 --- a/managed/services/agents/rds.go +++ b/managed/services/agents/rds.go @@ -80,11 +80,11 @@ func rdsExporterConfig(pairs map[*models.Node]*models.Agent, redactMode redactMo config.Instances = append(config.Instances, rdsInstance{ Region: pointer.GetString(node.Region), Instance: node.Address, - AWSAccessKey: pointer.GetString(exporter.AWSAccessKey), - AWSSecretKey: pointer.GetString(exporter.AWSSecretKey), + AWSAccessKey: exporter.AWSOptions.AWSAccessKey, + AWSSecretKey: exporter.AWSOptions.AWSSecretKey, Labels: labels, - DisableBasicMetrics: exporter.RDSBasicMetricsDisabled, - DisableEnhancedMetrics: exporter.RDSEnhancedMetricsDisabled, + DisableBasicMetrics: exporter.AWSOptions.RDSBasicMetricsDisabled, + DisableEnhancedMetrics: exporter.AWSOptions.RDSEnhancedMetricsDisabled, }) if redactMode != exposeSecrets { diff --git a/managed/services/agents/registry.go b/managed/services/agents/registry.go index de86982afc..3b63fc19dc 100644 --- a/managed/services/agents/registry.go +++ b/managed/services/agents/registry.go @@ -321,9 +321,11 @@ func (r *Registry) addVMAgentToPMMAgent(q *reform.Querier, pmmAgentID, runsOnNod } if len(vmAgent) == 0 { if _, err := models.CreateAgent(q, models.VMAgentType, &models.CreateAgentParams{ - PMMAgentID: pmmAgentID, - PushMetrics: true, - NodeID: runsOnNodeID, + PMMAgentID: pmmAgentID, + NodeID: runsOnNodeID, + ExporterOptions: &models.ExporterOptions{ + PushMetrics: true, + }, }); err != nil { return errors.Wrapf(err, "Can't create 'vmAgent' for pmm-agent with ID %q", pmmAgentID) } @@ -349,9 +351,9 @@ func removeVMAgentFromPMMAgent(q *reform.Querier, pmmAgentID string) error { return errors.Wrapf(err, "Can't find agents for pmm-agent with ID %q", pmmAgentID) } for _, agent := range agents { - if agent.PushMetrics { + if agent.ExporterOptions.PushMetrics { logrus.Warnf("disabling push_metrics for agent with unsupported version ID %q with pmm-agent ID %q", agent.AgentID, pmmAgentID) - agent.PushMetrics = false + agent.ExporterOptions.PushMetrics = false if err := q.Update(agent); err != nil { return errors.Wrapf(err, "Can't set push_metrics=false for agent %q at pmm-agent with ID %q", agent.AgentID, pmmAgentID) } diff --git a/managed/services/agents/roster_test.go b/managed/services/agents/roster_test.go index 1b67f0f458..f7b8b5fcf4 100644 --- a/managed/services/agents/roster_test.go +++ b/managed/services/agents/roster_test.go @@ -18,7 +18,6 @@ package agents import ( "testing" - "github.com/AlekSi/pointer" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "gopkg.in/reform.v1" @@ -56,9 +55,11 @@ func TestRoster(t *testing.T) { } awsAccessKey := "aws" exporters[node] = &models.Agent{ - AgentID: "agent1", - AgentType: models.RDSExporterType, - AWSAccessKey: pointer.ToString(awsAccessKey), + AgentID: "agent1", + AgentType: models.RDSExporterType, + AWSOptions: &models.AWSOptions{ + AWSAccessKey: awsAccessKey, + }, } const expected = "pmm-server:rds/aws" @@ -94,9 +95,11 @@ func TestRoster(t *testing.T) { } awsAccessKey := "aws" exporters[node] = &models.Agent{ - AgentID: "agent1", - AgentType: models.RDSExporterType, - AWSAccessKey: pointer.ToString(awsAccessKey), + AgentID: "agent1", + AgentType: models.RDSExporterType, + AWSOptions: &models.AWSOptions{ + AWSAccessKey: awsAccessKey, + }, } const expectedGroupID = "pmm-server:rds/aws" diff --git a/managed/services/agents/service_info_broker.go b/managed/services/agents/service_info_broker.go index 6802de66f8..420615401a 100644 --- a/managed/services/agents/service_info_broker.go +++ b/managed/services/agents/service_info_broker.go @@ -180,7 +180,7 @@ func (c *ServiceInfoBroker) GetInfoFromService(ctx context.Context, q *reform.Qu stype := service.ServiceType switch stype { case models.MySQLServiceType: - agent.TableCount = &sInfo.TableCount + agent.MySQLOptions.TableCount = &sInfo.TableCount l.Debugf("Updating table count: %d.", sInfo.TableCount) encryptedAgent := models.EncryptAgent(*agent) if err = q.Update(&encryptedAgent); err != nil { diff --git a/managed/services/agents/state.go b/managed/services/agents/state.go index afc92816a0..c10c71b567 100644 --- a/managed/services/agents/state.go +++ b/managed/services/agents/state.go @@ -268,7 +268,7 @@ func (u *StateUpdater) sendSetStateRequest(ctx context.Context, agent *pmmAgentI // Iterate over the rdsExporters map for node, exporter := range rdsExporters { - awsAccessKey := pointer.GetString(exporter.AWSAccessKey) + awsAccessKey := exporter.AWSOptions.AWSAccessKey if _, ok := groupedRdsExporters[awsAccessKey]; !ok { groupedRdsExporters[awsAccessKey] = make(map[*models.Node]*models.Agent) diff --git a/managed/services/victoriametrics/prometheus.go b/managed/services/victoriametrics/prometheus.go index 35e1f6b366..a4f957cc6c 100644 --- a/managed/services/victoriametrics/prometheus.go +++ b/managed/services/victoriametrics/prometheus.go @@ -108,15 +108,15 @@ func AddScrapeConfigs(l *logrus.Entry, cfg *config.Config, q *reform.Querier, // } mr := *globalResolutions // copy global resolutions - if agent.MetricsResolutions != nil { - if agent.MetricsResolutions.MR != 0 { - mr.MR = agent.MetricsResolutions.MR + if agent.ExporterOptions.MetricsResolutions != nil { + if agent.ExporterOptions.MetricsResolutions.MR != 0 { + mr.MR = agent.ExporterOptions.MetricsResolutions.MR } - if agent.MetricsResolutions.HR != 0 { - mr.HR = agent.MetricsResolutions.HR + if agent.ExporterOptions.MetricsResolutions.HR != 0 { + mr.HR = agent.ExporterOptions.MetricsResolutions.HR } - if agent.MetricsResolutions.LR != 0 { - mr.LR = agent.MetricsResolutions.LR + if agent.ExporterOptions.MetricsResolutions.LR != 0 { + mr.LR = agent.ExporterOptions.MetricsResolutions.LR } } diff --git a/managed/services/victoriametrics/scrape_configs.go b/managed/services/victoriametrics/scrape_configs.go index fcc000e22b..4d9ce14014 100644 --- a/managed/services/victoriametrics/scrape_configs.go +++ b/managed/services/victoriametrics/scrape_configs.go @@ -214,7 +214,7 @@ func scrapeConfigsForNodeExporter(params *scrapeConfigParams) ([]*config.ScrapeC "hwmon", "textfile.mr", } - mrCollect = collectors.FilterOutCollectors("", mrCollect, params.agent.DisabledCollectors) + mrCollect = collectors.FilterOutCollectors("", mrCollect, params.agent.ExporterOptions.DisabledCollectors) mr, err = scrapeConfigForStandardExporter("mr", params.metricsResolution.MR, params, mrCollect) if err != nil { return nil, err @@ -227,7 +227,7 @@ func scrapeConfigsForNodeExporter(params *scrapeConfigParams) ([]*config.ScrapeC "uname", "os", } - lrCollect = collectors.FilterOutCollectors("", lrCollect, params.agent.DisabledCollectors) + lrCollect = collectors.FilterOutCollectors("", lrCollect, params.agent.ExporterOptions.DisabledCollectors) lr, err = scrapeConfigForStandardExporter("lr", params.metricsResolution.LR, params, lrCollect) if err != nil { return nil, err @@ -255,7 +255,7 @@ func scrapeConfigsForNodeExporter(params *scrapeConfigParams) ([]*config.ScrapeC "meminfo", "netdev", "time") - hrCollect = collectors.FilterOutCollectors("", hrCollect, params.agent.DisabledCollectors) + hrCollect = collectors.FilterOutCollectors("", hrCollect, params.agent.ExporterOptions.DisabledCollectors) hr, err = scrapeConfigForStandardExporter("hr", params.metricsResolution.HR, params, hrCollect) if err != nil { @@ -285,7 +285,7 @@ func scrapeConfigsForMySQLdExporter(params *scrapeConfigParams) ([]*config.Scrap "standard.go", "standard.process", } - hrOptions = collectors.FilterOutCollectors("", hrOptions, params.agent.DisabledCollectors) + hrOptions = collectors.FilterOutCollectors("", hrOptions, params.agent.ExporterOptions.DisabledCollectors) hr, err := scrapeConfigForStandardExporter("hr", params.metricsResolution.HR, params, hrOptions) if err != nil { @@ -307,7 +307,7 @@ func scrapeConfigsForMySQLdExporter(params *scrapeConfigParams) ([]*config.Scrap mrOptions = append(mrOptions, "perf_schema.tablelocks") } - mrOptions = collectors.FilterOutCollectors("", mrOptions, params.agent.DisabledCollectors) + mrOptions = collectors.FilterOutCollectors("", mrOptions, params.agent.ExporterOptions.DisabledCollectors) mr, err := scrapeConfigForStandardExporter("mr", params.metricsResolution.MR, params, mrOptions) if err != nil { return nil, err @@ -335,7 +335,7 @@ func scrapeConfigsForMySQLdExporter(params *scrapeConfigParams) ([]*config.Scrap "perf_schema.tableiowaits") } - lrOptions = collectors.FilterOutCollectors("", lrOptions, params.agent.DisabledCollectors) + lrOptions = collectors.FilterOutCollectors("", lrOptions, params.agent.ExporterOptions.DisabledCollectors) lr, err := scrapeConfigForStandardExporter("lr", params.metricsResolution.LR, params, lrOptions) if err != nil { @@ -375,7 +375,7 @@ func scrapeConfigsForMongoDBExporter(params *scrapeConfigParams) ([]*config.Scra "replicasetstatus", "topmetrics", } - hrOptions = collectors.FilterOutCollectors("", hrOptions, params.agent.DisabledCollectors) + hrOptions = collectors.FilterOutCollectors("", hrOptions, params.agent.ExporterOptions.DisabledCollectors) hr, err := scrapeConfigForStandardExporter("hr", params.metricsResolution.HR, params, hrOptions) if err != nil { return nil, err @@ -404,7 +404,7 @@ func scrapeConfigsForMongoDBExporter(params *scrapeConfigParams) ([]*config.Scra defaultCollectors = append(defaultCollectors, "pbm") } - defaultCollectors = collectors.FilterOutCollectors("", defaultCollectors, params.agent.DisabledCollectors) + defaultCollectors = collectors.FilterOutCollectors("", defaultCollectors, params.agent.ExporterOptions.DisabledCollectors) lr, err := scrapeConfigForStandardExporter("lr", params.metricsResolution.LR, params, defaultCollectors) if err != nil { return nil, err @@ -425,7 +425,7 @@ func scrapeConfigsForPostgresExporter(params *scrapeConfigParams) ([]*config.Scr "standard.process", "postgres", } - hrOptions = collectors.FilterOutCollectors("", hrOptions, params.agent.DisabledCollectors) + hrOptions = collectors.FilterOutCollectors("", hrOptions, params.agent.ExporterOptions.DisabledCollectors) hr, err := scrapeConfigForStandardExporter("hr", params.metricsResolution.HR, params, hrOptions) if err != nil { return nil, err @@ -434,7 +434,7 @@ func scrapeConfigsForPostgresExporter(params *scrapeConfigParams) ([]*config.Scr mrOptions := []string{ "custom_query.mr", } - mrOptions = collectors.FilterOutCollectors("", mrOptions, params.agent.DisabledCollectors) + mrOptions = collectors.FilterOutCollectors("", mrOptions, params.agent.ExporterOptions.DisabledCollectors) mr, err := scrapeConfigForStandardExporter("mr", params.metricsResolution.MR, params, mrOptions) if err != nil { return nil, err @@ -443,7 +443,7 @@ func scrapeConfigsForPostgresExporter(params *scrapeConfigParams) ([]*config.Scr lrOptions := []string{ "custom_query.lr", } - lrOptions = collectors.FilterOutCollectors("", lrOptions, params.agent.DisabledCollectors) + lrOptions = collectors.FilterOutCollectors("", lrOptions, params.agent.ExporterOptions.DisabledCollectors) lr, err := scrapeConfigForStandardExporter("lr", params.metricsResolution.LR, params, lrOptions) if err != nil { return nil, err @@ -538,8 +538,8 @@ func scrapeConfigsForExternalExporter(s *models.MetricsResolutions, params *scra JobName: jobName(params.agent, "mr"), ScrapeInterval: config.Duration(interval), ScrapeTimeout: scrapeTimeout(interval), - Scheme: pointer.GetString(params.agent.MetricsScheme), - MetricsPath: pointer.GetString(params.agent.MetricsPath), + Scheme: params.agent.ExporterOptions.MetricsScheme, + MetricsPath: params.agent.ExporterOptions.MetricsPath, } if pointer.GetString(params.agent.Username) != "" { @@ -575,8 +575,8 @@ func scrapeConfigsForVMAgent(s *models.MetricsResolutions, params *scrapeConfigP JobName: jobName(params.agent, "mr"), ScrapeInterval: config.Duration(interval), ScrapeTimeout: scrapeTimeout(interval), - Scheme: pointer.GetString(params.agent.MetricsScheme), - MetricsPath: pointer.GetString(params.agent.MetricsPath), + Scheme: params.agent.ExporterOptions.MetricsScheme, + MetricsPath: params.agent.ExporterOptions.MetricsPath, } if pointer.GetString(params.agent.Username) != "" { diff --git a/managed/services/victoriametrics/scrape_configs_test.go b/managed/services/victoriametrics/scrape_configs_test.go index 63d77db19c..614cd47b52 100644 --- a/managed/services/victoriametrics/scrape_configs_test.go +++ b/managed/services/victoriametrics/scrape_configs_test.go @@ -47,11 +47,13 @@ func TestScrapeConfig(t *testing.T) { CustomLabels: []byte(`{"_some_node_label": "foo"}`), } agent := &models.Agent{ - AgentID: "75bb30d3-ef4a-4147-97a8-621a996611dd", - AgentType: models.NodeExporterType, - CustomLabels: []byte(`{"_some_agent_label": "baz"}`), - ListenPort: pointer.ToUint16(12345), - DisabledCollectors: []string{"cpu", "entropy"}, + AgentID: "75bb30d3-ef4a-4147-97a8-621a996611dd", + AgentType: models.NodeExporterType, + CustomLabels: []byte(`{"_some_agent_label": "baz"}`), + ListenPort: pointer.ToUint16(12345), + ExporterOptions: &models.ExporterOptions{ + DisabledCollectors: []string{"cpu", "entropy"}, + }, } expected := []*config.ScrapeConfig{{ @@ -182,11 +184,13 @@ func TestScrapeConfig(t *testing.T) { CustomLabels: []byte(`{"_some_node_label": "foo"}`), } agent := &models.Agent{ - AgentID: "75bb30d3-ef4a-4147-97a8-621a996611dd", - AgentType: models.NodeExporterType, - CustomLabels: []byte(`{"_some_agent_label": "baz"}`), - ListenPort: pointer.ToUint16(12345), - DisabledCollectors: []string{"cpu", "time"}, + AgentID: "75bb30d3-ef4a-4147-97a8-621a996611dd", + AgentType: models.NodeExporterType, + CustomLabels: []byte(`{"_some_agent_label": "baz"}`), + ListenPort: pointer.ToUint16(12345), + ExporterOptions: &models.ExporterOptions{ + DisabledCollectors: []string{"cpu", "time"}, + }, } expected := []*config.ScrapeConfig{{ @@ -407,11 +411,13 @@ func TestScrapeConfig(t *testing.T) { CustomLabels: []byte(`{"_some_service_label": "bar"}`), } agent := &models.Agent{ - AgentID: "75bb30d3-ef4a-4147-97a8-621a996611dd", - AgentType: models.MySQLdExporterType, - CustomLabels: []byte(`{"_some_agent_label": "baz"}`), - ListenPort: pointer.ToUint16(12345), - DisabledCollectors: []string{"global_status", "info_schema.innodb_cmp", "info_schema.query_response_time", "perf_schema.eventsstatements", "heartbeat"}, + AgentID: "75bb30d3-ef4a-4147-97a8-621a996611dd", + AgentType: models.MySQLdExporterType, + CustomLabels: []byte(`{"_some_agent_label": "baz"}`), + ListenPort: pointer.ToUint16(12345), + ExporterOptions: &models.ExporterOptions{ + DisabledCollectors: []string{"global_status", "info_schema.innodb_cmp", "info_schema.query_response_time", "perf_schema.eventsstatements", "heartbeat"}, + }, } expected := []*config.ScrapeConfig{{ @@ -837,11 +843,13 @@ func TestScrapeConfig(t *testing.T) { CustomLabels: []byte(`{"_some_service_label": "bar"}`), } agent := &models.Agent{ - AgentID: "75bb30d3-ef4a-4147-97a8-621a996611dd", - AgentType: models.PostgresExporterType, - CustomLabels: []byte(`{"_some_agent_label": "baz"}`), - ListenPort: pointer.ToUint16(12345), - DisabledCollectors: []string{"standard.process", "custom_query.lr"}, + AgentID: "75bb30d3-ef4a-4147-97a8-621a996611dd", + AgentType: models.PostgresExporterType, + CustomLabels: []byte(`{"_some_agent_label": "baz"}`), + ListenPort: pointer.ToUint16(12345), + ExporterOptions: &models.ExporterOptions{ + DisabledCollectors: []string{"standard.process", "custom_query.lr"}, + }, } expected := []*config.ScrapeConfig{{ @@ -1217,14 +1225,16 @@ func TestScrapeConfig(t *testing.T) { t.Run("WithExtraParams", func(t *testing.T) { agent := &models.Agent{ - AgentID: "75bb30d3-ef4a-4147-97a8-621a996611dd", - AgentType: models.ExternalExporterType, - CustomLabels: []byte(`{"_some_agent_label": "baz"}`), - Username: pointer.ToString("username"), - Password: pointer.ToString("password"), - ListenPort: pointer.ToUint16(12345), - MetricsPath: pointer.ToString("/some-metric-path"), - MetricsScheme: pointer.ToString("https"), + AgentID: "75bb30d3-ef4a-4147-97a8-621a996611dd", + AgentType: models.ExternalExporterType, + CustomLabels: []byte(`{"_some_agent_label": "baz"}`), + Username: pointer.ToString("username"), + Password: pointer.ToString("password"), + ListenPort: pointer.ToUint16(12345), + ExporterOptions: &models.ExporterOptions{ + MetricsPath: "/some-metric-path", + MetricsScheme: "https", + }, } expected := []*config.ScrapeConfig{{ diff --git a/managed/services/victoriametrics/victoriametrics_test.go b/managed/services/victoriametrics/victoriametrics_test.go index d85e082aaf..f2b47c2bd9 100644 --- a/managed/services/victoriametrics/victoriametrics_test.go +++ b/managed/services/victoriametrics/victoriametrics_test.go @@ -230,7 +230,9 @@ func TestVictoriaMetrics(t *testing.T) { CustomLabels: []byte(`{"_agent_label": "mongodb-baz-push"}`), ListenPort: pointer.ToUint16(12346), MongoDBOptions: &models.MongoDBOptions{EnableAllCollectors: true}, - PushMetrics: true, + ExporterOptions: &models.ExporterOptions{ + PushMetrics: true, + }, }, // Agent with pull model @@ -242,7 +244,9 @@ func TestVictoriaMetrics(t *testing.T) { CustomLabels: []byte(`{"_agent_label": "mongodb-baz-pull"}`), ListenPort: pointer.ToUint16(12346), MongoDBOptions: &models.MongoDBOptions{EnableAllCollectors: true}, - PushMetrics: false, + ExporterOptions: &models.ExporterOptions{ + PushMetrics: false, + }, }, } { check.NoError(db.Insert(str), "%+v", str) From 8c84682d6de173738e69c3de914704fd7232ed75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Mon, 25 Nov 2024 13:15:09 +0100 Subject: [PATCH 08/60] PMM-5086-12634 Fix for encryption. --- managed/models/encryption_helpers.go | 7 +++++-- managed/services/encryption/encryption_rotation_test.go | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/managed/models/encryption_helpers.go b/managed/models/encryption_helpers.go index ccda8b75a0..e7929dc13d 100644 --- a/managed/models/encryption_helpers.go +++ b/managed/models/encryption_helpers.go @@ -129,12 +129,12 @@ func agentEncryption(agent Agent, handler func(string) (string, error)) Agent { // EncryptAWSOptionsHandler returns encrypted AWS Options. func EncryptAWSOptionsHandler(e *encryption.Encryption, val any) (any, error) { - return azureOptionsHandler(val, e.Encrypt) + return awsOptionsHandler(val, e.Encrypt) } // DecryptAWSOptionsHandler returns decrypted AWS Options. func DecryptAWSOptionsHandler(e *encryption.Encryption, val any) (any, error) { - return azureOptionsHandler(val, e.Decrypt) + return awsOptionsHandler(val, e.Decrypt) } func awsOptionsHandler(val any, handler func(string) (string, error)) (any, error) { @@ -154,6 +154,9 @@ func awsOptionsHandler(val any, handler func(string) (string, error)) (any, erro return nil, err } o.AWSSecretKey, err = handler(o.AWSSecretKey) + if err != nil { + return nil, err + } res, err := json.Marshal(o) if err != nil { diff --git a/managed/services/encryption/encryption_rotation_test.go b/managed/services/encryption/encryption_rotation_test.go index 99a1579916..0573576422 100644 --- a/managed/services/encryption/encryption_rotation_test.go +++ b/managed/services/encryption/encryption_rotation_test.go @@ -76,7 +76,7 @@ func createOriginEncryptionKey() error { //nolint:dupword func insertTestData(db *sql.DB) error { _, err := models.UpdateSettings(db, &models.ChangeSettingsParams{ - EncryptedItems: []string{"pmm-managed-dev.agents.username", "pmm-managed-dev.agents.password", "pmm-managed-dev.agents.aws_access_key", "pmm-managed-dev.agents.aws_secret_key", "pmm-managed-dev.agents.mongo_db_tls_options", "pmm-managed-dev.agents.azure_options", "pmm-managed-dev.agents.mysql_options", "pmm-managed-dev.agents.postgresql_options", "pmm-managed-dev.agents.agent_password"}, + EncryptedItems: []string{"pmm-managed-dev.agents.username", "pmm-managed-dev.agents.password", "pmm-managed-dev.agents.agent_password", "pmm-managed-dev.agents.aws_options", "pmm-managed-dev.agents.azure_options", "pmm-managed-dev.agents.mongo_options", "pmm-managed-dev.agents.mysql_options", "pmm-managed-dev.agents.postgresql_options"}, }) if err != nil { return err From 9621d71c79ed9baba81c14db767ef3e40bb29017 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Mon, 25 Nov 2024 13:15:24 +0100 Subject: [PATCH 09/60] PMM-5086-12634 Database changes. --- managed/models/agent_model.go | 2 +- managed/models/agent_model_reform.go | 4 ++-- managed/models/database.go | 12 +++++++++++- managed/models/database_test.go | 8 ++++---- 4 files changed, 18 insertions(+), 8 deletions(-) diff --git a/managed/models/agent_model.go b/managed/models/agent_model.go index 829b360e12..90f6542d63 100644 --- a/managed/models/agent_model.go +++ b/managed/models/agent_model.go @@ -232,7 +232,7 @@ type Agent struct { AWSOptions *AWSOptions `reform:"aws_options"` AzureOptions *AzureOptions `reform:"azure_options"` - MongoDBOptions *MongoDBOptions `reform:"mongo_db_tls_options"` + MongoDBOptions *MongoDBOptions `reform:"mongo_options"` MySQLOptions *MySQLOptions `reform:"mysql_options"` PostgreSQLOptions *PostgreSQLOptions `reform:"postgresql_options"` } diff --git a/managed/models/agent_model_reform.go b/managed/models/agent_model_reform.go index f916bb2d3b..2bce8110e1 100644 --- a/managed/models/agent_model_reform.go +++ b/managed/models/agent_model_reform.go @@ -52,7 +52,7 @@ func (v *agentTableType) Columns() []string { "qan_options", "aws_options", "azure_options", - "mongo_db_tls_options", + "mongo_options", "mysql_options", "postgresql_options", } @@ -103,7 +103,7 @@ var AgentTable = &agentTableType{ {Name: "QANOptions", Type: "*QANOptions", Column: "qan_options"}, {Name: "AWSOptions", Type: "*AWSOptions", Column: "aws_options"}, {Name: "AzureOptions", Type: "*AzureOptions", Column: "azure_options"}, - {Name: "MongoDBOptions", Type: "*MongoDBOptions", Column: "mongo_db_tls_options"}, + {Name: "MongoDBOptions", Type: "*MongoDBOptions", Column: "mongo_options"}, {Name: "MySQLOptions", Type: "*MySQLOptions", Column: "mysql_options"}, {Name: "PostgreSQLOptions", Type: "*PostgreSQLOptions", Column: "postgresql_options"}, }, diff --git a/managed/models/database.go b/managed/models/database.go index 47f7757c57..3900c3b79e 100644 --- a/managed/models/database.go +++ b/managed/models/database.go @@ -71,7 +71,7 @@ var DefaultAgentEncryptionColumns = []encryption.Table{ {Name: "agent_password"}, {Name: "aws_options", CustomHandler: EncryptAWSOptionsHandler}, {Name: "azure_options", CustomHandler: EncryptAzureOptionsHandler}, - {Name: "mongo_db_tls_options", CustomHandler: EncryptMongoDBOptionsHandler}, + {Name: "mongo_options", CustomHandler: EncryptMongoDBOptionsHandler}, {Name: "mysql_options", CustomHandler: EncryptMySQLOptionsHandler}, {Name: "postgresql_options", CustomHandler: EncryptPostgreSQLOptionsHandler}, }, @@ -1076,6 +1076,16 @@ var databaseSchema = [][]string{ `ALTER TABLE user_flags ADD COLUMN snoozed_pmm_version VARCHAR NOT NULL DEFAULT ''`, }, + 107: { + `ALTER TABLE agents ADD COLUMN exporter_options JSONB`, + `ALTER TABLE agents ADD COLUMN qan_options JSONB`, + `ALTER TABLE agents ADD COLUMN aws_options JSONB`, + + `ALTER TABLE agents ALTER COLUMN azure_options JSONB`, + `ALTER TABLE agents ALTER COLUMN mysql_options JSONB`, + + `ALTER TABLE agent RENAME COLUMN mongo_db_tls_options TO mongo_options`, + }, } // ^^^ Avoid default values in schema definition. ^^^ diff --git a/managed/models/database_test.go b/managed/models/database_test.go index efafd5e2f7..363942283e 100644 --- a/managed/models/database_test.go +++ b/managed/models/database_test.go @@ -342,7 +342,7 @@ func TestDatabaseMigrations(t *testing.T) { // Insert dummy agent in DB _, err = sqlDB.ExecContext(context.Background(), `INSERT INTO - agents(mongo_db_tls_options, agent_id, agent_type, runs_on_node_id, created_at, updated_at, disabled, status, tls, tls_skip_verify, query_examples_disabled, max_query_log_size, table_count_tablestats_group_limit, rds_basic_metrics_disabled, rds_enhanced_metrics_disabled, push_metrics) + agents(mongo_options, agent_id, agent_type, runs_on_node_id, created_at, updated_at, disabled, status, tls, tls_skip_verify, query_examples_disabled, max_query_log_size, table_count_tablestats_group_limit, rds_basic_metrics_disabled, rds_enhanced_metrics_disabled, push_metrics) VALUES ('{"stats_collections": "db.col1,db.col2,db.col3"}', 'id', 'pmm-agent', 'node_id' , '03/03/2014 02:03:04', '03/03/2014 02:03:04', false, 'alive', false, false, false, 0, 0, false, false, false)`, ) @@ -353,7 +353,7 @@ func TestDatabaseMigrations(t *testing.T) { var agentID string var mongoDBOptions *models.MongoDBOptions - err = sqlDB.QueryRow(`SELECT agent_id, mongo_db_tls_options FROM agents WHERE agent_id = $1`, "id").Scan(&agentID, &mongoDBOptions) + err = sqlDB.QueryRow(`SELECT agent_id, mongo_options FROM agents WHERE agent_id = $1`, "id").Scan(&agentID, &mongoDBOptions) require.NoError(t, err) require.Equal(t, "id", agentID) @@ -376,7 +376,7 @@ func TestDatabaseMigrations(t *testing.T) { // Insert dummy agent in DB _, err = sqlDB.ExecContext(context.Background(), `INSERT INTO - agents(mongo_db_tls_options, agent_id, agent_type, runs_on_node_id, created_at, updated_at, disabled, status, tls, tls_skip_verify, query_examples_disabled, max_query_log_size, table_count_tablestats_group_limit, rds_basic_metrics_disabled, rds_enhanced_metrics_disabled, push_metrics) + agents(mongo_options, agent_id, agent_type, runs_on_node_id, created_at, updated_at, disabled, status, tls, tls_skip_verify, query_examples_disabled, max_query_log_size, table_count_tablestats_group_limit, rds_basic_metrics_disabled, rds_enhanced_metrics_disabled, push_metrics) VALUES ('{"stats_collections": ["db.col1", "db.col2", "db.col3"]}', 'id', 'pmm-agent', 'node_id' , '03/03/2014 02:03:04', '03/03/2014 02:03:04', false, 'alive', false, false, false, 0, 0, false, false, false)`, ) @@ -387,7 +387,7 @@ func TestDatabaseMigrations(t *testing.T) { var agentID string var mongoDBOptions *models.MongoDBOptions - err = sqlDB.QueryRow(`SELECT agent_id, mongo_db_tls_options FROM agents WHERE agent_id = $1`, "id").Scan(&agentID, &mongoDBOptions) + err = sqlDB.QueryRow(`SELECT agent_id, mongo_options FROM agents WHERE agent_id = $1`, "id").Scan(&agentID, &mongoDBOptions) require.NoError(t, err) require.Equal(t, "id", agentID) From 88a2affb4f577292262c1720dcb590e1e93a537e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Tue, 26 Nov 2024 10:57:45 +0100 Subject: [PATCH 10/60] PMM-5086-12634 Fix DB JSONB conversion. --- managed/models/database.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/managed/models/database.go b/managed/models/database.go index 3900c3b79e..216b596b5d 100644 --- a/managed/models/database.go +++ b/managed/models/database.go @@ -1081,8 +1081,8 @@ var databaseSchema = [][]string{ `ALTER TABLE agents ADD COLUMN qan_options JSONB`, `ALTER TABLE agents ADD COLUMN aws_options JSONB`, - `ALTER TABLE agents ALTER COLUMN azure_options JSONB`, - `ALTER TABLE agents ALTER COLUMN mysql_options JSONB`, + `ALTER TABLE agents ALTER COLUMN azure_options JSONB USING to_jsonb(azure_options)`, + `ALTER TABLE agents ALTER COLUMN mysql_options JSONB USING to_jsonb(mysql_options)`, `ALTER TABLE agent RENAME COLUMN mongo_db_tls_options TO mongo_options`, }, From a8c144b59d9f2703e8a2f4fb42b51f0a0fe2825e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Tue, 26 Nov 2024 11:06:26 +0100 Subject: [PATCH 11/60] PMM-5086-12634 Missed type. --- managed/models/database.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/managed/models/database.go b/managed/models/database.go index 216b596b5d..c29c5a37c7 100644 --- a/managed/models/database.go +++ b/managed/models/database.go @@ -1081,8 +1081,8 @@ var databaseSchema = [][]string{ `ALTER TABLE agents ADD COLUMN qan_options JSONB`, `ALTER TABLE agents ADD COLUMN aws_options JSONB`, - `ALTER TABLE agents ALTER COLUMN azure_options JSONB USING to_jsonb(azure_options)`, - `ALTER TABLE agents ALTER COLUMN mysql_options JSONB USING to_jsonb(mysql_options)`, + `ALTER TABLE agents ALTER COLUMN azure_options TYPE JSONB USING to_jsonb(azure_options)`, + `ALTER TABLE agents ALTER COLUMN mysql_options TYPE JSONB USING to_jsonb(mysql_options)`, `ALTER TABLE agent RENAME COLUMN mongo_db_tls_options TO mongo_options`, }, From 0d9a0b6b00b4d5694d2f6ac50756491932b3069b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Tue, 26 Nov 2024 11:13:08 +0100 Subject: [PATCH 12/60] PMM-5086-12634 Typo in mongo options renaming. --- managed/models/database.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/managed/models/database.go b/managed/models/database.go index c29c5a37c7..082ccd4005 100644 --- a/managed/models/database.go +++ b/managed/models/database.go @@ -1084,7 +1084,7 @@ var databaseSchema = [][]string{ `ALTER TABLE agents ALTER COLUMN azure_options TYPE JSONB USING to_jsonb(azure_options)`, `ALTER TABLE agents ALTER COLUMN mysql_options TYPE JSONB USING to_jsonb(mysql_options)`, - `ALTER TABLE agent RENAME COLUMN mongo_db_tls_options TO mongo_options`, + `ALTER TABLE agents RENAME COLUMN mongo_db_tls_options TO mongo_options`, }, } From b2ab11ba9babf3861b74d5f732dcb197950c9a89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Tue, 26 Nov 2024 12:25:50 +0100 Subject: [PATCH 13/60] PMM-5086-12634 Changes. --- managed/models/agent_helpers.go | 4 ++-- managed/models/agent_helpers_test.go | 4 ++-- managed/models/agent_model.go | 10 +++++----- managed/models/agent_model_test.go | 20 +++++++++---------- managed/services/agents/mongodb.go | 4 ++-- managed/services/agents/mysql.go | 4 ++-- managed/services/agents/node.go | 11 +++++++--- managed/services/agents/postgresql.go | 8 ++++++-- managed/services/agents/proxysql.go | 4 ++-- managed/services/converters.go | 4 ++-- managed/services/management/agent.go | 4 ++-- .../services/victoriametrics/prometheus.go | 3 +++ .../victoriametrics/scrape_configs.go | 8 ++++---- .../victoriametrics/scrape_configs_test.go | 4 ++-- 14 files changed, 52 insertions(+), 40 deletions(-) diff --git a/managed/models/agent_helpers.go b/managed/models/agent_helpers.go index 1c96e164a3..790187794a 100644 --- a/managed/models/agent_helpers.go +++ b/managed/models/agent_helpers.go @@ -737,8 +737,8 @@ func CreateExternalExporter(q *reform.Querier, params *CreateExternalExporterPar ListenPort: pointer.ToUint16(uint16(params.ListenPort)), ExporterOptions: &ExporterOptions{ PushMetrics: params.PushMetrics, - MetricsPath: metricsPath, - MetricsScheme: scheme, + MetricsPath: pointer.ToStringOrNil(metricsPath), + MetricsScheme: pointer.ToStringOrNil(scheme), }, } if err := row.SetCustomLabels(params.CustomLabels); err != nil { diff --git a/managed/models/agent_helpers_test.go b/managed/models/agent_helpers_test.go index 32516f1c14..48e363ff80 100644 --- a/managed/models/agent_helpers_test.go +++ b/managed/models/agent_helpers_test.go @@ -490,8 +490,8 @@ func TestAgentHelpers(t *testing.T) { ServiceID: pointer.ToString("S1"), ListenPort: pointer.ToUint16(9104), ExporterOptions: &models.ExporterOptions{ - MetricsPath: "/metrics", - MetricsScheme: "http", + MetricsPath: pointer.ToString("/metrics"), + MetricsScheme: pointer.ToString("http"), }, CreatedAt: now, UpdatedAt: now, diff --git a/managed/models/agent_model.go b/managed/models/agent_model.go index 90f6542d63..8b177d6797 100644 --- a/managed/models/agent_model.go +++ b/managed/models/agent_model.go @@ -84,8 +84,8 @@ type ExporterOptions struct { PushMetrics bool `reform:"push_metrics"` DisabledCollectors pq.StringArray `reform:"disabled_collectors"` MetricsResolutions *MetricsResolutions `reform:"metrics_resolutions"` - MetricsPath string `reform:"metrics_path"` - MetricsScheme string `reform:"metrics_scheme"` + MetricsPath *string `reform:"metrics_path"` + MetricsScheme *string `reform:"metrics_scheme"` } // Value implements database/sql/driver.Valuer interface. Should be defined on the value. @@ -597,8 +597,8 @@ func (s *Agent) ExporterURL(q *reform.Querier) (string, error) { s.ExporterOptions = &ExporterOptions{} } - scheme := s.ExporterOptions.MetricsScheme - path := s.ExporterOptions.MetricsPath + scheme := pointer.GetString(s.ExporterOptions.MetricsScheme) + path := pointer.GetString(s.ExporterOptions.MetricsPath) listenPort := int(pointer.GetUint16(s.ListenPort)) username := pointer.GetString(s.Username) password := pointer.GetString(s.Password) @@ -721,7 +721,7 @@ func (s Agent) TemplateDelimiters(svc *Service) *DelimiterPair { pointer.GetString(svc.Address), pointer.GetString(s.Username), pointer.GetString(s.Password), - s.ExporterOptions.MetricsPath, + pointer.GetString(s.ExporterOptions.MetricsPath), } switch svc.ServiceType { diff --git a/managed/models/agent_model_test.go b/managed/models/agent_model_test.go index c06c5e13a8..d00bbdd37d 100644 --- a/managed/models/agent_model_test.go +++ b/managed/models/agent_model_test.go @@ -448,8 +448,8 @@ func TestExporterURL(t *testing.T) { ListenPort: pointer.ToUint16(9121), ExporterOptions: &models.ExporterOptions{ PushMetrics: true, - MetricsPath: "/metrics", - MetricsScheme: "http", + MetricsPath: pointer.ToString("/metrics"), + MetricsScheme: pointer.ToString("http"), }, }, @@ -463,8 +463,8 @@ func TestExporterURL(t *testing.T) { Password: pointer.ToString("secret"), ExporterOptions: &models.ExporterOptions{ PushMetrics: false, - MetricsPath: "/metrics", - MetricsScheme: "http", + MetricsPath: pointer.ToString("/metrics"), + MetricsScheme: pointer.ToString("http"), }, }, @@ -478,8 +478,8 @@ func TestExporterURL(t *testing.T) { Password: pointer.ToString("secret"), ExporterOptions: &models.ExporterOptions{ PushMetrics: false, - MetricsPath: "/metrics", - MetricsScheme: "http", + MetricsPath: pointer.ToString("/metrics"), + MetricsScheme: pointer.ToString("http"), }, }, @@ -493,8 +493,8 @@ func TestExporterURL(t *testing.T) { Password: pointer.ToString("secret"), ExporterOptions: &models.ExporterOptions{ PushMetrics: false, - MetricsPath: "/metrics?format=prometheus&output=json", - MetricsScheme: "http", + MetricsPath: pointer.ToString("/metrics?format=prometheus&output=json"), + MetricsScheme: pointer.ToString("http"), }, }, @@ -508,8 +508,8 @@ func TestExporterURL(t *testing.T) { Password: pointer.ToString("secret"), ExporterOptions: &models.ExporterOptions{ PushMetrics: false, - MetricsPath: "/", - MetricsScheme: "http", + MetricsPath: pointer.ToString("/"), + MetricsScheme: pointer.ToString("http"), }, }, } { diff --git a/managed/services/agents/mongodb.go b/managed/services/agents/mongodb.go index 647559df30..8f076905cb 100644 --- a/managed/services/agents/mongodb.go +++ b/managed/services/agents/mongodb.go @@ -47,8 +47,8 @@ func mongodbExporterConfig(node *models.Node, service *models.Service, exporter args := getArgs(exporter, tdp, listenAddress, pmmAgentVersion) - if exporter.ExporterOptions != nil && exporter.ExporterOptions.MetricsPath != "" { - args = append(args, "--web.telemetry-path="+exporter.ExporterOptions.MetricsPath) //nolint:goconst + if exporter.ExporterOptions != nil && exporter.ExporterOptions.MetricsPath != nil { + args = append(args, "--web.telemetry-path="+*exporter.ExporterOptions.MetricsPath) //nolint:goconst } args = withLogLevel(args, exporter.LogLevel, pmmAgentVersion, true) diff --git a/managed/services/agents/mysql.go b/managed/services/agents/mysql.go index 34140b0f89..45e5016df4 100644 --- a/managed/services/agents/mysql.go +++ b/managed/services/agents/mysql.go @@ -106,8 +106,8 @@ func mysqldExporterConfig( args = collectors.FilterOutCollectors("--collect.", args, exporter.ExporterOptions.DisabledCollectors) - if exporter.ExporterOptions.MetricsPath != "" { - args = append(args, "--web.telemetry-path="+exporter.ExporterOptions.MetricsPath) + if exporter.ExporterOptions.MetricsPath != nil { + args = append(args, "--web.telemetry-path="+*exporter.ExporterOptions.MetricsPath) } files := exporter.Files() diff --git a/managed/services/agents/node.go b/managed/services/agents/node.go index b81c763ad2..faf5c5b1e7 100644 --- a/managed/services/agents/node.go +++ b/managed/services/agents/node.go @@ -18,6 +18,7 @@ package agents import ( "sort" + "github.com/AlekSi/pointer" agentv1 "github.com/percona/pmm/api/agent/v1" inventoryv1 "github.com/percona/pmm/api/inventory/v1" "github.com/percona/pmm/managed/models" @@ -34,7 +35,11 @@ var ( func nodeExporterConfig(node *models.Node, exporter *models.Agent, agentVersion *version.Parsed) (*agentv1.SetStateRequest_AgentProcess, error) { listenAddress := getExporterListenAddress(node, exporter) - tdp := models.TemplateDelimsPair(exporter.ExporterOptions.MetricsPath) + + if exporter.ExporterOptions == nil { + exporter.ExporterOptions = &models.ExporterOptions{} + } + tdp := models.TemplateDelimsPair(pointer.GetString(exporter.ExporterOptions.MetricsPath)) args := []string{ "--collector.textfile.directory.lr=" + pathsBase(agentVersion, tdp.Left, tdp.Right) + "/collectors/textfile-collector/low-resolution", "--collector.textfile.directory.mr=" + pathsBase(agentVersion, tdp.Left, tdp.Right) + "/collectors/textfile-collector/medium-resolution", @@ -123,8 +128,8 @@ func nodeExporterConfig(node *models.Node, exporter *models.Agent, agentVersion args = collectors.FilterOutCollectors("--collector.", args, exporter.ExporterOptions.DisabledCollectors) - if exporter.ExporterOptions.MetricsPath != "" { - args = append(args, "--web.telemetry-path="+exporter.ExporterOptions.MetricsPath) + if exporter.ExporterOptions.MetricsPath != nil { + args = append(args, "--web.telemetry-path="+*exporter.ExporterOptions.MetricsPath) } args = withLogLevel(args, exporter.LogLevel, agentVersion, false) diff --git a/managed/services/agents/postgresql.go b/managed/services/agents/postgresql.go index cca54ba767..411f059dee 100644 --- a/managed/services/agents/postgresql.go +++ b/managed/services/agents/postgresql.go @@ -108,8 +108,12 @@ func postgresExporterConfig(node *models.Node, service *models.Service, exporter args = append(args, "--max-connections="+strconv.Itoa(int(exporter.PostgreSQLOptions.MaxExporterConnections))) } - if exporter.ExporterOptions.MetricsPath != "" { - args = append(args, "--web.telemetry-path="+exporter.ExporterOptions.MetricsPath) + if exporter.ExporterOptions == nil { + exporter.ExporterOptions = &models.ExporterOptions{} + } + + if exporter.ExporterOptions.MetricsPath != nil { + args = append(args, "--web.telemetry-path="+*exporter.ExporterOptions.MetricsPath) } args = collectors.FilterOutCollectors("--collect.", args, exporter.ExporterOptions.DisabledCollectors) diff --git a/managed/services/agents/proxysql.go b/managed/services/agents/proxysql.go index 8f786c25e1..9700af1944 100644 --- a/managed/services/agents/proxysql.go +++ b/managed/services/agents/proxysql.go @@ -55,8 +55,8 @@ func proxysqlExporterConfig(node *models.Node, service *models.Service, exporter args = append(args, "-collect.runtime_mysql_servers") } - if exporter.ExporterOptions.MetricsPath != "" { - args = append(args, "-web.telemetry-path="+exporter.ExporterOptions.MetricsPath) + if exporter.ExporterOptions.MetricsPath != nil { + args = append(args, "-web.telemetry-path="+*exporter.ExporterOptions.MetricsPath) } args = collectors.FilterOutCollectors("-collect.", args, exporter.ExporterOptions.DisabledCollectors) diff --git a/managed/services/converters.go b/managed/services/converters.go index 280e274746..38b9d7ed72 100644 --- a/managed/services/converters.go +++ b/managed/services/converters.go @@ -457,8 +457,8 @@ func ToAPIAgent(q *reform.Querier, agent *models.Agent) (inventoryv1.Agent, erro ServiceId: pointer.GetString(agent.ServiceID), Username: pointer.GetString(agent.Username), Disabled: agent.Disabled, - Scheme: agent.ExporterOptions.MetricsScheme, - MetricsPath: agent.ExporterOptions.MetricsPath, + Scheme: pointer.GetString(agent.ExporterOptions.MetricsScheme), + MetricsPath: pointer.GetString(agent.ExporterOptions.MetricsPath), ListenPort: uint32(pointer.GetUint16(agent.ListenPort)), CustomLabels: labels, PushMetricsEnabled: agent.ExporterOptions.PushMetrics, diff --git a/managed/services/management/agent.go b/managed/services/management/agent.go index 565bc3f385..aaeba05d26 100644 --- a/managed/services/management/agent.go +++ b/managed/services/management/agent.go @@ -138,8 +138,8 @@ func (s *ManagementService) agentToAPI(agent *models.Agent) (*managementv1.Unive LogLevel: inventoryv1.LogLevelAPIValue(agent.LogLevel), MaxQueryLength: agent.QANOptions.MaxQueryLength, MaxQueryLogSize: agent.QANOptions.MaxQueryLogSize, - MetricsPath: agent.ExporterOptions.MetricsPath, - MetricsScheme: agent.ExporterOptions.MetricsScheme, + MetricsPath: pointer.GetString(agent.ExporterOptions.MetricsPath), + MetricsScheme: pointer.GetString(agent.ExporterOptions.MetricsScheme), NodeId: pointer.GetString(agent.NodeID), PmmAgentId: pointer.GetString(agent.PMMAgentID), ProcessExecPath: pointer.GetString(agent.ProcessExecPath), diff --git a/managed/services/victoriametrics/prometheus.go b/managed/services/victoriametrics/prometheus.go index a4f957cc6c..7bde451e07 100644 --- a/managed/services/victoriametrics/prometheus.go +++ b/managed/services/victoriametrics/prometheus.go @@ -108,6 +108,9 @@ func AddScrapeConfigs(l *logrus.Entry, cfg *config.Config, q *reform.Querier, // } mr := *globalResolutions // copy global resolutions + if agent.ExporterOptions == nil { + agent.ExporterOptions = &models.ExporterOptions{} + } if agent.ExporterOptions.MetricsResolutions != nil { if agent.ExporterOptions.MetricsResolutions.MR != 0 { mr.MR = agent.ExporterOptions.MetricsResolutions.MR diff --git a/managed/services/victoriametrics/scrape_configs.go b/managed/services/victoriametrics/scrape_configs.go index 4d9ce14014..4324afc33f 100644 --- a/managed/services/victoriametrics/scrape_configs.go +++ b/managed/services/victoriametrics/scrape_configs.go @@ -538,8 +538,8 @@ func scrapeConfigsForExternalExporter(s *models.MetricsResolutions, params *scra JobName: jobName(params.agent, "mr"), ScrapeInterval: config.Duration(interval), ScrapeTimeout: scrapeTimeout(interval), - Scheme: params.agent.ExporterOptions.MetricsScheme, - MetricsPath: params.agent.ExporterOptions.MetricsPath, + Scheme: pointer.GetString(params.agent.ExporterOptions.MetricsScheme), + MetricsPath: pointer.GetString(params.agent.ExporterOptions.MetricsPath), } if pointer.GetString(params.agent.Username) != "" { @@ -575,8 +575,8 @@ func scrapeConfigsForVMAgent(s *models.MetricsResolutions, params *scrapeConfigP JobName: jobName(params.agent, "mr"), ScrapeInterval: config.Duration(interval), ScrapeTimeout: scrapeTimeout(interval), - Scheme: params.agent.ExporterOptions.MetricsScheme, - MetricsPath: params.agent.ExporterOptions.MetricsPath, + Scheme: pointer.GetString(params.agent.ExporterOptions.MetricsScheme), + MetricsPath: pointer.GetString(params.agent.ExporterOptions.MetricsPath), } if pointer.GetString(params.agent.Username) != "" { diff --git a/managed/services/victoriametrics/scrape_configs_test.go b/managed/services/victoriametrics/scrape_configs_test.go index 614cd47b52..2322427521 100644 --- a/managed/services/victoriametrics/scrape_configs_test.go +++ b/managed/services/victoriametrics/scrape_configs_test.go @@ -1232,8 +1232,8 @@ func TestScrapeConfig(t *testing.T) { Password: pointer.ToString("password"), ListenPort: pointer.ToUint16(12345), ExporterOptions: &models.ExporterOptions{ - MetricsPath: "/some-metric-path", - MetricsScheme: "https", + MetricsPath: pointer.ToString("/some-metric-path"), + MetricsScheme: pointer.ToString("https"), }, } From 1355f5effdb52491641e80f9d23e6ae85073d7cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Wed, 27 Nov 2024 10:56:29 +0100 Subject: [PATCH 14/60] PMM-5086-12634 Nil check. --- managed/services/agents/postgresql.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/managed/services/agents/postgresql.go b/managed/services/agents/postgresql.go index 411f059dee..b2de7d427f 100644 --- a/managed/services/agents/postgresql.go +++ b/managed/services/agents/postgresql.go @@ -167,6 +167,9 @@ func qanPostgreSQLPgStatementsAgentConfig(service *models.Service, agent *models Database: service.DatabaseName, PostgreSQLSupportsSSLSNI: !pmmAgentVersion.Less(postgresSSLSniVersion), } + if agent.QANOptions == nil { + agent.QANOptions = &models.QANOptions{} + } return &agentv1.SetStateRequest_BuiltinAgent{ Type: inventoryv1.AgentType_AGENT_TYPE_QAN_POSTGRESQL_PGSTATEMENTS_AGENT, Dsn: agent.DSN(service, dnsParams, nil, pmmAgentVersion), From 86bdee96e6875f5afe4d568365fc12c5ab0296c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Thu, 28 Nov 2024 11:35:54 +0100 Subject: [PATCH 15/60] PMM-5086-12634 Migrations, nil checks. --- managed/models/database.go | 37 +++++++++ managed/services/agents/mongodb.go | 11 ++- managed/services/agents/mysql.go | 12 +++ managed/services/agents/node.go | 1 - managed/services/agents/postgresql.go | 12 +++ managed/services/agents/rds.go | 4 + managed/services/management/agent.go | 108 ++++++++++++++------------ 7 files changed, 135 insertions(+), 50 deletions(-) diff --git a/managed/models/database.go b/managed/models/database.go index 082ccd4005..8ee8b13696 100644 --- a/managed/models/database.go +++ b/managed/models/database.go @@ -1085,6 +1085,43 @@ var databaseSchema = [][]string{ `ALTER TABLE agents ALTER COLUMN mysql_options TYPE JSONB USING to_jsonb(mysql_options)`, `ALTER TABLE agents RENAME COLUMN mongo_db_tls_options TO mongo_options`, + + // migrate values into new fields in options and drop original columns + `UPDATE agents SET exporter_options = jsonb_set(COALESCE(exporter_options, '{}'::jsonb), '{expose_exporter}', to_jsonb(expose_exporter));`, + `UPDATE agents SET exporter_options = jsonb_set(COALESCE(exporter_options, '{}'::jsonb), '{push_metrics}', to_jsonb(push_metrics));`, + `UPDATE agents SET exporter_options = jsonb_set(COALESCE(exporter_options, '{}'::jsonb), '{disabled_collectors}', to_jsonb(disabled_collectors));`, + `UPDATE agents SET exporter_options = jsonb_set(COALESCE(exporter_options, '{}'::jsonb), '{metrics_resolutions}', to_jsonb(metrics_resolutions));`, + `UPDATE agents SET exporter_options = jsonb_set(COALESCE(exporter_options, '{}'::jsonb), '{metrics_path}', to_jsonb(metrics_path));`, + `UPDATE agents SET exporter_options = jsonb_set(COALESCE(exporter_options, '{}'::jsonb), '{metrics_scheme}', to_jsonb(metrics_scheme));`, + `ALTER TABLE agents DROP COLUMN expose_exporter`, + `ALTER TABLE agents DROP COLUMN push_metrics`, + `ALTER TABLE agents DROP COLUMN disabled_collectors`, + `ALTER TABLE agents DROP COLUMN metrics_resolutions`, + `ALTER TABLE agents DROP COLUMN metrics_path`, + `ALTER TABLE agents DROP COLUMN metrics_scheme`, + + `UPDATE agents SET qan_options = jsonb_set(COALESCE(qan_options, '{}'::jsonb), '{max_query_length}', to_jsonb(max_query_length));`, + `UPDATE agents SET qan_options = jsonb_set(COALESCE(qan_options, '{}'::jsonb), '{max_query_log_size}', to_jsonb(max_query_log_size));`, + `UPDATE agents SET qan_options = jsonb_set(COALESCE(qan_options, '{}'::jsonb), '{query_examples_disabled}', to_jsonb(query_examples_disabled));`, + `UPDATE agents SET qan_options = jsonb_set(COALESCE(qan_options, '{}'::jsonb), '{comments_parsing_disabled}', to_jsonb(comments_parsing_disabled));`, + `ALTER TABLE agents DROP COLUMN max_query_length`, + `ALTER TABLE agents DROP COLUMN max_query_log_size`, + `ALTER TABLE agents DROP COLUMN query_examples_disabled`, + `ALTER TABLE agents DROP COLUMN comments_parsing_disabled`, + + `UPDATE agents SET aws_options = jsonb_set(COALESCE(aws_options, '{}'::jsonb), '{aws_access_key}', to_jsonb(aws_access_key));`, + `UPDATE agents SET aws_options = jsonb_set(COALESCE(aws_options, '{}'::jsonb), '{aws_secret_key}', to_jsonb(aws_secret_key));`, + `UPDATE agents SET aws_options = jsonb_set(COALESCE(aws_options, '{}'::jsonb), '{rds_basic_metrics_disabled}', to_jsonb(rds_basic_metrics_disabled));`, + `UPDATE agents SET aws_options = jsonb_set(COALESCE(aws_options, '{}'::jsonb), '{rds_enhanced_metrics_disabled}', to_jsonb(rds_enhanced_metrics_disabled));`, + `ALTER TABLE agents DROP COLUMN aws_access_key`, + `ALTER TABLE agents DROP COLUMN aws_secret_key`, + `ALTER TABLE agents DROP COLUMN rds_basic_metrics_disabled`, + `ALTER TABLE agents DROP COLUMN rds_enhanced_metrics_disabled`, + + `UPDATE agents SET mysql_options = jsonb_set(COALESCE(mysql_options, '{}'::jsonb), '{table_count}', to_jsonb(table_count));`, + `UPDATE agents SET mysql_options = jsonb_set(COALESCE(mysql_options, '{}'::jsonb), '{table_count_tablestats_group_limit}', to_jsonb(table_count_tablestats_group_limit));`, + `ALTER TABLE agents DROP COLUMN table_count`, + `ALTER TABLE agents DROP COLUMN table_count_tablestats_group_limit`, }, } diff --git a/managed/services/agents/mongodb.go b/managed/services/agents/mongodb.go index 8f076905cb..5152065eb2 100644 --- a/managed/services/agents/mongodb.go +++ b/managed/services/agents/mongodb.go @@ -21,6 +21,7 @@ import ( "strings" "time" + "github.com/AlekSi/pointer" agentv1 "github.com/percona/pmm/api/agent/v1" inventoryv1 "github.com/percona/pmm/api/inventory/v1" "github.com/percona/pmm/managed/models" @@ -47,7 +48,11 @@ func mongodbExporterConfig(node *models.Node, service *models.Service, exporter args := getArgs(exporter, tdp, listenAddress, pmmAgentVersion) - if exporter.ExporterOptions != nil && exporter.ExporterOptions.MetricsPath != nil { + if exporter.ExporterOptions == nil { + exporter.ExporterOptions = &models.ExporterOptions{} + } + + if pointer.GetString(exporter.ExporterOptions.MetricsPath) != "" { args = append(args, "--web.telemetry-path="+*exporter.ExporterOptions.MetricsPath) //nolint:goconst } @@ -163,6 +168,10 @@ func buildBaseArgs(listenAddress string, tdp *models.DelimiterPair) []string { // qanMongoDBProfilerAgentConfig returns desired configuration of qan-mongodb-profiler-agent built-in agent. func qanMongoDBProfilerAgentConfig(service *models.Service, agent *models.Agent, pmmAgentVersion *version.Parsed) *agentv1.SetStateRequest_BuiltinAgent { tdp := agent.TemplateDelimiters(service) + if agent.QANOptions == nil { + agent.QANOptions = &models.QANOptions{} + } + return &agentv1.SetStateRequest_BuiltinAgent{ Type: inventoryv1.AgentType_AGENT_TYPE_QAN_MONGODB_PROFILER_AGENT, Dsn: agent.DSN(service, models.DSNParams{DialTimeout: time.Second, Database: ""}, nil, pmmAgentVersion), diff --git a/managed/services/agents/mysql.go b/managed/services/agents/mysql.go index 45e5016df4..f60acca943 100644 --- a/managed/services/agents/mysql.go +++ b/managed/services/agents/mysql.go @@ -104,6 +104,10 @@ func mysqldExporterConfig( args = append(args, tablestatsGroup...) } + if exporter.MySQLOptions == nil { + exporter.MySQLOptions = &models.MySQLOptions{} + } + args = collectors.FilterOutCollectors("--collect.", args, exporter.ExporterOptions.DisabledCollectors) if exporter.ExporterOptions.MetricsPath != nil { @@ -154,6 +158,10 @@ func mysqldExporterConfig( // qanMySQLPerfSchemaAgentConfig returns desired configuration of qan-mysql-perfschema built-in agent. func qanMySQLPerfSchemaAgentConfig(service *models.Service, agent *models.Agent, pmmAgentVersion *version.Parsed) *agentv1.SetStateRequest_BuiltinAgent { tdp := agent.TemplateDelimiters(service) + if agent.QANOptions == nil { + agent.QANOptions = &models.QANOptions{} + } + return &agentv1.SetStateRequest_BuiltinAgent{ Type: inventoryv1.AgentType_AGENT_TYPE_QAN_MYSQL_PERFSCHEMA_AGENT, Dsn: agent.DSN(service, models.DSNParams{DialTimeout: time.Second, Database: ""}, nil, pmmAgentVersion), @@ -172,6 +180,10 @@ func qanMySQLPerfSchemaAgentConfig(service *models.Service, agent *models.Agent, // qanMySQLSlowlogAgentConfig returns desired configuration of qan-mysql-slowlog built-in agent. func qanMySQLSlowlogAgentConfig(service *models.Service, agent *models.Agent, pmmAgentVersion *version.Parsed) *agentv1.SetStateRequest_BuiltinAgent { tdp := agent.TemplateDelimiters(service) + if agent.QANOptions == nil { + agent.QANOptions = &models.QANOptions{} + } + return &agentv1.SetStateRequest_BuiltinAgent{ Type: inventoryv1.AgentType_AGENT_TYPE_QAN_MYSQL_SLOWLOG_AGENT, Dsn: agent.DSN(service, models.DSNParams{DialTimeout: time.Second, Database: ""}, nil, pmmAgentVersion), diff --git a/managed/services/agents/node.go b/managed/services/agents/node.go index faf5c5b1e7..cc20300ccf 100644 --- a/managed/services/agents/node.go +++ b/managed/services/agents/node.go @@ -35,7 +35,6 @@ var ( func nodeExporterConfig(node *models.Node, exporter *models.Agent, agentVersion *version.Parsed) (*agentv1.SetStateRequest_AgentProcess, error) { listenAddress := getExporterListenAddress(node, exporter) - if exporter.ExporterOptions == nil { exporter.ExporterOptions = &models.ExporterOptions{} } diff --git a/managed/services/agents/postgresql.go b/managed/services/agents/postgresql.go index b2de7d427f..ad524a33c8 100644 --- a/managed/services/agents/postgresql.go +++ b/managed/services/agents/postgresql.go @@ -84,6 +84,10 @@ func postgresExporterConfig(node *models.Node, service *models.Service, exporter "--web.listen-address=" + listenAddress + ":" + tdp.Left + " .listen_port " + tdp.Right, } + if exporter.PostgreSQLOptions == nil { + exporter.PostgreSQLOptions = &models.PostgreSQLOptions{} + } + autoDiscovery := false if !pmmAgentVersion.Less(postgresExporterAutodiscoveryVersion) { switch { @@ -133,6 +137,9 @@ func postgresExporterConfig(node *models.Node, service *models.Service, exporter PostgreSQLSupportsSSLSNI: !pmmAgentVersion.Less(postgresSSLSniVersion), } + if exporter.AzureOptions == nil { + exporter.AzureOptions = &models.AzureOptions{} + } if exporter.AzureOptions != nil { dnsParams.DialTimeout = 5 * time.Second } @@ -170,6 +177,7 @@ func qanPostgreSQLPgStatementsAgentConfig(service *models.Service, agent *models if agent.QANOptions == nil { agent.QANOptions = &models.QANOptions{} } + return &agentv1.SetStateRequest_BuiltinAgent{ Type: inventoryv1.AgentType_AGENT_TYPE_QAN_POSTGRESQL_PGSTATEMENTS_AGENT, Dsn: agent.DSN(service, dnsParams, nil, pmmAgentVersion), @@ -191,6 +199,10 @@ func qanPostgreSQLPgStatMonitorAgentConfig(service *models.Service, agent *model Database: service.DatabaseName, PostgreSQLSupportsSSLSNI: !pmmAgentVersion.Less(postgresSSLSniVersion), } + if agent.QANOptions == nil { + agent.QANOptions = &models.QANOptions{} + } + return &agentv1.SetStateRequest_BuiltinAgent{ Type: inventoryv1.AgentType_AGENT_TYPE_QAN_POSTGRESQL_PGSTATMONITOR_AGENT, Dsn: agent.DSN(service, dnsParams, nil, pmmAgentVersion), diff --git a/managed/services/agents/rds.go b/managed/services/agents/rds.go index 1be5fe70ee..dcd5d1ba70 100644 --- a/managed/services/agents/rds.go +++ b/managed/services/agents/rds.go @@ -77,6 +77,10 @@ func rdsExporterConfig(pairs map[*models.Node]*models.Agent, redactMode redactMo return nil, err } + if exporter.AWSOptions == nil { + exporter.AWSOptions = &models.AWSOptions{} + } + config.Instances = append(config.Instances, rdsInstance{ Region: pointer.GetString(node.Region), Instance: node.Address, diff --git a/managed/services/management/agent.go b/managed/services/management/agent.go index aaeba05d26..1528922b16 100644 --- a/managed/services/management/agent.go +++ b/managed/services/management/agent.go @@ -123,42 +123,51 @@ func (s *ManagementService) agentToAPI(agent *models.Agent) (*managementv1.Unive } ua := &managementv1.UniversalAgent{ - AgentId: agent.AgentID, - AgentType: string(agent.AgentType), - AwsAccessKey: agent.AWSOptions.AWSAccessKey, - CreatedAt: timestamppb.New(agent.CreatedAt), - CustomLabels: labels, - Disabled: agent.Disabled, - DisabledCollectors: agent.ExporterOptions.DisabledCollectors, - IsConnected: s.r.IsConnected(agent.AgentID), - IsAgentPasswordSet: pointer.GetString(agent.AgentPassword) != "", - IsAwsSecretKeySet: agent.AWSOptions.AWSAccessKey != "", - IsPasswordSet: pointer.GetString(agent.Password) != "", - ListenPort: uint32(pointer.GetUint16(agent.ListenPort)), - LogLevel: inventoryv1.LogLevelAPIValue(agent.LogLevel), - MaxQueryLength: agent.QANOptions.MaxQueryLength, - MaxQueryLogSize: agent.QANOptions.MaxQueryLogSize, - MetricsPath: pointer.GetString(agent.ExporterOptions.MetricsPath), - MetricsScheme: pointer.GetString(agent.ExporterOptions.MetricsScheme), - NodeId: pointer.GetString(agent.NodeID), - PmmAgentId: pointer.GetString(agent.PMMAgentID), - ProcessExecPath: pointer.GetString(agent.ProcessExecPath), - PushMetrics: agent.ExporterOptions.PushMetrics, - ExposeExporter: agent.ExporterOptions.ExposeExporter, - QueryExamplesDisabled: agent.QANOptions.QueryExamplesDisabled, - CommentsParsingDisabled: agent.QANOptions.CommentsParsingDisabled, - RdsBasicMetricsDisabled: agent.AWSOptions.RDSBasicMetricsDisabled, - RdsEnhancedMetricsDisabled: agent.AWSOptions.RDSEnhancedMetricsDisabled, - RunsOnNodeId: pointer.GetString(agent.RunsOnNodeID), - ServiceId: pointer.GetString(agent.ServiceID), - Status: agent.Status, - TableCount: pointer.GetInt32(agent.MySQLOptions.TableCount), - TableCountTablestatsGroupLimit: agent.MySQLOptions.TableCountTablestatsGroupLimit, - Tls: agent.TLS, - TlsSkipVerify: agent.TLSSkipVerify, - Username: pointer.GetString(agent.Username), - UpdatedAt: timestamppb.New(agent.UpdatedAt), - Version: pointer.GetString(agent.Version), + AgentId: agent.AgentID, + AgentType: string(agent.AgentType), + CreatedAt: timestamppb.New(agent.CreatedAt), + CustomLabels: labels, + Disabled: agent.Disabled, + IsConnected: s.r.IsConnected(agent.AgentID), + IsAgentPasswordSet: pointer.GetString(agent.AgentPassword) != "", + IsPasswordSet: pointer.GetString(agent.Password) != "", + ListenPort: uint32(pointer.GetUint16(agent.ListenPort)), + LogLevel: inventoryv1.LogLevelAPIValue(agent.LogLevel), + NodeId: pointer.GetString(agent.NodeID), + PmmAgentId: pointer.GetString(agent.PMMAgentID), + ProcessExecPath: pointer.GetString(agent.ProcessExecPath), + RunsOnNodeId: pointer.GetString(agent.RunsOnNodeID), + ServiceId: pointer.GetString(agent.ServiceID), + Status: agent.Status, + Tls: agent.TLS, + TlsSkipVerify: agent.TLSSkipVerify, + Username: pointer.GetString(agent.Username), + UpdatedAt: timestamppb.New(agent.UpdatedAt), + Version: pointer.GetString(agent.Version), + } + + if agent.ExporterOptions != nil { + ua.DisabledCollectors = agent.ExporterOptions.DisabledCollectors + ua.MetricsPath = pointer.GetString(agent.ExporterOptions.MetricsPath) + ua.MetricsScheme = pointer.GetString(agent.ExporterOptions.MetricsScheme) + ua.PushMetrics = agent.ExporterOptions.PushMetrics + ua.ExposeExporter = agent.ExporterOptions.ExposeExporter + + } + + if agent.QANOptions != nil { + ua.MaxQueryLength = agent.QANOptions.MaxQueryLength + ua.MaxQueryLogSize = agent.QANOptions.MaxQueryLogSize + ua.QueryExamplesDisabled = agent.QANOptions.QueryExamplesDisabled + ua.CommentsParsingDisabled = agent.QANOptions.CommentsParsingDisabled + } + + if agent.AWSOptions != nil { + ua.IsAwsSecretKeySet = agent.AWSOptions.AWSAccessKey != "" + ua.AwsAccessKey = agent.AWSOptions.AWSAccessKey + ua.RdsBasicMetricsDisabled = agent.AWSOptions.RDSBasicMetricsDisabled + ua.RdsEnhancedMetricsDisabled = agent.AWSOptions.RDSEnhancedMetricsDisabled + } if agent.AzureOptions != nil { @@ -171,10 +180,25 @@ func (s *ManagementService) agentToAPI(agent *models.Agent) (*managementv1.Unive } } + if agent.MongoDBOptions != nil { + ua.MongoDbOptions = &managementv1.UniversalAgent_MongoDBOptions{ + AuthenticationMechanism: agent.MongoDBOptions.AuthenticationMechanism, + AuthenticationDatabase: agent.MongoDBOptions.AuthenticationDatabase, + CollectionsLimit: agent.MongoDBOptions.CollectionsLimit, + EnableAllCollectors: agent.MongoDBOptions.EnableAllCollectors, + StatsCollections: agent.MongoDBOptions.StatsCollections, + IsTlsCertificateKeySet: agent.MongoDBOptions.TLSCertificateKey != "", + IsTlsCertificateKeyFilePasswordSet: agent.MongoDBOptions.TLSCertificateKeyFilePassword != "", + } + } + if agent.MySQLOptions != nil { ua.MysqlOptions = &managementv1.UniversalAgent_MySQLOptions{ IsTlsKeySet: agent.MySQLOptions.TLSKey != "", } + ua.TableCount = pointer.GetInt32(agent.MySQLOptions.TableCount) + ua.TableCountTablestatsGroupLimit = agent.MySQLOptions.TableCountTablestatsGroupLimit + } if agent.PostgreSQLOptions != nil { @@ -185,18 +209,6 @@ func (s *ManagementService) agentToAPI(agent *models.Agent) (*managementv1.Unive } } - if agent.MongoDBOptions != nil { - ua.MongoDbOptions = &managementv1.UniversalAgent_MongoDBOptions{ - AuthenticationMechanism: agent.MongoDBOptions.AuthenticationMechanism, - AuthenticationDatabase: agent.MongoDBOptions.AuthenticationDatabase, - CollectionsLimit: agent.MongoDBOptions.CollectionsLimit, - EnableAllCollectors: agent.MongoDBOptions.EnableAllCollectors, - StatsCollections: agent.MongoDBOptions.StatsCollections, - IsTlsCertificateKeySet: agent.MongoDBOptions.TLSCertificateKey != "", - IsTlsCertificateKeyFilePasswordSet: agent.MongoDBOptions.TLSCertificateKeyFilePassword != "", - } - } - return ua, nil } From 325551c08b10e4f580d889c1fd0573c314d3610e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Fri, 29 Nov 2024 13:34:59 +0100 Subject: [PATCH 16/60] PMM-5086-12634 Two models of encryption, db changes. --- managed/models/database.go | 47 ++++++++++++++-- managed/models/database_test.go | 55 +++++++++---------- .../encryption/encryption_rotation.go | 4 +- 3 files changed, 71 insertions(+), 35 deletions(-) diff --git a/managed/models/database.go b/managed/models/database.go index 8ee8b13696..ce2d68abd8 100644 --- a/managed/models/database.go +++ b/managed/models/database.go @@ -60,8 +60,8 @@ const ( VerifyFullSSLMode string = "verify-full" ) -// DefaultAgentEncryptionColumns contains all tables and it's columns to be encrypted in PMM Server DB. -var DefaultAgentEncryptionColumns = []encryption.Table{ +// DefaultAgentEncryptionColumns since 3.0.0 contains all tables and it's columns to be encrypted in PMM Server DB. +var DefaultAgentEncryptionColumnsV3 = []encryption.Table{ { Name: "agents", Identifiers: []string{"agent_id"}, @@ -78,6 +78,25 @@ var DefaultAgentEncryptionColumns = []encryption.Table{ }, } +// DefaultAgentEncryptionColumns contains all tables and it's columns to be encrypted in PMM Server DB. +var DefaultAgentEncryptionColumns = []encryption.Table{ + { + Name: "agents", + Identifiers: []string{"agent_id"}, + Columns: []encryption.Column{ + {Name: "username"}, + {Name: "password"}, + {Name: "agent_password"}, + {Name: "aws_access_key"}, + {Name: "aws_secret_key"}, + {Name: "azure_options", CustomHandler: EncryptAzureOptionsHandler}, + {Name: "mongo_db_tls_options", CustomHandler: EncryptMongoDBOptionsHandler}, + {Name: "mysql_options", CustomHandler: EncryptMySQLOptionsHandler}, + {Name: "postgresql_options", CustomHandler: EncryptPostgreSQLOptionsHandler}, + }, + }, +} + // databaseSchema maps schema version from schema_migrations table (id column) to a slice of DDL queries. var databaseSchema = [][]string{ 1: { @@ -1122,6 +1141,21 @@ var databaseSchema = [][]string{ `UPDATE agents SET mysql_options = jsonb_set(COALESCE(mysql_options, '{}'::jsonb), '{table_count_tablestats_group_limit}', to_jsonb(table_count_tablestats_group_limit));`, `ALTER TABLE agents DROP COLUMN table_count`, `ALTER TABLE agents DROP COLUMN table_count_tablestats_group_limit`, + + // update settings -> encrypted items!! + `UPDATE settings SET settings = jsonb_set(settings, '{encrypted_items}', + '[ + "pmm-managed.agents.username", + "pmm-managed.agents.password", + "pmm-managed.agents.agent_password", + "pmm-managed.agents.aws_options", + "pmm-managed.agents.azure_options", + "pmm-managed.agents.mongo_options", + "pmm-managed.agents.mysql_options", + "pmm-managed.agents.postgresql_options" + ]'::jsonb + ) + WHERE settings ? 'encrypted_items';`, }, } @@ -1213,7 +1247,7 @@ func SetupDB(ctx context.Context, sqlDB *sql.DB, params SetupDBParams) (*reform. return nil, errCV } - if err := migrateDB(db, params, DefaultAgentEncryptionColumns); err != nil { + if err := migrateDB(db, params); err != nil { return nil, err } @@ -1350,7 +1384,7 @@ func initWithRoot(params SetupDBParams) error { } // migrateDB runs PostgreSQL database migrations. -func migrateDB(db *reform.DB, params SetupDBParams, itemsToEncrypt []encryption.Table) error { +func migrateDB(db *reform.DB, params SetupDBParams) error { var currentVersion int errDB := db.QueryRow("SELECT id FROM schema_migrations ORDER BY id DESC LIMIT 1").Scan(¤tVersion) // undefined_table (see https://www.postgresql.org/docs/current/errcodes-appendix.html) @@ -1386,6 +1420,11 @@ func migrateDB(db *reform.DB, params SetupDBParams, itemsToEncrypt []encryption. } } + itemsToEncrypt := DefaultAgentEncryptionColumnsV3 + if params.MigrationVersion != nil && currentVersion < 107 { + itemsToEncrypt = DefaultAgentEncryptionColumns + } + err := EncryptDB(tx, params.Name, itemsToEncrypt) if err != nil { return err diff --git a/managed/models/database_test.go b/managed/models/database_test.go index 363942283e..1a6e790923 100644 --- a/managed/models/database_test.go +++ b/managed/models/database_test.go @@ -205,8 +205,8 @@ func TestDatabaseChecks(t *testing.T) { now, now) require.NoError(t, err) _, err = db.Exec( - "INSERT INTO agents (agent_id, agent_type, runs_on_node_id, pmm_agent_id, disabled, status, created_at, updated_at, tls, tls_skip_verify, max_query_length, query_examples_disabled, comments_parsing_disabled, max_query_log_size, table_count_tablestats_group_limit, rds_basic_metrics_disabled, rds_enhanced_metrics_disabled, push_metrics, expose_exporter) "+ - "VALUES ('1', 'pmm-agent', '1', NULL, false, '', $1, $2, false, false, 0, false, true, 0, 0, true, true, false, false)", + `INSERT INTO agents (agent_id, agent_type, runs_on_node_id, pmm_agent_id, disabled, status, created_at, updated_at, tls, tls_skip_verify, qan_options, mysql_options, aws_options, exporter_options) `+ + `VALUES ('1', 'pmm-agent', '1', NULL, false, '', $1, $2, false, false, '{"max_query_length": 0, "query_examples_disabled": false, "comments_parsing_disabled": true, "max_query_log_size": 0}', '{"table_count_tablestats_group_limit": 0}', '{"rds_basic_metrics_disabled": true, "rds_enhanced_metrics_disabled": true}', '{"push_metrics": false, "expose_exporter": false}')`, now, now) require.NoError(t, err) @@ -216,13 +216,13 @@ func TestDatabaseChecks(t *testing.T) { defer rollback() _, err = tx.Exec( - "INSERT INTO agents (agent_id, agent_type, runs_on_node_id, pmm_agent_id, disabled, status, created_at, updated_at, tls, tls_skip_verify, max_query_length, query_examples_disabled, comments_parsing_disabled, max_query_log_size, table_count_tablestats_group_limit, rds_basic_metrics_disabled, rds_enhanced_metrics_disabled, push_metrics, expose_exporter) "+ - "VALUES ('2', 'pmm-agent', '1', NULL, false, '', $1, $2, false, false, 0, false, true, 0, 0, false, false, false, false)", + `INSERT INTO agents (agent_id, agent_type, runs_on_node_id, pmm_agent_id, disabled, status, created_at, updated_at, tls, tls_skip_verify, qan_options, mysql_options, aws_options, exporter_options) `+ + `VALUES ('2', 'pmm-agent', '1', NULL, false, '', $1, $2, false, false, '{"max_query_length": 0, "query_examples_disabled": false, "comments_parsing_disabled": true, "max_query_log_size": 0}', '{"table_count_tablestats_group_limit": 0}', '{"rds_basic_metrics_disabled": false, "rds_enhanced_metrics_disabled": false}', '{"push_metrics": false, "expose_exporter": false}')`, now, now) require.NoError(t, err) _, err = tx.Exec( - "INSERT INTO agents (agent_id, agent_type, runs_on_node_id, pmm_agent_id, node_id, disabled, status, created_at, updated_at, tls, tls_skip_verify, max_query_length, query_examples_disabled, comments_parsing_disabled, max_query_log_size, table_count_tablestats_group_limit, rds_basic_metrics_disabled, rds_enhanced_metrics_disabled, push_metrics, expose_exporter) "+ - "VALUES ('3', 'mysqld_exporter', NULL, '1', '1', false, '', $1, $2, false, false, 0, false, true, 0, 0, false, false, false, false)", + `INSERT INTO agents (agent_id, agent_type, runs_on_node_id, pmm_agent_id, node_id, disabled, status, created_at, updated_at, tls, tls_skip_verify, qan_options, mysql_options, aws_options, exporter_options) `+ + `VALUES ('3', 'mysqld_exporter', NULL, '1', '1', false, '', $1, $2, false, false, '{"max_query_length": 0, "query_examples_disabled": false, "comments_parsing_disabled": true, "max_query_log_size": 0}', '{"table_count_tablestats_group_limit": 0}', '{"rds_basic_metrics_disabled": false, "rds_enhanced_metrics_disabled": false}', '{"push_metrics": false, "expose_exporter": false}')`, now, now) require.NoError(t, err) }) @@ -232,8 +232,8 @@ func TestDatabaseChecks(t *testing.T) { defer rollback() _, err = tx.Exec( - "INSERT INTO agents (agent_id, agent_type, runs_on_node_id, pmm_agent_id, node_id, disabled, status, created_at, updated_at, tls, tls_skip_verify, max_query_length, query_examples_disabled, comments_parsing_disabled, max_query_log_size, table_count_tablestats_group_limit, rds_basic_metrics_disabled, rds_enhanced_metrics_disabled, push_metrics, expose_exporter) "+ - "VALUES ('4', 'mysqld_exporter', NULL, NULL, '1', false, '', $1, $2, false, false, 0, false, true, 0, 0, false, false, false, false)", + `INSERT INTO agents (agent_id, agent_type, runs_on_node_id, pmm_agent_id, node_id, disabled, status, created_at, updated_at, tls, tls_skip_verify, qan_options, mysql_options, aws_options, exporter_options) `+ + `VALUES ('4', 'mysqld_exporter', NULL, NULL, '1', false, '', $1, $2, false, false, '{"max_query_length": 0, "query_examples_disabled": false, "comments_parsing_disabled": true, "max_query_log_size": 0}', '{"table_count_tablestats_group_limit": 0}', '{"rds_basic_metrics_disabled": false, "rds_enhanced_metrics_disabled": false}', '{"push_metrics": false, "expose_exporter": false}')`, now, now) assertCheckViolation(t, err, "agents", "runs_on_node_id_xor_pmm_agent_id") }) @@ -243,8 +243,8 @@ func TestDatabaseChecks(t *testing.T) { defer rollback() _, err = tx.Exec( - "INSERT INTO agents (agent_id, agent_type, runs_on_node_id, pmm_agent_id, node_id, disabled, status, created_at, updated_at, tls, tls_skip_verify, max_query_length, query_examples_disabled, comments_parsing_disabled, max_query_log_size, table_count_tablestats_group_limit, rds_basic_metrics_disabled, rds_enhanced_metrics_disabled, push_metrics, expose_exporter) "+ - "VALUES ('5', 'pmm-agent', '1', '1', '1', false, '', $1, $2, false, false, 0, false, true, 0, 0, false, false, false, false)", + `INSERT INTO agents (agent_id, agent_type, runs_on_node_id, pmm_agent_id, node_id, disabled, status, created_at, updated_at, tls, tls_skip_verify, qan_options, mysql_options, aws_options, exporter_options) `+ + `VALUES ('5', 'pmm-agent', '1', '1', '1', false, '', $1, $2, false, false, '{"max_query_length": 0, "query_examples_disabled": false, "comments_parsing_disabled": true, "max_query_log_size": 0}', '{"table_count_tablestats_group_limit": 0}', '{"rds_basic_metrics_disabled": false, "rds_enhanced_metrics_disabled": false}', '{"push_metrics": false, "expose_exporter": false}')`, now, now) assertCheckViolation(t, err, "agents", "runs_on_node_id_xor_pmm_agent_id") }) @@ -255,8 +255,8 @@ func TestDatabaseChecks(t *testing.T) { defer rollback() _, err = tx.Exec( - "INSERT INTO agents (agent_id, agent_type, runs_on_node_id, pmm_agent_id, node_id, disabled, status, created_at, updated_at, tls, tls_skip_verify, max_query_length, query_examples_disabled, comments_parsing_disabled, max_query_log_size, table_count_tablestats_group_limit, rds_basic_metrics_disabled, rds_enhanced_metrics_disabled, push_metrics, expose_exporter) "+ - "VALUES ('6', 'mysqld_exporter', '1', NULL, '1', false, '', $1, $2, false, false, 0, false, true, 0, 0, false, false, false, false)", + `INSERT INTO agents (agent_id, agent_type, runs_on_node_id, pmm_agent_id, node_id, disabled, status, created_at, updated_at, tls, tls_skip_verify, qan_options, mysql_options, aws_options, exporter_options) `+ + `VALUES ('6', 'mysqld_exporter', '1', NULL, '1', false, '', $1, $2, false, false, '{"max_query_length": 0, "query_examples_disabled": false, "comments_parsing_disabled": true, "max_query_log_size": 0}', '{"table_count_tablestats_group_limit": 0}', '{"rds_basic_metrics_disabled": false, "rds_enhanced_metrics_disabled": false}', '{"push_metrics": false, "expose_exporter": false}')`, now, now) assertCheckViolation(t, err, "agents", "runs_on_node_id_only_for_pmm_agent") }) @@ -266,8 +266,8 @@ func TestDatabaseChecks(t *testing.T) { defer rollback() _, err = tx.Exec( - "INSERT INTO agents (agent_id, agent_type, runs_on_node_id, pmm_agent_id, node_id, disabled, status, created_at, updated_at, tls, tls_skip_verify, max_query_length, query_examples_disabled, comments_parsing_disabled, max_query_log_size, table_count_tablestats_group_limit, rds_basic_metrics_disabled, rds_enhanced_metrics_disabled, push_metrics, expose_exporter) "+ - "VALUES ('7', 'pmm-agent', NULL, '1', '1', false, '', $1, $2, false, false, 0, false, true, 0, 0, false, false, false, false)", + `INSERT INTO agents (agent_id, agent_type, runs_on_node_id, pmm_agent_id, node_id, disabled, status, created_at, updated_at, tls, tls_skip_verify, qan_options, mysql_options, aws_options, exporter_options) `+ + `VALUES ('7', 'pmm-agent', NULL, '1', '1', false, '', $1, $2, false, false, '{"max_query_length": 0, "query_examples_disabled": false, "comments_parsing_disabled": true, "max_query_log_size": 0}', '{"table_count_tablestats_group_limit": 0}', '{"rds_basic_metrics_disabled": false, "rds_enhanced_metrics_disabled": false}', '{"push_metrics": false, "expose_exporter": false}')`, now, now) assertCheckViolation(t, err, "agents", "runs_on_node_id_only_for_pmm_agent") }) @@ -280,10 +280,9 @@ func TestDatabaseChecks(t *testing.T) { defer rollback() _, err = tx.Exec( - "INSERT INTO agents (agent_id, agent_type, runs_on_node_id, pmm_agent_id, node_id, service_id, disabled, status, created_at, updated_at, tls, tls_skip_verify, max_query_length, query_examples_disabled, comments_parsing_disabled, max_query_log_size, table_count_tablestats_group_limit, rds_basic_metrics_disabled, rds_enhanced_metrics_disabled, push_metrics, expose_exporter) "+ - "VALUES ('8', 'node_exporter', NULL, '1', '1', NULL, false, '', $1, $2, false, false, 0, false, true, 0, 0, false, false, false, false)", + `INSERT INTO agents (agent_id, agent_type, runs_on_node_id, pmm_agent_id, node_id, service_id, disabled, status, created_at, updated_at, tls, tls_skip_verify, qan_options, mysql_options, aws_options, exporter_options) `+ + `VALUES ('8', 'node_exporter', NULL, '1', '1', NULL, false, '', $1, $2, false, false, '{"max_query_length": 0, "query_examples_disabled": false, "comments_parsing_disabled": true, "max_query_log_size": 0}', '{"table_count_tablestats_group_limit": 0}', '{"rds_basic_metrics_disabled": false, "rds_enhanced_metrics_disabled": false}', '{"push_metrics": false, "expose_exporter": false}')`, now, now) - assert.NoError(t, err) }) @@ -292,10 +291,9 @@ func TestDatabaseChecks(t *testing.T) { defer rollback() _, err = tx.Exec( - "INSERT INTO agents (agent_id, agent_type, runs_on_node_id, pmm_agent_id, node_id, service_id, disabled, status, created_at, updated_at, tls, tls_skip_verify, max_query_length, query_examples_disabled, comments_parsing_disabled, max_query_log_size, table_count_tablestats_group_limit, rds_basic_metrics_disabled, rds_enhanced_metrics_disabled, push_metrics, expose_exporter) "+ - "VALUES ('8', 'mysqld_exporter', NULL, '1', NULL, '1', false, '', $1, $2, false, false, 0, false, true, 0, 0, false, false, false, false)", + `INSERT INTO agents (agent_id, agent_type, runs_on_node_id, pmm_agent_id, node_id, service_id, disabled, status, created_at, updated_at, tls, tls_skip_verify, qan_options, mysql_options, aws_options, exporter_options) `+ + `VALUES ('8', 'mysqld_exporter', NULL, '1', NULL, '1', false, '', $1, $2, false, false, '{"max_query_length": 0, "query_examples_disabled": false, "comments_parsing_disabled": true, "max_query_log_size": 0}', '{"table_count_tablestats_group_limit": 0}', '{"rds_basic_metrics_disabled": false, "rds_enhanced_metrics_disabled": false}', '{"push_metrics": false, "expose_exporter": false}')`, now, now) - assert.NoError(t, err) }) @@ -304,10 +302,9 @@ func TestDatabaseChecks(t *testing.T) { defer rollback() _, err = tx.Exec( - "INSERT INTO agents (agent_id, agent_type, runs_on_node_id, pmm_agent_id, node_id, service_id, disabled, status, created_at, updated_at, tls, tls_skip_verify, max_query_length, query_examples_disabled, comments_parsing_disabled, max_query_log_size, table_count_tablestats_group_limit, rds_basic_metrics_disabled, rds_enhanced_metrics_disabled, push_metrics, expose_exporter) "+ - "VALUES ('8', 'mysqld_exporter', NULL, '1', NULL, NULL, false, '', $1, $2, false, false, 0, false, true, 0, 0, false, false, false, false)", + `INSERT INTO agents (agent_id, agent_type, runs_on_node_id, pmm_agent_id, node_id, service_id, disabled, status, created_at, updated_at, tls, tls_skip_verify, qan_options, mysql_options, aws_options, exporter_options) `+ + `VALUES ('8', 'mysqld_exporter', NULL, '1', NULL, NULL, false, '', $1, $2, false, false, '{"max_query_length": 0, "query_examples_disabled": false, "comments_parsing_disabled": true, "max_query_log_size": 0}', '{"table_count_tablestats_group_limit": 0}', '{"rds_basic_metrics_disabled": false, "rds_enhanced_metrics_disabled": false}', '{"push_metrics": false, "expose_exporter": false}')`, now, now) - assertCheckViolation(t, err, "agents", "node_id_or_service_id_for_non_pmm_agent") }) @@ -316,8 +313,8 @@ func TestDatabaseChecks(t *testing.T) { defer rollback() _, err = tx.Exec( - "INSERT INTO agents (agent_id, agent_type, runs_on_node_id, pmm_agent_id, node_id, service_id, disabled, status, created_at, updated_at, tls, tls_skip_verify, max_query_length, query_examples_disabled, comments_parsing_disabled, max_query_log_size, table_count_tablestats_group_limit, rds_basic_metrics_disabled, rds_enhanced_metrics_disabled, push_metrics, expose_exporter) "+ - "VALUES ('8', 'mysqld_exporter', NULL, '1', '1', '1', false, '', $1, $2, false, false, 0, false, true, 0, 0, false, false, false, false)", + `INSERT INTO agents (agent_id, agent_type, runs_on_node_id, pmm_agent_id, node_id, service_id, disabled, status, created_at, updated_at, tls, tls_skip_verify, qan_options, mysql_options, aws_options, exporter_options) `+ + `VALUES ('8', 'mysqld_exporter', NULL, '1', '1', '1', false, '', $1, $2, false, false, '{"max_query_length": 0, "query_examples_disabled": false, "comments_parsing_disabled": true, "max_query_log_size": 0}', '{"table_count_tablestats_group_limit": 0}', '{"rds_basic_metrics_disabled": false, "rds_enhanced_metrics_disabled": false}', '{"push_metrics": false, "expose_exporter": false}')`, now, now) assertCheckViolation(t, err, "agents", "node_id_or_service_id_for_non_pmm_agent") }) @@ -342,7 +339,7 @@ func TestDatabaseMigrations(t *testing.T) { // Insert dummy agent in DB _, err = sqlDB.ExecContext(context.Background(), `INSERT INTO - agents(mongo_options, agent_id, agent_type, runs_on_node_id, created_at, updated_at, disabled, status, tls, tls_skip_verify, query_examples_disabled, max_query_log_size, table_count_tablestats_group_limit, rds_basic_metrics_disabled, rds_enhanced_metrics_disabled, push_metrics) + agents(mongo_db_tls_options, agent_id, agent_type, runs_on_node_id, created_at, updated_at, disabled, status, tls, tls_skip_verify, query_examples_disabled, max_query_log_size, table_count_tablestats_group_limit, rds_basic_metrics_disabled, rds_enhanced_metrics_disabled, push_metrics) VALUES ('{"stats_collections": "db.col1,db.col2,db.col3"}', 'id', 'pmm-agent', 'node_id' , '03/03/2014 02:03:04', '03/03/2014 02:03:04', false, 'alive', false, false, false, 0, 0, false, false, false)`, ) @@ -353,7 +350,7 @@ func TestDatabaseMigrations(t *testing.T) { var agentID string var mongoDBOptions *models.MongoDBOptions - err = sqlDB.QueryRow(`SELECT agent_id, mongo_options FROM agents WHERE agent_id = $1`, "id").Scan(&agentID, &mongoDBOptions) + err = sqlDB.QueryRow(`SELECT agent_id, mongo_db_tls_options FROM agents WHERE agent_id = $1`, "id").Scan(&agentID, &mongoDBOptions) require.NoError(t, err) require.Equal(t, "id", agentID) @@ -376,7 +373,7 @@ func TestDatabaseMigrations(t *testing.T) { // Insert dummy agent in DB _, err = sqlDB.ExecContext(context.Background(), `INSERT INTO - agents(mongo_options, agent_id, agent_type, runs_on_node_id, created_at, updated_at, disabled, status, tls, tls_skip_verify, query_examples_disabled, max_query_log_size, table_count_tablestats_group_limit, rds_basic_metrics_disabled, rds_enhanced_metrics_disabled, push_metrics) + agents(mongo_db_tls_options, agent_id, agent_type, runs_on_node_id, created_at, updated_at, disabled, status, tls, tls_skip_verify, query_examples_disabled, max_query_log_size, table_count_tablestats_group_limit, rds_basic_metrics_disabled, rds_enhanced_metrics_disabled, push_metrics) VALUES ('{"stats_collections": ["db.col1", "db.col2", "db.col3"]}', 'id', 'pmm-agent', 'node_id' , '03/03/2014 02:03:04', '03/03/2014 02:03:04', false, 'alive', false, false, false, 0, 0, false, false, false)`, ) @@ -387,7 +384,7 @@ func TestDatabaseMigrations(t *testing.T) { var agentID string var mongoDBOptions *models.MongoDBOptions - err = sqlDB.QueryRow(`SELECT agent_id, mongo_options FROM agents WHERE agent_id = $1`, "id").Scan(&agentID, &mongoDBOptions) + err = sqlDB.QueryRow(`SELECT agent_id, mongo_db_tls_options FROM agents WHERE agent_id = $1`, "id").Scan(&agentID, &mongoDBOptions) require.NoError(t, err) require.Equal(t, "id", agentID) diff --git a/managed/services/encryption/encryption_rotation.go b/managed/services/encryption/encryption_rotation.go index f73f6d96ed..b522998fa9 100644 --- a/managed/services/encryption/encryption_rotation.go +++ b/managed/services/encryption/encryption_rotation.go @@ -130,7 +130,7 @@ func pmmServerStatusWithRetries(status string) bool { func rotateEncryptionKey(db *reform.DB, dbName string) error { return db.InTransaction(func(tx *reform.TX) error { logrus.Infof("DB %s is being decrypted", dbName) - err := models.DecryptDB(tx, dbName, models.DefaultAgentEncryptionColumns) + err := models.DecryptDB(tx, dbName, models.DefaultAgentEncryptionColumnsV3) if err != nil { return err } @@ -144,7 +144,7 @@ func rotateEncryptionKey(db *reform.DB, dbName string) error { logrus.Infof("New encryption key generated") logrus.Infof("DB %s is being encrypted", dbName) - err = models.EncryptDB(tx, dbName, models.DefaultAgentEncryptionColumns) + err = models.EncryptDB(tx, dbName, models.DefaultAgentEncryptionColumnsV3) if err != nil { if e := encryption.RestoreOldEncryptionKey(); e != nil { return errors.Wrap(err, e.Error()) From e6addf1321ab1736974402414f773d6afaacb280 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Mon, 2 Dec 2024 10:12:36 +0100 Subject: [PATCH 17/60] PMM-5086-12634 Format. --- managed/services/agents/mongodb.go | 1 + managed/services/agents/node.go | 1 + 2 files changed, 2 insertions(+) diff --git a/managed/services/agents/mongodb.go b/managed/services/agents/mongodb.go index 5152065eb2..c59ff62a19 100644 --- a/managed/services/agents/mongodb.go +++ b/managed/services/agents/mongodb.go @@ -22,6 +22,7 @@ import ( "time" "github.com/AlekSi/pointer" + agentv1 "github.com/percona/pmm/api/agent/v1" inventoryv1 "github.com/percona/pmm/api/inventory/v1" "github.com/percona/pmm/managed/models" diff --git a/managed/services/agents/node.go b/managed/services/agents/node.go index cc20300ccf..db8a4d5951 100644 --- a/managed/services/agents/node.go +++ b/managed/services/agents/node.go @@ -19,6 +19,7 @@ import ( "sort" "github.com/AlekSi/pointer" + agentv1 "github.com/percona/pmm/api/agent/v1" inventoryv1 "github.com/percona/pmm/api/inventory/v1" "github.com/percona/pmm/managed/models" From e72961b7f5e42c21d4888e0e5aafa1a7aa85e136 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Mon, 2 Dec 2024 12:08:25 +0100 Subject: [PATCH 18/60] PMM-5086-12634 Fix double decrypt bug. --- managed/models/agent_helpers.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/managed/models/agent_helpers.go b/managed/models/agent_helpers.go index 790187794a..1d39d39aac 100644 --- a/managed/models/agent_helpers.go +++ b/managed/models/agent_helpers.go @@ -236,8 +236,7 @@ func FindAgents(q *reform.Querier, filters AgentFilters) ([]*Agent, error) { agents := make([]*Agent, len(structs)) for i, s := range structs { - decryptedAgent := DecryptAgent(*s.(*Agent)) //nolint:forcetypeassert - agents[i] = &decryptedAgent + agents[i] = s.(*Agent) //nolint:forcetypeassert } return agents, nil From b420d9be62e288ccc04fe8cde9df85618dfc115e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Mon, 2 Dec 2024 13:56:33 +0100 Subject: [PATCH 19/60] PMM-508612634 Helpers push metrics fix. --- managed/models/agent_helpers.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/managed/models/agent_helpers.go b/managed/models/agent_helpers.go index 1d39d39aac..51acd43cc1 100644 --- a/managed/models/agent_helpers.go +++ b/managed/models/agent_helpers.go @@ -31,6 +31,12 @@ import ( "github.com/percona/pmm/version" ) +const ( + // TODO figure out why it is included as PushMetrics not push_metrics + pushMetricsTrue = "((exporter_options ? 'PushMetrics') AND (exporter_options->>'PushMetrics')::boolean = true)" + pushMetricsFalse = "(NOT (exporter_options ? 'PushMetrics') OR (exporter_options->>'PushMetrics')::boolean = false)" +) + // MySQLOptionsParams contains methods to create MySQLOptions object. type MySQLOptionsParams interface { GetTlsCa() string @@ -473,9 +479,9 @@ func FindAgentsForScrapeConfig(q *reform.Querier, pmmAgentID *string, pushMetric } if pushMetrics { - conditions = append(conditions, "push_metrics") + conditions = append(conditions, pushMetricsTrue) } else { - conditions = append(conditions, "NOT push_metrics") + conditions = append(conditions, pushMetricsFalse) } conditions = append(conditions, "NOT disabled", "listen_port IS NOT NULL") @@ -495,7 +501,7 @@ func FindAgentsForScrapeConfig(q *reform.Querier, pmmAgentID *string, pushMetric // FindPMMAgentsIDsWithPushMetrics returns pmm-agents-ids with agent, that use push_metrics mode. func FindPMMAgentsIDsWithPushMetrics(q *reform.Querier) ([]string, error) { - structs, err := q.SelectAllFrom(AgentTable, "WHERE NOT disabled AND pmm_agent_id IS NOT NULL AND push_metrics ORDER BY agent_id") + structs, err := q.SelectAllFrom(AgentTable, fmt.Sprintf("WHERE NOT disabled AND pmm_agent_id IS NOT NULL AND %s ORDER BY agent_id", pushMetricsTrue)) if err != nil { return nil, status.Error(codes.FailedPrecondition, "Couldn't get agents") } From 7a0500a0acc380dfd030fe8c74c0a6178535ebe2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Mon, 2 Dec 2024 14:00:56 +0100 Subject: [PATCH 20/60] PMM-12634 Expect false in exporter options. --- managed/models/agent_helpers_test.go | 94 ++++++++++++++-------------- 1 file changed, 46 insertions(+), 48 deletions(-) diff --git a/managed/models/agent_helpers_test.go b/managed/models/agent_helpers_test.go index 48e363ff80..2deebfafcd 100644 --- a/managed/models/agent_helpers_test.go +++ b/managed/models/agent_helpers_test.go @@ -165,17 +165,15 @@ func TestAgentHelpers(t *testing.T) { }, }, &models.Agent{ - AgentID: "A9", - AgentType: models.MongoDBExporterType, - PMMAgentID: pointer.ToString("A9"), - RunsOnNodeID: nil, - NodeID: pointer.ToString("N1"), - ListenPort: pointer.ToUint16(8200), - TLS: true, - TLSSkipVerify: true, - ExporterOptions: &models.ExporterOptions{ - PushMetrics: false, - }, + AgentID: "A9", + AgentType: models.MongoDBExporterType, + PMMAgentID: pointer.ToString("A9"), + RunsOnNodeID: nil, + NodeID: pointer.ToString("N1"), + ListenPort: pointer.ToUint16(8200), + TLS: true, + TLSSkipVerify: true, + ExporterOptions: &models.ExporterOptions{}, MongoDBOptions: &models.MongoDBOptions{ TLSCertificateKey: "tls_certificate_key", TLSCertificateKeyFilePassword: "tls_certificate_key_file_password", @@ -188,18 +186,16 @@ func TestAgentHelpers(t *testing.T) { }, }, &models.Agent{ - AgentID: "A10", - AgentType: models.MongoDBExporterType, - PMMAgentID: pointer.ToString("A10"), - RunsOnNodeID: nil, - NodeID: pointer.ToString("N1"), - ListenPort: pointer.ToUint16(8200), - TLS: true, - TLSSkipVerify: true, - ExporterOptions: &models.ExporterOptions{ - PushMetrics: false, - }, - MongoDBOptions: nil, // this test is specific for nil MongoDBOptions + AgentID: "A10", + AgentType: models.MongoDBExporterType, + PMMAgentID: pointer.ToString("A10"), + RunsOnNodeID: nil, + NodeID: pointer.ToString("N1"), + ListenPort: pointer.ToUint16(8200), + TLS: true, + TLSSkipVerify: true, + ExporterOptions: &models.ExporterOptions{}, + MongoDBOptions: nil, // this test is specific for nil MongoDBOptions }, } { require.NoError(t, q.Insert(str)) @@ -220,21 +216,19 @@ func TestAgentHelpers(t *testing.T) { require.NoError(t, err) expected := []*models.Agent{ { - CreatedAt: now, - UpdatedAt: now, - Status: models.AgentStatusUnknown, - AgentID: "A10", - AgentType: models.MongoDBExporterType, - PMMAgentID: pointer.ToString("A10"), - RunsOnNodeID: nil, - NodeID: pointer.ToString("N1"), - ListenPort: pointer.ToUint16(8200), - TLS: true, - TLSSkipVerify: true, - ExporterOptions: &models.ExporterOptions{ - PushMetrics: false, - }, - MongoDBOptions: nil, // this test is specific for nil MongoDBOptions + CreatedAt: now, + UpdatedAt: now, + Status: models.AgentStatusUnknown, + AgentID: "A10", + AgentType: models.MongoDBExporterType, + PMMAgentID: pointer.ToString("A10"), + RunsOnNodeID: nil, + NodeID: pointer.ToString("N1"), + ListenPort: pointer.ToUint16(8200), + TLS: true, + TLSSkipVerify: true, + ExporterOptions: &models.ExporterOptions{}, + MongoDBOptions: nil, // this test is specific for nil MongoDBOptions }, { AgentID: "A3", @@ -281,6 +275,9 @@ func TestAgentHelpers(t *testing.T) { ListenPort: pointer.ToUint16OrNil(8200), TLS: true, TLSSkipVerify: true, + ExporterOptions: &models.ExporterOptions{ + ExposeExporter: false, + }, MongoDBOptions: &models.MongoDBOptions{ TLSCertificateKey: "tls_certificate_key", TLSCertificateKeyFilePassword: "tls_certificate_key_file_password", @@ -292,16 +289,17 @@ func TestAgentHelpers(t *testing.T) { }, }, { - AgentID: "A9", - AgentType: "mongodb_exporter", - NodeID: pointer.ToStringOrNil("N1"), - PMMAgentID: pointer.ToStringOrNil("A9"), - CreatedAt: now, - UpdatedAt: now, - Status: models.AgentStatusUnknown, - ListenPort: pointer.ToUint16OrNil(8200), - TLS: true, - TLSSkipVerify: true, + AgentID: "A9", + AgentType: "mongodb_exporter", + NodeID: pointer.ToStringOrNil("N1"), + PMMAgentID: pointer.ToStringOrNil("A9"), + CreatedAt: now, + UpdatedAt: now, + Status: models.AgentStatusUnknown, + ListenPort: pointer.ToUint16OrNil(8200), + TLS: true, + TLSSkipVerify: true, + ExporterOptions: &models.ExporterOptions{}, MongoDBOptions: &models.MongoDBOptions{ TLSCertificateKey: "tls_certificate_key", TLSCertificateKeyFilePassword: "tls_certificate_key_file_password", From 0b355127b045e6d32ae75124c6ae93f7e422d76d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Mon, 2 Dec 2024 16:12:35 +0100 Subject: [PATCH 21/60] Revert "PMM-5086-12634 Fix double decrypt bug." This reverts commit e72961b7f5e42c21d4888e0e5aafa1a7aa85e136. --- managed/models/agent_helpers.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/managed/models/agent_helpers.go b/managed/models/agent_helpers.go index 51acd43cc1..f5e595b1ea 100644 --- a/managed/models/agent_helpers.go +++ b/managed/models/agent_helpers.go @@ -242,7 +242,8 @@ func FindAgents(q *reform.Querier, filters AgentFilters) ([]*Agent, error) { agents := make([]*Agent, len(structs)) for i, s := range structs { - agents[i] = s.(*Agent) //nolint:forcetypeassert + decryptedAgent := DecryptAgent(*s.(*Agent)) //nolint:forcetypeassert + agents[i] = &decryptedAgent } return agents, nil From 463b6f0feec4146f9fd669c5b63d7f3ded971d1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Mon, 2 Dec 2024 16:34:23 +0100 Subject: [PATCH 22/60] PMM-5086-12634 Handle nil in scrape configs. --- .../victoriametrics/scrape_configs.go | 30 +++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/managed/services/victoriametrics/scrape_configs.go b/managed/services/victoriametrics/scrape_configs.go index 4324afc33f..1420a09be8 100644 --- a/managed/services/victoriametrics/scrape_configs.go +++ b/managed/services/victoriametrics/scrape_configs.go @@ -209,6 +209,10 @@ func scrapeConfigsForNodeExporter(params *scrapeConfigParams) ([]*config.ScrapeC var err error var hrCollect []string + if params.agent.ExporterOptions == nil { + params.agent.ExporterOptions = &models.ExporterOptions{} + } + if params.node.Distro != "darwin" { mrCollect := []string{ "hwmon", @@ -285,6 +289,11 @@ func scrapeConfigsForMySQLdExporter(params *scrapeConfigParams) ([]*config.Scrap "standard.go", "standard.process", } + + if params.agent.ExporterOptions == nil { + params.agent.ExporterOptions = &models.ExporterOptions{} + } + hrOptions = collectors.FilterOutCollectors("", hrOptions, params.agent.ExporterOptions.DisabledCollectors) hr, err := scrapeConfigForStandardExporter("hr", params.metricsResolution.HR, params, hrOptions) @@ -356,6 +365,10 @@ func scrapeConfigsForMySQLdExporter(params *scrapeConfigParams) ([]*config.Scrap } func scrapeConfigsForMongoDBExporter(params *scrapeConfigParams) ([]*config.ScrapeConfig, error) { + if params.agent.ExporterOptions == nil { + params.agent.ExporterOptions = &models.ExporterOptions{} + } + // Old pmm-agents doesn't have support of multiple resolution, // so requesting mongodb_exporter metrics in two resolutions increases CPU and Memory usage. if params.pmmAgentVersion == nil || params.pmmAgentVersion.Less(version.MustParse("2.26.0-0")) { @@ -425,6 +438,11 @@ func scrapeConfigsForPostgresExporter(params *scrapeConfigParams) ([]*config.Scr "standard.process", "postgres", } + + if params.agent.ExporterOptions == nil { + params.agent.ExporterOptions = &models.ExporterOptions{} + } + hrOptions = collectors.FilterOutCollectors("", hrOptions, params.agent.ExporterOptions.DisabledCollectors) hr, err := scrapeConfigForStandardExporter("hr", params.metricsResolution.HR, params, hrOptions) if err != nil { @@ -532,8 +550,12 @@ func scrapeConfigsForExternalExporter(s *models.MetricsResolutions, params *scra if err != nil { return nil, err } - interval := s.MR + + if params.agent.ExporterOptions == nil { + params.agent.ExporterOptions = &models.ExporterOptions{} + } + cfg := &config.ScrapeConfig{ JobName: jobName(params.agent, "mr"), ScrapeInterval: config.Duration(interval), @@ -569,8 +591,12 @@ func scrapeConfigsForVMAgent(s *models.MetricsResolutions, params *scrapeConfigP if err != nil { return nil, err } - interval := s.MR + + if params.agent.ExporterOptions == nil { + params.agent.ExporterOptions = &models.ExporterOptions{} + } + cfg := &config.ScrapeConfig{ JobName: jobName(params.agent, "mr"), ScrapeInterval: config.Duration(interval), From 78867ac6ef21fc0aeb04b960b8a8342ce75f03a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Mon, 2 Dec 2024 16:53:33 +0100 Subject: [PATCH 23/60] PMM-5086-12634 Fix encryption rotation test. --- managed/services/encryption/encryption_rotation_test.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/managed/services/encryption/encryption_rotation_test.go b/managed/services/encryption/encryption_rotation_test.go index 0573576422..72254f8e22 100644 --- a/managed/services/encryption/encryption_rotation_test.go +++ b/managed/services/encryption/encryption_rotation_test.go @@ -21,7 +21,6 @@ import ( "testing" "time" - "github.com/AlekSi/pointer" "github.com/pkg/errors" "github.com/stretchr/testify/require" @@ -40,7 +39,7 @@ const ( ) func TestEncryptionRotation(t *testing.T) { - db := testdb.Open(t, models.SkipFixtures, pointer.ToInt(88)) + db := testdb.Open(t, models.SkipFixtures, nil) defer db.Close() //nolint:errcheck err := createOriginEncryptionKey() @@ -98,8 +97,8 @@ func insertTestData(db *sql.DB) error { return err } _, err = db.Exec( - "INSERT INTO agents (agent_id, agent_type, username, password, runs_on_node_id, pmm_agent_id, disabled, status, created_at, updated_at, tls, tls_skip_verify, max_query_length, query_examples_disabled, comments_parsing_disabled, max_query_log_size, table_count_tablestats_group_limit, rds_basic_metrics_disabled, rds_enhanced_metrics_disabled, push_metrics, expose_exporter) "+ - "VALUES ('1', 'pmm-agent', $1, $2, '1', NULL, false, '', $3, $4, false, false, 0, false, true, 0, 0, true, true, false, false)", + `INSERT INTO agents (agent_id, agent_type, username, password, runs_on_node_id, pmm_agent_id, disabled, status, created_at, updated_at, tls, tls_skip_verify, qan_options, mysql_options, aws_options, exporter_options) `+ + `VALUES ('1', 'pmm-agent', $1, $2, '1', NULL, false, '', $3, $4, false, false, '{"max_query_length": 0, "query_examples_disabled": false, "comments_parsing_disabled": true, "max_query_log_size": 0}', '{"table_count_tablestats_group_limit": 0}', '{"rds_basic_metrics_disabled": true, "rds_enhanced_metrics_disabled": true}', '{"push_metrics": false, "expose_exporter": false}')`, originUsernameHash, originPasswordHash, now, now) if err != nil { return err From 9e4982ce8c97f5ff86739a46c3f236043c22577a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Mon, 2 Dec 2024 17:04:55 +0100 Subject: [PATCH 24/60] PMM-5086-12634 Mongo options nil. --- managed/services/agents/mongodb.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/managed/services/agents/mongodb.go b/managed/services/agents/mongodb.go index c59ff62a19..51da37fb98 100644 --- a/managed/services/agents/mongodb.go +++ b/managed/services/agents/mongodb.go @@ -93,6 +93,10 @@ func mongodbExporterConfig(node *models.Node, service *models.Service, exporter func getArgs(exporter *models.Agent, tdp *models.DelimiterPair, listenAddress string, pmmAgentVersion *version.Parsed) []string { var args []string + if exporter.ExporterOptions == nil { + exporter.ExporterOptions = &models.ExporterOptions{} + } + switch { case !pmmAgentVersion.Less(v2_25_0): // >= 2.26.0 args = buildBaseArgs(listenAddress, tdp) From 403bf8ea83ee027b082445d7412ce64bfabf4c78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Mon, 2 Dec 2024 17:34:13 +0100 Subject: [PATCH 25/60] PMM-5086-12634 Converters. --- managed/models/agent_helpers.go | 2 +- managed/services/converters.go | 63 ++++++++++++++++++++++++++++----- 2 files changed, 55 insertions(+), 10 deletions(-) diff --git a/managed/models/agent_helpers.go b/managed/models/agent_helpers.go index f5e595b1ea..288d0c3a2f 100644 --- a/managed/models/agent_helpers.go +++ b/managed/models/agent_helpers.go @@ -53,7 +53,7 @@ func MySQLOptionsFromRequest(params MySQLOptionsParams) *MySQLOptions { TLSKey: params.GetTlsKey(), } } - return nil + return &MySQLOptions{} } // PostgreSQLOptionsParams contains methods to create PostgreSQLOptions object. diff --git a/managed/services/converters.go b/managed/services/converters.go index 38b9d7ed72..c950216a34 100644 --- a/managed/services/converters.go +++ b/managed/services/converters.go @@ -218,6 +218,11 @@ func ToAPIAgent(q *reform.Querier, agent *models.Agent) (inventoryv1.Agent, erro } serviceID = service.ServiceID } + + if agent.ExporterOptions == nil { + agent.ExporterOptions = &models.ExporterOptions{} + } + processExecPath := pointer.GetString(agent.ProcessExecPath) switch agent.AgentType { case models.PMMAgentType: @@ -245,6 +250,10 @@ func ToAPIAgent(q *reform.Querier, agent *models.Agent) (inventoryv1.Agent, erro }, nil case models.MySQLdExporterType: + if agent.MySQLOptions == nil { + agent.MySQLOptions = &models.MySQLOptions{} + } + return &inventoryv1.MySQLdExporter{ AgentId: agent.AgentID, PmmAgentId: pointer.GetString(agent.PMMAgentID), @@ -268,6 +277,10 @@ func ToAPIAgent(q *reform.Querier, agent *models.Agent) (inventoryv1.Agent, erro }, nil case models.MongoDBExporterType: + if agent.MongoDBOptions == nil { + agent.MongoDBOptions = &models.MongoDBOptions{} + } + exporter := &inventoryv1.MongoDBExporter{ AgentId: agent.AgentID, PmmAgentId: pointer.GetString(agent.PMMAgentID), @@ -286,14 +299,18 @@ func ToAPIAgent(q *reform.Querier, agent *models.Agent) (inventoryv1.Agent, erro ExposeExporter: agent.ExporterOptions.ExposeExporter, MetricsResolutions: ConvertMetricsResolutions(agent.ExporterOptions.MetricsResolutions), } - if agent.MongoDBOptions != nil { - exporter.StatsCollections = agent.MongoDBOptions.StatsCollections - exporter.CollectionsLimit = agent.MongoDBOptions.CollectionsLimit - exporter.EnableAllCollectors = agent.MongoDBOptions.EnableAllCollectors - } + + exporter.StatsCollections = agent.MongoDBOptions.StatsCollections + exporter.CollectionsLimit = agent.MongoDBOptions.CollectionsLimit + exporter.EnableAllCollectors = agent.MongoDBOptions.EnableAllCollectors + return exporter, nil case models.PostgresExporterType: + if agent.PostgreSQLOptions == nil { + agent.PostgreSQLOptions = &models.PostgreSQLOptions{} + } + exporter := &inventoryv1.PostgresExporter{ AgentId: agent.AgentID, PmmAgentId: pointer.GetString(agent.PMMAgentID), @@ -312,12 +329,16 @@ func ToAPIAgent(q *reform.Querier, agent *models.Agent) (inventoryv1.Agent, erro ExposeExporter: agent.ExporterOptions.ExposeExporter, MetricsResolutions: ConvertMetricsResolutions(agent.ExporterOptions.MetricsResolutions), } - if agent.PostgreSQLOptions != nil { - exporter.AutoDiscoveryLimit = agent.PostgreSQLOptions.AutoDiscoveryLimit - exporter.MaxExporterConnections = agent.PostgreSQLOptions.MaxExporterConnections - } + + exporter.AutoDiscoveryLimit = agent.PostgreSQLOptions.AutoDiscoveryLimit + exporter.MaxExporterConnections = agent.PostgreSQLOptions.MaxExporterConnections + return exporter, nil case models.QANMySQLPerfSchemaAgentType: + if agent.QANOptions == nil { + agent.QANOptions = &models.QANOptions{} + } + return &inventoryv1.QANMySQLPerfSchemaAgent{ AgentId: agent.AgentID, PmmAgentId: pointer.GetString(agent.PMMAgentID), @@ -336,6 +357,10 @@ func ToAPIAgent(q *reform.Querier, agent *models.Agent) (inventoryv1.Agent, erro }, nil case models.QANMySQLSlowlogAgentType: + if agent.QANOptions == nil { + agent.QANOptions = &models.QANOptions{} + } + return &inventoryv1.QANMySQLSlowlogAgent{ AgentId: agent.AgentID, PmmAgentId: pointer.GetString(agent.PMMAgentID), @@ -354,6 +379,10 @@ func ToAPIAgent(q *reform.Querier, agent *models.Agent) (inventoryv1.Agent, erro }, nil case models.QANMongoDBProfilerAgentType: + if agent.QANOptions == nil { + agent.QANOptions = &models.QANOptions{} + } + return &inventoryv1.QANMongoDBProfilerAgent{ AgentId: agent.AgentID, PmmAgentId: pointer.GetString(agent.PMMAgentID), @@ -391,6 +420,10 @@ func ToAPIAgent(q *reform.Querier, agent *models.Agent) (inventoryv1.Agent, erro }, nil case models.QANPostgreSQLPgStatementsAgentType: + if agent.QANOptions == nil { + agent.QANOptions = &models.QANOptions{} + } + return &inventoryv1.QANPostgreSQLPgStatementsAgent{ AgentId: agent.AgentID, PmmAgentId: pointer.GetString(agent.PMMAgentID), @@ -408,6 +441,10 @@ func ToAPIAgent(q *reform.Querier, agent *models.Agent) (inventoryv1.Agent, erro }, nil case models.QANPostgreSQLPgStatMonitorAgentType: + if agent.QANOptions == nil { + agent.QANOptions = &models.QANOptions{} + } + return &inventoryv1.QANPostgreSQLPgStatMonitorAgent{ AgentId: agent.AgentID, PmmAgentId: pointer.GetString(agent.PMMAgentID), @@ -426,6 +463,10 @@ func ToAPIAgent(q *reform.Querier, agent *models.Agent) (inventoryv1.Agent, erro }, nil case models.RDSExporterType: + if agent.AWSOptions == nil { + agent.AWSOptions = &models.AWSOptions{} + } + return &inventoryv1.RDSExporter{ AgentId: agent.AgentID, PmmAgentId: pointer.GetString(agent.PMMAgentID), @@ -467,6 +508,10 @@ func ToAPIAgent(q *reform.Querier, agent *models.Agent) (inventoryv1.Agent, erro }, nil case models.AzureDatabaseExporterType: + if agent.AzureOptions == nil { + agent.AzureOptions = &models.AzureOptions{} + } + return &inventoryv1.AzureDatabaseExporter{ AgentId: agent.AgentID, PmmAgentId: pointer.GetString(agent.PMMAgentID), From 95b7264992b3460c50deb748324d0f0245032768 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Mon, 2 Dec 2024 17:44:40 +0100 Subject: [PATCH 26/60] PMM-5086-12634 MySQL fix. --- managed/services/agents/mysql.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/managed/services/agents/mysql.go b/managed/services/agents/mysql.go index f60acca943..60e3aec6d0 100644 --- a/managed/services/agents/mysql.go +++ b/managed/services/agents/mysql.go @@ -104,8 +104,8 @@ func mysqldExporterConfig( args = append(args, tablestatsGroup...) } - if exporter.MySQLOptions == nil { - exporter.MySQLOptions = &models.MySQLOptions{} + if exporter.ExporterOptions == nil { + exporter.ExporterOptions = &models.ExporterOptions{} } args = collectors.FilterOutCollectors("--collect.", args, exporter.ExporterOptions.DisabledCollectors) From fb80295b74f565d710b7dac4acee3d381421b6c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Tue, 3 Dec 2024 09:11:43 +0100 Subject: [PATCH 27/60] PMM-5086-12634 Fix proxysql nil. --- managed/services/agents/proxysql.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/managed/services/agents/proxysql.go b/managed/services/agents/proxysql.go index 9700af1944..4c36919ca8 100644 --- a/managed/services/agents/proxysql.go +++ b/managed/services/agents/proxysql.go @@ -55,6 +55,10 @@ func proxysqlExporterConfig(node *models.Node, service *models.Service, exporter args = append(args, "-collect.runtime_mysql_servers") } + if exporter.ExporterOptions == nil { + exporter.ExporterOptions = &models.ExporterOptions{} + } + if exporter.ExporterOptions.MetricsPath != nil { args = append(args, "-web.telemetry-path="+*exporter.ExporterOptions.MetricsPath) } From d7c6980a8468aff5247511e832178acd0803e92d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Tue, 3 Dec 2024 09:12:00 +0100 Subject: [PATCH 28/60] PMM-5086-12634 Fix roster test. --- managed/models/agent_helpers.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/managed/models/agent_helpers.go b/managed/models/agent_helpers.go index 288d0c3a2f..1a5cd4f6ab 100644 --- a/managed/models/agent_helpers.go +++ b/managed/models/agent_helpers.go @@ -227,7 +227,7 @@ func FindAgents(q *reform.Querier, filters AgentFilters) ([]*Agent, error) { idx++ } if filters.AWSAccessKey != "" { - conditions = append(conditions, fmt.Sprintf("aws_access_key = %s", q.Placeholder(idx))) + conditions = append(conditions, fmt.Sprintf("(aws_options ? 'aws_access_key' AND aws_options->>'aws_access_key' = %s)", q.Placeholder(idx))) args = append(args, filters.AWSAccessKey) } From d3ca4e3fdd026113577fcc726813e7992fd403aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Tue, 3 Dec 2024 09:48:55 +0100 Subject: [PATCH 29/60] PMM-5086-12634 Fix for TextFiles and templateParams. --- managed/models/agent_model.go | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/managed/models/agent_model.go b/managed/models/agent_model.go index 8b177d6797..c0f78e385a 100644 --- a/managed/models/agent_model.go +++ b/managed/models/agent_model.go @@ -673,7 +673,10 @@ func (s Agent) Files() map[string]string { if s.MySQLOptions.TLSKey != "" { files["tlsKey"] = s.MySQLOptions.TLSKey } - return files + + if len(files) != 0 { + return files + } } return nil case ProxySQLExporterType: @@ -687,7 +690,10 @@ func (s Agent) Files() map[string]string { if s.MongoDBOptions.TLSCertificateKey != "" { files[certificateKeyFilePlaceholder] = s.MongoDBOptions.TLSCertificateKey } - return files + + if len(files) != 0 { + return files + } } return nil case PostgresExporterType, QANPostgreSQLPgStatementsAgentType, QANPostgreSQLPgStatMonitorAgentType: @@ -703,7 +709,10 @@ func (s Agent) Files() map[string]string { if s.PostgreSQLOptions.SSLKey != "" { files[certificateKeyFilePlaceholder] = s.PostgreSQLOptions.SSLKey } - return files + + if len(files) != 0 { + return files + } } return nil default: @@ -726,15 +735,15 @@ func (s Agent) TemplateDelimiters(svc *Service) *DelimiterPair { switch svc.ServiceType { case MySQLServiceType: - if s.MySQLOptions != nil { + if s.MySQLOptions != nil && s.MySQLOptions.TLSKey != "" { templateParams = append(templateParams, s.MySQLOptions.TLSKey) } case MongoDBServiceType: - if s.MongoDBOptions != nil { + if s.MongoDBOptions != nil && s.MongoDBOptions.TLSCertificateKeyFilePassword != "" { templateParams = append(templateParams, s.MongoDBOptions.TLSCertificateKeyFilePassword) } case PostgreSQLServiceType: - if s.PostgreSQLOptions != nil { + if s.PostgreSQLOptions != nil && s.PostgreSQLOptions.SSLKey != "" { templateParams = append(templateParams, s.PostgreSQLOptions.SSLKey) } case ProxySQLServiceType: From b1e2d6525dbce1cb9435a9d2c67ba32928bb8c2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Tue, 3 Dec 2024 10:27:07 +0100 Subject: [PATCH 30/60] PMM-5086-12634 Fix PG test. --- managed/services/agents/postgresql_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/managed/services/agents/postgresql_test.go b/managed/services/agents/postgresql_test.go index 444d370c43..62838401ae 100644 --- a/managed/services/agents/postgresql_test.go +++ b/managed/services/agents/postgresql_test.go @@ -154,7 +154,9 @@ func (s *PostgresExporterConfigTestSuite) TestDisabledCollectors() { s.postgresql.Address = nil s.postgresql.Port = nil s.postgresql.Socket = pointer.ToString("/var/run/postgres") - s.exporter.ExporterOptions.DisabledCollectors = []string{"custom_query.hr", "custom_query.hr.directory", "locks"} + s.exporter.ExporterOptions = &models.ExporterOptions{ + DisabledCollectors: []string{"custom_query.hr", "custom_query.hr.directory", "locks"}, + } actual, err := postgresExporterConfig(s.node, s.postgresql, s.exporter, exposeSecrets, s.pmmAgentVersion) s.NoError(err, "Failed to create exporter config") From c2dbecb2c96e553258452c8fee7131f81a463416 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Tue, 3 Dec 2024 15:32:19 +0100 Subject: [PATCH 31/60] PMM-5086-12634 Fix wrong annotation. --- managed/models/agent_model.go | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/managed/models/agent_model.go b/managed/models/agent_model.go index c0f78e385a..be69e7c675 100644 --- a/managed/models/agent_model.go +++ b/managed/models/agent_model.go @@ -80,12 +80,12 @@ const PMMServerAgentID = string("pmm-server") // a special ID, reserved for PMM // ExporterOptions represents structure for special Exporter options. type ExporterOptions struct { - ExposeExporter bool `reform:"expose_exporter"` - PushMetrics bool `reform:"push_metrics"` - DisabledCollectors pq.StringArray `reform:"disabled_collectors"` - MetricsResolutions *MetricsResolutions `reform:"metrics_resolutions"` - MetricsPath *string `reform:"metrics_path"` - MetricsScheme *string `reform:"metrics_scheme"` + ExposeExporter bool `json:"expose_exporter"` + PushMetrics bool `json:"push_metrics"` + DisabledCollectors pq.StringArray `json:"disabled_collectors"` + MetricsResolutions *MetricsResolutions `json:"metrics_resolutions"` + MetricsPath *string `json:"metrics_path"` + MetricsScheme *string `json:"metrics_scheme"` } // Value implements database/sql/driver.Valuer interface. Should be defined on the value. @@ -96,10 +96,10 @@ func (c *ExporterOptions) Scan(src interface{}) error { return jsonScan(c, src) // QANOptions represents structure for special QAN options. type QANOptions struct { - MaxQueryLength int32 `reform:"max_query_length"` - MaxQueryLogSize int64 `reform:"max_query_log_size"` - QueryExamplesDisabled bool `reform:"query_examples_disabled"` - CommentsParsingDisabled bool `reform:"comments_parsing_disabled"` + MaxQueryLength int32 `json:"max_query_length"` + MaxQueryLogSize int64 `json:"max_query_log_size"` + QueryExamplesDisabled bool `json:"query_examples_disabled"` + CommentsParsingDisabled bool `json:"comments_parsing_disabled"` } // Value implements database/sql/driver.Valuer interface. Should be defined on the value. @@ -110,10 +110,10 @@ func (c *QANOptions) Scan(src interface{}) error { return jsonScan(c, src) } // AWSOptions represents structure for special AWS options. type AWSOptions struct { - AWSAccessKey string `reform:"aws_access_key"` - AWSSecretKey string `reform:"aws_secret_key"` - RDSBasicMetricsDisabled bool `reform:"rds_basic_metrics_disabled"` - RDSEnhancedMetricsDisabled bool `reform:"rds_enhanced_metrics_disabled"` + AWSAccessKey string `json:"aws_access_key"` + AWSSecretKey string `json:"aws_secret_key"` + RDSBasicMetricsDisabled bool `json:"rds_basic_metrics_disabled"` + RDSEnhancedMetricsDisabled bool `json:"rds_enhanced_metrics_disabled"` } // Value implements database/sql/driver.Valuer interface. Should be defined on the value. @@ -162,13 +162,13 @@ type MySQLOptions struct { TLSKey string `json:"tls_key"` // TableCount stores last known table count. NULL if unknown. - TableCount *int32 `reform:"table_count"` + TableCount *int32 `json:"table_count"` // Tablestats group collectors are disabled if there are more than that number of tables. // 0 means tablestats group collectors are always enabled (no limit). // Negative value means tablestats group collectors are always disabled. // See IsMySQLTablestatsGroupEnabled method. - TableCountTablestatsGroupLimit int32 `reform:"table_count_tablestats_group_limit"` + TableCountTablestatsGroupLimit int32 `json:"table_count_tablestats_group_limit"` } // Value implements database/sql/driver.Valuer interface. Should be defined on the value. From 233775240345be2d8c24e635fa882ba1dbd8496c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Tue, 3 Dec 2024 15:33:55 +0100 Subject: [PATCH 32/60] PMM-5086-12634 Fix lint about receivers. --- managed/models/agent_model.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/managed/models/agent_model.go b/managed/models/agent_model.go index be69e7c675..8cfb62f231 100644 --- a/managed/models/agent_model.go +++ b/managed/models/agent_model.go @@ -89,7 +89,7 @@ type ExporterOptions struct { } // Value implements database/sql/driver.Valuer interface. Should be defined on the value. -func (c ExporterOptions) Value() (driver.Value, error) { return jsonValue(c) } +func (c *ExporterOptions) Value() (driver.Value, error) { return jsonValue(c) } // Scan implements database/sql.Scanner interface. Should be defined on the pointer. func (c *ExporterOptions) Scan(src interface{}) error { return jsonScan(c, src) } @@ -103,7 +103,7 @@ type QANOptions struct { } // Value implements database/sql/driver.Valuer interface. Should be defined on the value. -func (c QANOptions) Value() (driver.Value, error) { return jsonValue(c) } +func (c *QANOptions) Value() (driver.Value, error) { return jsonValue(c) } // Scan implements database/sql.Scanner interface. Should be defined on the pointer. func (c *QANOptions) Scan(src interface{}) error { return jsonScan(c, src) } @@ -117,7 +117,7 @@ type AWSOptions struct { } // Value implements database/sql/driver.Valuer interface. Should be defined on the value. -func (c AWSOptions) Value() (driver.Value, error) { return jsonValue(c) } +func (c *AWSOptions) Value() (driver.Value, error) { return jsonValue(c) } // Scan implements database/sql.Scanner interface. Should be defined on the pointer. func (c *AWSOptions) Scan(src interface{}) error { return jsonScan(c, src) } @@ -132,7 +132,7 @@ type AzureOptions struct { } // Value implements database/sql/driver.Valuer interface. Should be defined on the value. -func (c AzureOptions) Value() (driver.Value, error) { return jsonValue(c) } +func (c *AzureOptions) Value() (driver.Value, error) { return jsonValue(c) } // Scan implements database/sql.Scanner interface. Should be defined on the pointer. func (c *AzureOptions) Scan(src interface{}) error { return jsonScan(c, src) } @@ -150,7 +150,7 @@ type MongoDBOptions struct { } // Value implements database/sql/driver.Valuer interface. Should be defined on the value. -func (c MongoDBOptions) Value() (driver.Value, error) { return jsonValue(c) } +func (c *MongoDBOptions) Value() (driver.Value, error) { return jsonValue(c) } // Scan implements database/sql.Scanner interface. Should be defined on the pointer. func (c *MongoDBOptions) Scan(src interface{}) error { return jsonScan(c, src) } @@ -172,7 +172,7 @@ type MySQLOptions struct { } // Value implements database/sql/driver.Valuer interface. Should be defined on the value. -func (c MySQLOptions) Value() (driver.Value, error) { return jsonValue(c) } +func (c *MySQLOptions) Value() (driver.Value, error) { return jsonValue(c) } // Scan implements database/sql.Scanner interface. Should be defined on the pointer. func (c *MySQLOptions) Scan(src interface{}) error { return jsonScan(c, src) } @@ -189,7 +189,7 @@ type PostgreSQLOptions struct { } // Value implements database/sql/driver.Valuer interface. Should be defined on the value. -func (c PostgreSQLOptions) Value() (driver.Value, error) { return jsonValue(c) } +func (c *PostgreSQLOptions) Value() (driver.Value, error) { return jsonValue(c) } // Scan implements database/sql.Scanner interface. Should be defined on the pointer. func (c *PostgreSQLOptions) Scan(src interface{}) error { return jsonScan(c, src) } From dd4a7fc29d8cf3447b1313e8efc2370faa3a3bfb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Tue, 3 Dec 2024 16:09:59 +0100 Subject: [PATCH 33/60] PMM-5086-12634 Migration changes. --- managed/models/database.go | 40 +++++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/managed/models/database.go b/managed/models/database.go index ce2d68abd8..ace26f32d0 100644 --- a/managed/models/database.go +++ b/managed/models/database.go @@ -1097,21 +1097,29 @@ var databaseSchema = [][]string{ }, 107: { `ALTER TABLE agents ADD COLUMN exporter_options JSONB`, + `INSERT INTO agents (exporter_options) VALUES ('{}'::jsonb)`, `ALTER TABLE agents ADD COLUMN qan_options JSONB`, + `INSERT INTO agents (qan_options) VALUES ('{}'::jsonb)`, `ALTER TABLE agents ADD COLUMN aws_options JSONB`, + `INSERT INTO agents (aws_options) VALUES ('{}'::jsonb)`, `ALTER TABLE agents ALTER COLUMN azure_options TYPE JSONB USING to_jsonb(azure_options)`, + `UPDATE agents SET azure_options = '{}'::jsonb WHERE azure_options IS NULL`, `ALTER TABLE agents ALTER COLUMN mysql_options TYPE JSONB USING to_jsonb(mysql_options)`, + `UPDATE agents SET mysql_options = '{}'::jsonb WHERE mysql_options IS NULL`, `ALTER TABLE agents RENAME COLUMN mongo_db_tls_options TO mongo_options`, + `UPDATE agents SET mongo_options = '{}'::jsonb WHERE mongo_options IS NULL`, + + `UPDATE agents SET postgresql_options = '{}'::jsonb WHERE postgresql_options IS NULL`, // migrate values into new fields in options and drop original columns - `UPDATE agents SET exporter_options = jsonb_set(COALESCE(exporter_options, '{}'::jsonb), '{expose_exporter}', to_jsonb(expose_exporter));`, - `UPDATE agents SET exporter_options = jsonb_set(COALESCE(exporter_options, '{}'::jsonb), '{push_metrics}', to_jsonb(push_metrics));`, - `UPDATE agents SET exporter_options = jsonb_set(COALESCE(exporter_options, '{}'::jsonb), '{disabled_collectors}', to_jsonb(disabled_collectors));`, - `UPDATE agents SET exporter_options = jsonb_set(COALESCE(exporter_options, '{}'::jsonb), '{metrics_resolutions}', to_jsonb(metrics_resolutions));`, - `UPDATE agents SET exporter_options = jsonb_set(COALESCE(exporter_options, '{}'::jsonb), '{metrics_path}', to_jsonb(metrics_path));`, - `UPDATE agents SET exporter_options = jsonb_set(COALESCE(exporter_options, '{}'::jsonb), '{metrics_scheme}', to_jsonb(metrics_scheme));`, + `UPDATE agents SET exporter_options = jsonb_set(exporter_options, '{expose_exporter}', to_jsonb(expose_exporter));`, + `UPDATE agents SET exporter_options = jsonb_set(exporter_options, '{push_metrics}', to_jsonb(push_metrics));`, + `UPDATE agents SET exporter_options = jsonb_set(exporter_options, '{disabled_collectors}', to_jsonb(disabled_collectors));`, + `UPDATE agents SET exporter_options = jsonb_set(exporter_options, '{metrics_resolutions}', to_jsonb(metrics_resolutions));`, + `UPDATE agents SET exporter_options = jsonb_set(exporter_options, '{metrics_path}', to_jsonb(metrics_path));`, + `UPDATE agents SET exporter_options = jsonb_set(exporter_options, '{metrics_scheme}', to_jsonb(metrics_scheme));`, `ALTER TABLE agents DROP COLUMN expose_exporter`, `ALTER TABLE agents DROP COLUMN push_metrics`, `ALTER TABLE agents DROP COLUMN disabled_collectors`, @@ -1119,26 +1127,26 @@ var databaseSchema = [][]string{ `ALTER TABLE agents DROP COLUMN metrics_path`, `ALTER TABLE agents DROP COLUMN metrics_scheme`, - `UPDATE agents SET qan_options = jsonb_set(COALESCE(qan_options, '{}'::jsonb), '{max_query_length}', to_jsonb(max_query_length));`, - `UPDATE agents SET qan_options = jsonb_set(COALESCE(qan_options, '{}'::jsonb), '{max_query_log_size}', to_jsonb(max_query_log_size));`, - `UPDATE agents SET qan_options = jsonb_set(COALESCE(qan_options, '{}'::jsonb), '{query_examples_disabled}', to_jsonb(query_examples_disabled));`, - `UPDATE agents SET qan_options = jsonb_set(COALESCE(qan_options, '{}'::jsonb), '{comments_parsing_disabled}', to_jsonb(comments_parsing_disabled));`, + `UPDATE agents SET qan_options = jsonb_set(qan_options, '{max_query_length}', to_jsonb(max_query_length));`, + `UPDATE agents SET qan_options = jsonb_set(qan_options, '{max_query_log_size}', to_jsonb(max_query_log_size));`, + `UPDATE agents SET qan_options = jsonb_set(qan_options, '{query_examples_disabled}', to_jsonb(query_examples_disabled));`, + `UPDATE agents SET qan_options = jsonb_set(qan_options, '{comments_parsing_disabled}', to_jsonb(comments_parsing_disabled));`, `ALTER TABLE agents DROP COLUMN max_query_length`, `ALTER TABLE agents DROP COLUMN max_query_log_size`, `ALTER TABLE agents DROP COLUMN query_examples_disabled`, `ALTER TABLE agents DROP COLUMN comments_parsing_disabled`, - `UPDATE agents SET aws_options = jsonb_set(COALESCE(aws_options, '{}'::jsonb), '{aws_access_key}', to_jsonb(aws_access_key));`, - `UPDATE agents SET aws_options = jsonb_set(COALESCE(aws_options, '{}'::jsonb), '{aws_secret_key}', to_jsonb(aws_secret_key));`, - `UPDATE agents SET aws_options = jsonb_set(COALESCE(aws_options, '{}'::jsonb), '{rds_basic_metrics_disabled}', to_jsonb(rds_basic_metrics_disabled));`, - `UPDATE agents SET aws_options = jsonb_set(COALESCE(aws_options, '{}'::jsonb), '{rds_enhanced_metrics_disabled}', to_jsonb(rds_enhanced_metrics_disabled));`, + `UPDATE agents SET aws_options = jsonb_set(aws_options, '{aws_access_key}', to_jsonb(aws_access_key));`, + `UPDATE agents SET aws_options = jsonb_set(aws_options, '{aws_secret_key}', to_jsonb(aws_secret_key));`, + `UPDATE agents SET aws_options = jsonb_set(aws_options, '{rds_basic_metrics_disabled}', to_jsonb(rds_basic_metrics_disabled));`, + `UPDATE agents SET aws_options = jsonb_set(aws_options, '{rds_enhanced_metrics_disabled}', to_jsonb(rds_enhanced_metrics_disabled));`, `ALTER TABLE agents DROP COLUMN aws_access_key`, `ALTER TABLE agents DROP COLUMN aws_secret_key`, `ALTER TABLE agents DROP COLUMN rds_basic_metrics_disabled`, `ALTER TABLE agents DROP COLUMN rds_enhanced_metrics_disabled`, - `UPDATE agents SET mysql_options = jsonb_set(COALESCE(mysql_options, '{}'::jsonb), '{table_count}', to_jsonb(table_count));`, - `UPDATE agents SET mysql_options = jsonb_set(COALESCE(mysql_options, '{}'::jsonb), '{table_count_tablestats_group_limit}', to_jsonb(table_count_tablestats_group_limit));`, + `UPDATE agents SET mysql_options = jsonb_set(mysql_options, '{table_count}', to_jsonb(table_count));`, + `UPDATE agents SET mysql_options = jsonb_set(mysql_options, '{table_count_tablestats_group_limit}', to_jsonb(table_count_tablestats_group_limit));`, `ALTER TABLE agents DROP COLUMN table_count`, `ALTER TABLE agents DROP COLUMN table_count_tablestats_group_limit`, From 2897eeacb5cde627b84d4359b77a8e8b01a8735f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Tue, 3 Dec 2024 16:12:32 +0100 Subject: [PATCH 34/60] PMM-5086-12634 Correct push_metrics fields naming. --- managed/models/agent_helpers.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/managed/models/agent_helpers.go b/managed/models/agent_helpers.go index 1a5cd4f6ab..d8e75bcc79 100644 --- a/managed/models/agent_helpers.go +++ b/managed/models/agent_helpers.go @@ -32,9 +32,8 @@ import ( ) const ( - // TODO figure out why it is included as PushMetrics not push_metrics - pushMetricsTrue = "((exporter_options ? 'PushMetrics') AND (exporter_options->>'PushMetrics')::boolean = true)" - pushMetricsFalse = "(NOT (exporter_options ? 'PushMetrics') OR (exporter_options->>'PushMetrics')::boolean = false)" + pushMetricsTrue = "((exporter_options ? 'push_metrics') AND (exporter_options->>'push_metrics')::boolean = true)" + pushMetricsFalse = "(NOT (exporter_options ? 'push_metrics') OR (exporter_options->>'push_metrics')::boolean = false)" ) // MySQLOptionsParams contains methods to create MySQLOptions object. From d011239caf6ff9e65e6dd4c3595cdd54bf2d111b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Tue, 3 Dec 2024 16:17:50 +0100 Subject: [PATCH 35/60] PMM-5086-12634 Lint. --- managed/models/database.go | 2 +- managed/services/management/agent.go | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/managed/models/database.go b/managed/models/database.go index ace26f32d0..e7d698f1b3 100644 --- a/managed/models/database.go +++ b/managed/models/database.go @@ -60,7 +60,7 @@ const ( VerifyFullSSLMode string = "verify-full" ) -// DefaultAgentEncryptionColumns since 3.0.0 contains all tables and it's columns to be encrypted in PMM Server DB. +// DefaultAgentEncryptionColumnsV3 since 3.0.0 contains all tables and it's columns to be encrypted in PMM Server DB. var DefaultAgentEncryptionColumnsV3 = []encryption.Table{ { Name: "agents", diff --git a/managed/services/management/agent.go b/managed/services/management/agent.go index 1528922b16..f35945031f 100644 --- a/managed/services/management/agent.go +++ b/managed/services/management/agent.go @@ -152,7 +152,6 @@ func (s *ManagementService) agentToAPI(agent *models.Agent) (*managementv1.Unive ua.MetricsScheme = pointer.GetString(agent.ExporterOptions.MetricsScheme) ua.PushMetrics = agent.ExporterOptions.PushMetrics ua.ExposeExporter = agent.ExporterOptions.ExposeExporter - } if agent.QANOptions != nil { From 7c0fb2a847b14078a5e3173daa0d190ed1ceca43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Tue, 3 Dec 2024 20:40:58 +0100 Subject: [PATCH 36/60] PMM-5086-12634 Fix migrations, helpers test. --- managed/models/agent_helpers_test.go | 272 ++++++++++++++++----------- managed/models/database.go | 6 +- 2 files changed, 168 insertions(+), 110 deletions(-) diff --git a/managed/models/agent_helpers_test.go b/managed/models/agent_helpers_test.go index 2deebfafcd..bff0c90775 100644 --- a/managed/models/agent_helpers_test.go +++ b/managed/models/agent_helpers_test.go @@ -103,10 +103,10 @@ func TestAgentHelpers(t *testing.T) { PMMAgentID: pointer.ToString("A4"), RunsOnNodeID: nil, NodeID: pointer.ToString("N2"), + ListenPort: pointer.ToUint16(8200), ExporterOptions: &models.ExporterOptions{ PushMetrics: true, }, - ListenPort: pointer.ToUint16(8200), }, &models.Agent{ AgentID: "A6", @@ -114,10 +114,7 @@ func TestAgentHelpers(t *testing.T) { PMMAgentID: pointer.ToString("A4"), RunsOnNodeID: nil, NodeID: pointer.ToString("N2"), - ExporterOptions: &models.ExporterOptions{ - PushMetrics: false, - }, - ListenPort: pointer.ToUint16(8200), + ListenPort: pointer.ToUint16(8200), }, &models.Agent{ AgentID: "A7", @@ -129,7 +126,6 @@ func TestAgentHelpers(t *testing.T) { TLS: true, TLSSkipVerify: true, ExporterOptions: &models.ExporterOptions{ - PushMetrics: false, MetricsResolutions: &models.MetricsResolutions{ HR: 1 * time.Minute, MR: 5 * time.Minute, @@ -151,9 +147,6 @@ func TestAgentHelpers(t *testing.T) { ListenPort: pointer.ToUint16(8200), TLS: true, TLSSkipVerify: true, - ExporterOptions: &models.ExporterOptions{ - PushMetrics: false, - }, MongoDBOptions: &models.MongoDBOptions{ TLSCertificateKey: "tls_certificate_key", TLSCertificateKeyFilePassword: "tls_certificate_key_file_password", @@ -165,15 +158,14 @@ func TestAgentHelpers(t *testing.T) { }, }, &models.Agent{ - AgentID: "A9", - AgentType: models.MongoDBExporterType, - PMMAgentID: pointer.ToString("A9"), - RunsOnNodeID: nil, - NodeID: pointer.ToString("N1"), - ListenPort: pointer.ToUint16(8200), - TLS: true, - TLSSkipVerify: true, - ExporterOptions: &models.ExporterOptions{}, + AgentID: "A9", + AgentType: models.MongoDBExporterType, + PMMAgentID: pointer.ToString("A9"), + RunsOnNodeID: nil, + NodeID: pointer.ToString("N1"), + ListenPort: pointer.ToUint16(8200), + TLS: true, + TLSSkipVerify: true, MongoDBOptions: &models.MongoDBOptions{ TLSCertificateKey: "tls_certificate_key", TLSCertificateKeyFilePassword: "tls_certificate_key_file_password", @@ -186,16 +178,15 @@ func TestAgentHelpers(t *testing.T) { }, }, &models.Agent{ - AgentID: "A10", - AgentType: models.MongoDBExporterType, - PMMAgentID: pointer.ToString("A10"), - RunsOnNodeID: nil, - NodeID: pointer.ToString("N1"), - ListenPort: pointer.ToUint16(8200), - TLS: true, - TLSSkipVerify: true, - ExporterOptions: &models.ExporterOptions{}, - MongoDBOptions: nil, // this test is specific for nil MongoDBOptions + AgentID: "A10", + AgentType: models.MongoDBExporterType, + PMMAgentID: pointer.ToString("A10"), + RunsOnNodeID: nil, + NodeID: pointer.ToString("N1"), + ListenPort: pointer.ToUint16(8200), + TLS: true, + TLSSkipVerify: true, + MongoDBOptions: nil, }, } { require.NoError(t, q.Insert(str)) @@ -216,29 +207,41 @@ func TestAgentHelpers(t *testing.T) { require.NoError(t, err) expected := []*models.Agent{ { - CreatedAt: now, - UpdatedAt: now, - Status: models.AgentStatusUnknown, - AgentID: "A10", - AgentType: models.MongoDBExporterType, - PMMAgentID: pointer.ToString("A10"), - RunsOnNodeID: nil, - NodeID: pointer.ToString("N1"), - ListenPort: pointer.ToUint16(8200), - TLS: true, - TLSSkipVerify: true, - ExporterOptions: &models.ExporterOptions{}, - MongoDBOptions: nil, // this test is specific for nil MongoDBOptions + CreatedAt: now, + UpdatedAt: now, + Status: models.AgentStatusUnknown, + AgentID: "A10", + AgentType: models.MongoDBExporterType, + PMMAgentID: pointer.ToString("A10"), + RunsOnNodeID: nil, + NodeID: pointer.ToString("N1"), + ListenPort: pointer.ToUint16(8200), + TLS: true, + TLSSkipVerify: true, + ExporterOptions: &models.ExporterOptions{}, + QANOptions: &models.QANOptions{}, + AWSOptions: &models.AWSOptions{}, + AzureOptions: &models.AzureOptions{}, + MongoDBOptions: &models.MongoDBOptions{}, + MySQLOptions: &models.MySQLOptions{}, + PostgreSQLOptions: &models.PostgreSQLOptions{}, }, { - AgentID: "A3", - AgentType: models.NodeExporterType, - PMMAgentID: pointer.ToStringOrNil("A1"), - RunsOnNodeID: nil, - CreatedAt: now, - UpdatedAt: now, - NodeID: pointer.ToString("N1"), - Status: models.AgentStatusUnknown, + AgentID: "A3", + AgentType: models.NodeExporterType, + PMMAgentID: pointer.ToStringOrNil("A1"), + RunsOnNodeID: nil, + CreatedAt: now, + UpdatedAt: now, + NodeID: pointer.ToString("N1"), + Status: models.AgentStatusUnknown, + ExporterOptions: &models.ExporterOptions{}, + QANOptions: &models.QANOptions{}, + AWSOptions: &models.AWSOptions{}, + AzureOptions: &models.AzureOptions{}, + MongoDBOptions: &models.MongoDBOptions{}, + MySQLOptions: &models.MySQLOptions{}, + PostgreSQLOptions: &models.PostgreSQLOptions{}, }, { AgentID: "A7", @@ -258,6 +261,11 @@ func TestAgentHelpers(t *testing.T) { LR: 15 * time.Minute, }, }, + QANOptions: &models.QANOptions{}, + AWSOptions: &models.AWSOptions{}, + AzureOptions: &models.AzureOptions{}, + MongoDBOptions: &models.MongoDBOptions{}, + MySQLOptions: &models.MySQLOptions{}, PostgreSQLOptions: &models.PostgreSQLOptions{ SSLCa: "ssl_ca", SSLCert: "ssl_cert", @@ -265,19 +273,20 @@ func TestAgentHelpers(t *testing.T) { }, }, { - AgentID: "A8", - AgentType: "mongodb_exporter", - NodeID: pointer.ToStringOrNil("N1"), - PMMAgentID: pointer.ToStringOrNil("A8"), - CreatedAt: now, - UpdatedAt: now, - Status: models.AgentStatusUnknown, - ListenPort: pointer.ToUint16OrNil(8200), - TLS: true, - TLSSkipVerify: true, - ExporterOptions: &models.ExporterOptions{ - ExposeExporter: false, - }, + AgentID: "A8", + AgentType: "mongodb_exporter", + NodeID: pointer.ToStringOrNil("N1"), + PMMAgentID: pointer.ToStringOrNil("A8"), + CreatedAt: now, + UpdatedAt: now, + Status: models.AgentStatusUnknown, + ListenPort: pointer.ToUint16OrNil(8200), + TLS: true, + TLSSkipVerify: true, + ExporterOptions: &models.ExporterOptions{}, + QANOptions: &models.QANOptions{}, + AWSOptions: &models.AWSOptions{}, + AzureOptions: &models.AzureOptions{}, MongoDBOptions: &models.MongoDBOptions{ TLSCertificateKey: "tls_certificate_key", TLSCertificateKeyFilePassword: "tls_certificate_key_file_password", @@ -287,6 +296,8 @@ func TestAgentHelpers(t *testing.T) { StatsCollections: nil, CollectionsLimit: 0, // no limit }, + MySQLOptions: &models.MySQLOptions{}, + PostgreSQLOptions: &models.PostgreSQLOptions{}, }, { AgentID: "A9", @@ -300,6 +311,9 @@ func TestAgentHelpers(t *testing.T) { TLS: true, TLSSkipVerify: true, ExporterOptions: &models.ExporterOptions{}, + QANOptions: &models.QANOptions{}, + AWSOptions: &models.AWSOptions{}, + AzureOptions: &models.AzureOptions{}, MongoDBOptions: &models.MongoDBOptions{ TLSCertificateKey: "tls_certificate_key", TLSCertificateKeyFilePassword: "tls_certificate_key_file_password", @@ -310,6 +324,8 @@ func TestAgentHelpers(t *testing.T) { CollectionsLimit: 79014, EnableAllCollectors: true, }, + MySQLOptions: &models.MySQLOptions{}, + PostgreSQLOptions: &models.PostgreSQLOptions{}, }, } assert.Equal(t, expected, agents) @@ -322,23 +338,37 @@ func TestAgentHelpers(t *testing.T) { agents, err := models.FindAgents(q, models.AgentFilters{PMMAgentID: "A1"}) require.NoError(t, err) expected := []*models.Agent{{ - AgentID: "A2", - AgentType: models.MySQLdExporterType, - PMMAgentID: pointer.ToStringOrNil("A1"), - ServiceID: pointer.ToString("S1"), - RunsOnNodeID: nil, - CreatedAt: now, - UpdatedAt: now, - Status: models.AgentStatusUnknown, + AgentID: "A2", + AgentType: models.MySQLdExporterType, + PMMAgentID: pointer.ToStringOrNil("A1"), + ServiceID: pointer.ToString("S1"), + RunsOnNodeID: nil, + CreatedAt: now, + UpdatedAt: now, + Status: models.AgentStatusUnknown, + ExporterOptions: &models.ExporterOptions{}, + QANOptions: &models.QANOptions{}, + AWSOptions: &models.AWSOptions{}, + AzureOptions: &models.AzureOptions{}, + MongoDBOptions: &models.MongoDBOptions{}, + MySQLOptions: &models.MySQLOptions{}, + PostgreSQLOptions: &models.PostgreSQLOptions{}, }, { - AgentID: "A3", - AgentType: models.NodeExporterType, - PMMAgentID: pointer.ToStringOrNil("A1"), - NodeID: pointer.ToString("N1"), - RunsOnNodeID: nil, - CreatedAt: now, - UpdatedAt: now, - Status: models.AgentStatusUnknown, + AgentID: "A3", + AgentType: models.NodeExporterType, + PMMAgentID: pointer.ToStringOrNil("A1"), + NodeID: pointer.ToString("N1"), + RunsOnNodeID: nil, + CreatedAt: now, + UpdatedAt: now, + Status: models.AgentStatusUnknown, + ExporterOptions: &models.ExporterOptions{}, + QANOptions: &models.QANOptions{}, + AWSOptions: &models.AWSOptions{}, + AzureOptions: &models.AzureOptions{}, + MongoDBOptions: &models.MongoDBOptions{}, + MySQLOptions: &models.MySQLOptions{}, + PostgreSQLOptions: &models.PostgreSQLOptions{}, }} assert.Equal(t, expected, agents) }) @@ -350,14 +380,21 @@ func TestAgentHelpers(t *testing.T) { agents, err := models.FindAgents(q, models.AgentFilters{PMMAgentID: "A1", AgentType: pointerToAgentType(models.MySQLdExporterType)}) require.NoError(t, err) expected := []*models.Agent{{ - AgentID: "A2", - AgentType: models.MySQLdExporterType, - PMMAgentID: pointer.ToStringOrNil("A1"), - ServiceID: pointer.ToString("S1"), - RunsOnNodeID: nil, - CreatedAt: now, - UpdatedAt: now, - Status: models.AgentStatusUnknown, + AgentID: "A2", + AgentType: models.MySQLdExporterType, + PMMAgentID: pointer.ToStringOrNil("A1"), + ServiceID: pointer.ToString("S1"), + RunsOnNodeID: nil, + CreatedAt: now, + UpdatedAt: now, + Status: models.AgentStatusUnknown, + ExporterOptions: &models.ExporterOptions{}, + QANOptions: &models.QANOptions{}, + AWSOptions: &models.AWSOptions{}, + AzureOptions: &models.AzureOptions{}, + MongoDBOptions: &models.MongoDBOptions{}, + MySQLOptions: &models.MySQLOptions{}, + PostgreSQLOptions: &models.PostgreSQLOptions{}, }} assert.Equal(t, expected, agents) }) @@ -369,28 +406,42 @@ func TestAgentHelpers(t *testing.T) { agents, err := models.FindAgents(q, models.AgentFilters{ServiceID: "S1"}) require.NoError(t, err) expected := []*models.Agent{{ - AgentID: "A2", - AgentType: models.MySQLdExporterType, - PMMAgentID: pointer.ToStringOrNil("A1"), - ServiceID: pointer.ToString("S1"), - RunsOnNodeID: nil, - CreatedAt: now, - UpdatedAt: now, - Status: models.AgentStatusUnknown, + AgentID: "A2", + AgentType: models.MySQLdExporterType, + PMMAgentID: pointer.ToStringOrNil("A1"), + ServiceID: pointer.ToString("S1"), + RunsOnNodeID: nil, + CreatedAt: now, + UpdatedAt: now, + Status: models.AgentStatusUnknown, + ExporterOptions: &models.ExporterOptions{}, + QANOptions: &models.QANOptions{}, + AWSOptions: &models.AWSOptions{}, + AzureOptions: &models.AzureOptions{}, + MongoDBOptions: &models.MongoDBOptions{}, + MySQLOptions: &models.MySQLOptions{}, + PostgreSQLOptions: &models.PostgreSQLOptions{}, }} assert.Equal(t, expected, agents) agents, err = models.FindAgents(q, models.AgentFilters{ServiceID: "S1", AgentType: pointerToAgentType(models.MySQLdExporterType)}) require.NoError(t, err) expected = []*models.Agent{{ - AgentID: "A2", - AgentType: models.MySQLdExporterType, - PMMAgentID: pointer.ToStringOrNil("A1"), - ServiceID: pointer.ToString("S1"), - RunsOnNodeID: nil, - CreatedAt: now, - UpdatedAt: now, - Status: models.AgentStatusUnknown, + AgentID: "A2", + AgentType: models.MySQLdExporterType, + PMMAgentID: pointer.ToStringOrNil("A1"), + ServiceID: pointer.ToString("S1"), + RunsOnNodeID: nil, + CreatedAt: now, + UpdatedAt: now, + Status: models.AgentStatusUnknown, + ExporterOptions: &models.ExporterOptions{}, + QANOptions: &models.QANOptions{}, + AWSOptions: &models.AWSOptions{}, + AzureOptions: &models.AzureOptions{}, + MongoDBOptions: &models.MongoDBOptions{}, + MySQLOptions: &models.MySQLOptions{}, + PostgreSQLOptions: &models.PostgreSQLOptions{}, }} assert.Equal(t, expected, agents) @@ -420,11 +471,18 @@ func TestAgentHelpers(t *testing.T) { tests.AssertGRPCError(t, status.New(codes.FailedPrecondition, `pmm-agent with ID A1 has agents.`), err) expected := &models.Agent{ - AgentID: "A1", - AgentType: models.PMMAgentType, - RunsOnNodeID: pointer.ToString("N1"), - CreatedAt: now, - UpdatedAt: now, + AgentID: "A1", + AgentType: models.PMMAgentType, + RunsOnNodeID: pointer.ToString("N1"), + CreatedAt: now, + UpdatedAt: now, + ExporterOptions: &models.ExporterOptions{}, + QANOptions: &models.QANOptions{}, + AWSOptions: &models.AWSOptions{}, + AzureOptions: &models.AzureOptions{}, + MongoDBOptions: &models.MongoDBOptions{}, + MySQLOptions: &models.MySQLOptions{}, + PostgreSQLOptions: &models.PostgreSQLOptions{}, } agent, err = models.RemoveAgent(q, "A1", models.RemoveCascade) assert.Equal(t, expected, agent) diff --git a/managed/models/database.go b/managed/models/database.go index e7d698f1b3..977924ced5 100644 --- a/managed/models/database.go +++ b/managed/models/database.go @@ -1097,11 +1097,11 @@ var databaseSchema = [][]string{ }, 107: { `ALTER TABLE agents ADD COLUMN exporter_options JSONB`, - `INSERT INTO agents (exporter_options) VALUES ('{}'::jsonb)`, + `UPDATE agents SET exporter_options = '{}'::jsonb`, `ALTER TABLE agents ADD COLUMN qan_options JSONB`, - `INSERT INTO agents (qan_options) VALUES ('{}'::jsonb)`, + `UPDATE agents SET qan_options = '{}'::jsonb`, `ALTER TABLE agents ADD COLUMN aws_options JSONB`, - `INSERT INTO agents (aws_options) VALUES ('{}'::jsonb)`, + `UPDATE agents SET aws_options = '{}'::jsonb`, `ALTER TABLE agents ALTER COLUMN azure_options TYPE JSONB USING to_jsonb(azure_options)`, `UPDATE agents SET azure_options = '{}'::jsonb WHERE azure_options IS NULL`, From 06bb86f7c764c9a95ec4ec43f60d5343b7c32108 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Tue, 3 Dec 2024 20:48:24 +0100 Subject: [PATCH 37/60] PMM-5086-12634 Fix listen address test. --- managed/services/agents/agents.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/managed/services/agents/agents.go b/managed/services/agents/agents.go index bd06495740..90c898be57 100644 --- a/managed/services/agents/agents.go +++ b/managed/services/agents/agents.go @@ -171,8 +171,12 @@ func ensureAuthParams(exporter *models.Agent, params *agentv1.SetStateRequest_Ag // getExporterListenAddress returns the appropriate listen address to use for a given exporter. func getExporterListenAddress(_ *models.Node, exporter *models.Agent) string { - if exporter.ExporterOptions != nil && exporter.ExporterOptions.PushMetrics { + switch { + case exporter.ExporterOptions.ExposeExporter: + return "0.0.0.0" + case exporter.ExporterOptions.PushMetrics: return "127.0.0.1" + } return "0.0.0.0" From 517eeb0a3815016a3f30efbf352901295495c820 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Tue, 3 Dec 2024 21:06:36 +0100 Subject: [PATCH 38/60] PMM-5086-12634 Fix agent management tests. --- managed/services/management/agent_test.go | 154 +++++++++++++++++++--- 1 file changed, 134 insertions(+), 20 deletions(-) diff --git a/managed/services/management/agent_test.go b/managed/services/management/agent_test.go index e6d0461f73..40705bc901 100644 --- a/managed/services/management/agent_test.go +++ b/managed/services/management/agent_test.go @@ -148,36 +148,78 @@ func TestAgentService(t *testing.T) { expected := []*agentv1.UniversalAgent{ { - AgentId: pgExporterID, - AgentType: "postgres_exporter", - PmmAgentId: models.PMMServerAgentID, - IsConnected: false, - CreatedAt: timestamppb.New(now), - UpdatedAt: timestamppb.New(now), - Username: "postgres", - PostgresqlOptions: &agentv1.UniversalAgent_PostgreSQLOptions{ - IsSslKeySet: false, - }, + AgentId: pgExporterID, + AgentType: "postgres_exporter", + PmmAgentId: models.PMMServerAgentID, + IsConnected: false, + CreatedAt: timestamppb.New(now), + UpdatedAt: timestamppb.New(now), + Username: "postgres", ServiceId: "00000000-0000-4000-8000-000000000002", Status: "AGENT_STATUS_UNKNOWN", Tls: true, CommentsParsingDisabled: true, - }, - { - AgentId: pgStatStatementID, - AgentType: "qan-postgresql-pgstatements-agent", - PmmAgentId: models.PMMServerAgentID, - IsConnected: false, - CreatedAt: timestamppb.New(now), - UpdatedAt: timestamppb.New(now), - Username: "postgres", + AzureOptions: &agentv1.UniversalAgent_AzureOptions{ + ClientId: "", + IsClientSecretSet: false, + ResourceGroup: "", + SubscriptionId: "", + TenantId: "", + }, + MongoDbOptions: &agentv1.UniversalAgent_MongoDBOptions{ + IsTlsCertificateKeySet: false, + IsTlsCertificateKeyFilePasswordSet: false, + AuthenticationMechanism: "", + AuthenticationDatabase: "", + StatsCollections: nil, + CollectionsLimit: 0, + EnableAllCollectors: false, + }, + MysqlOptions: &agentv1.UniversalAgent_MySQLOptions{ + IsTlsKeySet: false, + }, PostgresqlOptions: &agentv1.UniversalAgent_PostgreSQLOptions{ - IsSslKeySet: false, + IsSslKeySet: false, + AutoDiscoveryLimit: 0, + MaxExporterConnections: 0, }, + }, + { + AgentId: pgStatStatementID, + AgentType: "qan-postgresql-pgstatements-agent", + PmmAgentId: models.PMMServerAgentID, + IsConnected: false, + CreatedAt: timestamppb.New(now), + UpdatedAt: timestamppb.New(now), + Username: "postgres", ServiceId: "00000000-0000-4000-8000-000000000002", Status: "AGENT_STATUS_UNKNOWN", Tls: true, CommentsParsingDisabled: true, + AzureOptions: &agentv1.UniversalAgent_AzureOptions{ + ClientId: "", + IsClientSecretSet: false, + ResourceGroup: "", + SubscriptionId: "", + TenantId: "", + }, + MongoDbOptions: &agentv1.UniversalAgent_MongoDBOptions{ + IsTlsCertificateKeySet: false, + IsTlsCertificateKeyFilePasswordSet: false, + AuthenticationMechanism: "", + AuthenticationDatabase: "", + StatsCollections: nil, + CollectionsLimit: 0, + EnableAllCollectors: false, + }, + MysqlOptions: &agentv1.UniversalAgent_MySQLOptions{ + IsTlsKeySet: false, + }, + PostgresqlOptions: &agentv1.UniversalAgent_PostgreSQLOptions{ + IsSslKeySet: false, + AutoDiscoveryLimit: 0, + MaxExporterConnections: 0, + }, }, { AgentId: models.PMMServerAgentID, @@ -186,6 +228,30 @@ func TestAgentService(t *testing.T) { IsConnected: true, CreatedAt: timestamppb.New(now), UpdatedAt: timestamppb.New(now), + AzureOptions: &agentv1.UniversalAgent_AzureOptions{ + ClientId: "", + IsClientSecretSet: false, + ResourceGroup: "", + SubscriptionId: "", + TenantId: "", + }, + MongoDbOptions: &agentv1.UniversalAgent_MongoDBOptions{ + IsTlsCertificateKeySet: false, + IsTlsCertificateKeyFilePasswordSet: false, + AuthenticationMechanism: "", + AuthenticationDatabase: "", + StatsCollections: nil, + CollectionsLimit: 0, + EnableAllCollectors: false, + }, + MysqlOptions: &agentv1.UniversalAgent_MySQLOptions{ + IsTlsKeySet: false, + }, + PostgresqlOptions: &agentv1.UniversalAgent_PostgreSQLOptions{ + IsSslKeySet: false, + AutoDiscoveryLimit: 0, + MaxExporterConnections: 0, + }, }, } @@ -237,6 +303,30 @@ func TestAgentService(t *testing.T) { UpdatedAt: timestamppb.New(now), ServiceId: "00000000-0000-4000-8000-000000000006", Status: "AGENT_STATUS_UNKNOWN", + AzureOptions: &agentv1.UniversalAgent_AzureOptions{ + ClientId: "", + IsClientSecretSet: false, + ResourceGroup: "", + SubscriptionId: "", + TenantId: "", + }, + MongoDbOptions: &agentv1.UniversalAgent_MongoDBOptions{ + IsTlsCertificateKeySet: false, + IsTlsCertificateKeyFilePasswordSet: false, + AuthenticationMechanism: "", + AuthenticationDatabase: "", + StatsCollections: nil, + CollectionsLimit: 0, + EnableAllCollectors: false, + }, + MysqlOptions: &agentv1.UniversalAgent_MySQLOptions{ + IsTlsKeySet: false, + }, + PostgresqlOptions: &agentv1.UniversalAgent_PostgreSQLOptions{ + IsSslKeySet: false, + AutoDiscoveryLimit: 0, + MaxExporterConnections: 0, + }, }, } assert.Equal(t, expected, response.Agents) @@ -287,6 +377,30 @@ func TestAgentService(t *testing.T) { UpdatedAt: timestamppb.New(now), ServiceId: "00000000-0000-4000-8000-000000000006", Status: "AGENT_STATUS_UNKNOWN", + AzureOptions: &agentv1.UniversalAgent_AzureOptions{ + ClientId: "", + IsClientSecretSet: false, + ResourceGroup: "", + SubscriptionId: "", + TenantId: "", + }, + MongoDbOptions: &agentv1.UniversalAgent_MongoDBOptions{ + IsTlsCertificateKeySet: false, + IsTlsCertificateKeyFilePasswordSet: false, + AuthenticationMechanism: "", + AuthenticationDatabase: "", + StatsCollections: nil, + CollectionsLimit: 0, + EnableAllCollectors: false, + }, + MysqlOptions: &agentv1.UniversalAgent_MySQLOptions{ + IsTlsKeySet: false, + }, + PostgresqlOptions: &agentv1.UniversalAgent_PostgreSQLOptions{ + IsSslKeySet: false, + AutoDiscoveryLimit: 0, + MaxExporterConnections: 0, + }, }, } assert.Equal(t, expected, response.Agents) From f945eb203c16e0f825f22f83afa5b6f6f085a825 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Tue, 3 Dec 2024 21:37:53 +0100 Subject: [PATCH 39/60] PMM-5086-12634 Mongo nil handling. --- managed/services/agents/mongodb.go | 30 ++++++------------------- managed/services/agents/mongodb_test.go | 25 ++++++++++++--------- 2 files changed, 22 insertions(+), 33 deletions(-) diff --git a/managed/services/agents/mongodb.go b/managed/services/agents/mongodb.go index 51da37fb98..c3a8bb6ebe 100644 --- a/managed/services/agents/mongodb.go +++ b/managed/services/agents/mongodb.go @@ -49,10 +49,6 @@ func mongodbExporterConfig(node *models.Node, service *models.Service, exporter args := getArgs(exporter, tdp, listenAddress, pmmAgentVersion) - if exporter.ExporterOptions == nil { - exporter.ExporterOptions = &models.ExporterOptions{} - } - if pointer.GetString(exporter.ExporterOptions.MetricsPath) != "" { args = append(args, "--web.telemetry-path="+*exporter.ExporterOptions.MetricsPath) //nolint:goconst } @@ -61,10 +57,7 @@ func mongodbExporterConfig(node *models.Node, service *models.Service, exporter sort.Strings(args) - database := "" - if exporter.MongoDBOptions != nil { - database = exporter.MongoDBOptions.AuthenticationDatabase - } + database := exporter.MongoDBOptions.AuthenticationDatabase env := []string{ fmt.Sprintf("MONGODB_URI=%s", exporter.DSN(service, models.DSNParams{DialTimeout: time.Second, Database: database}, tdp, pmmAgentVersion)), } @@ -93,17 +86,13 @@ func mongodbExporterConfig(node *models.Node, service *models.Service, exporter func getArgs(exporter *models.Agent, tdp *models.DelimiterPair, listenAddress string, pmmAgentVersion *version.Parsed) []string { var args []string - if exporter.ExporterOptions == nil { - exporter.ExporterOptions = &models.ExporterOptions{} - } - switch { case !pmmAgentVersion.Less(v2_25_0): // >= 2.26.0 args = buildBaseArgs(listenAddress, tdp) args = append(args, "--discovering-mode") defaultEnabledCollectors := []string{"diagnosticdata", "replicasetstatus"} - collectAll := exporter.MongoDBOptions != nil && exporter.MongoDBOptions.EnableAllCollectors + collectAll := exporter.MongoDBOptions.EnableAllCollectors if !pmmAgentVersion.Less(v2_26_0) { defaultEnabledCollectors = []string{} @@ -128,20 +117,18 @@ func getArgs(exporter *models.Agent, tdp *models.DelimiterPair, listenAddress st args = collectors.FilterOutCollectors("--collector.", args, exporter.ExporterOptions.DisabledCollectors) args = append(args, collectors.DisableDefaultEnabledCollectors("--no-collector.", defaultEnabledCollectors, exporter.ExporterOptions.DisabledCollectors)...) - if exporter.MongoDBOptions != nil && len(exporter.MongoDBOptions.StatsCollections) != 0 { + if len(exporter.MongoDBOptions.StatsCollections) != 0 { args = append(args, "--mongodb.collstats-colls="+strings.Join(exporter.MongoDBOptions.StatsCollections, ",")) if !pmmAgentVersion.Less(v2_26_0) { args = append(args, "--mongodb.indexstats-colls="+strings.Join(exporter.MongoDBOptions.StatsCollections, ",")) } } - if exporter.MongoDBOptions != nil { - collstatsLimit := int32(200) - if exporter.MongoDBOptions.CollectionsLimit != -1 { - collstatsLimit = exporter.MongoDBOptions.CollectionsLimit - } - args = append(args, fmt.Sprintf("--collector.collstats-limit=%d", collstatsLimit)) + collstatsLimit := int32(200) + if exporter.MongoDBOptions.CollectionsLimit != -1 { + collstatsLimit = exporter.MongoDBOptions.CollectionsLimit } + args = append(args, fmt.Sprintf("--collector.collstats-limit=%d", collstatsLimit)) case !pmmAgentVersion.Less(newMongoExporterPMMVersion): // >= 2.10.0 args = buildBaseArgs(listenAddress, tdp) @@ -173,9 +160,6 @@ func buildBaseArgs(listenAddress string, tdp *models.DelimiterPair) []string { // qanMongoDBProfilerAgentConfig returns desired configuration of qan-mongodb-profiler-agent built-in agent. func qanMongoDBProfilerAgentConfig(service *models.Service, agent *models.Agent, pmmAgentVersion *version.Parsed) *agentv1.SetStateRequest_BuiltinAgent { tdp := agent.TemplateDelimiters(service) - if agent.QANOptions == nil { - agent.QANOptions = &models.QANOptions{} - } return &agentv1.SetStateRequest_BuiltinAgent{ Type: inventoryv1.AgentType_AGENT_TYPE_QAN_MONGODB_PROFILER_AGENT, diff --git a/managed/services/agents/mongodb_test.go b/managed/services/agents/mongodb_test.go index 1c1a5ead65..c9459b4b99 100644 --- a/managed/services/agents/mongodb_test.go +++ b/managed/services/agents/mongodb_test.go @@ -38,11 +38,13 @@ func TestMongodbExporterConfig225(t *testing.T) { Port: pointer.ToUint16(27017), } exporter := &models.Agent{ - AgentID: "agent-id", - AgentType: models.MongoDBExporterType, - Username: pointer.ToString("username"), - Password: pointer.ToString("s3cur3 p@$$w0r4."), - AgentPassword: pointer.ToString("agent-password"), + AgentID: "agent-id", + AgentType: models.MongoDBExporterType, + Username: pointer.ToString("username"), + Password: pointer.ToString("s3cur3 p@$$w0r4."), + AgentPassword: pointer.ToString("agent-password"), + ExporterOptions: &models.ExporterOptions{}, + MongoDBOptions: &models.MongoDBOptions{}, } actual, err := mongodbExporterConfig(node, mongodb, exporter, redactSecrets, pmmAgentVersion) @@ -51,6 +53,7 @@ func TestMongodbExporterConfig225(t *testing.T) { TemplateLeftDelim: "{{", TemplateRightDelim: "}}", Args: []string{ + "--collector.collstats-limit=0", "--compatible-mode", "--discovering-mode", "--mongodb.global-conn-pool", @@ -97,11 +100,13 @@ func TestMongodbExporterConfig226(t *testing.T) { Port: pointer.ToUint16(27017), } exporter := &models.Agent{ - AgentID: "agent-id", - AgentType: models.MongoDBExporterType, - Username: pointer.ToString("username"), - Password: pointer.ToString("s3cur3 p@$$w0r4."), - AgentPassword: pointer.ToString("agent-password"), + AgentID: "agent-id", + AgentType: models.MongoDBExporterType, + Username: pointer.ToString("username"), + Password: pointer.ToString("s3cur3 p@$$w0r4."), + AgentPassword: pointer.ToString("agent-password"), + ExporterOptions: &models.ExporterOptions{}, + MongoDBOptions: &models.MongoDBOptions{}, } actual, err := mongodbExporterConfig(node, mongodb, exporter, redactSecrets, pmmAgentVersion) expected := &agentv1.SetStateRequest_AgentProcess{ From 26d6f8b00264a59347248cde26390af25584bc63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Tue, 3 Dec 2024 21:46:41 +0100 Subject: [PATCH 40/60] PMM-5086-12634 MySQL nil handling. --- managed/models/agent_model.go | 119 ++++++++++++-------------- managed/services/agents/mysql.go | 10 --- managed/services/agents/mysql_test.go | 24 +++--- 3 files changed, 67 insertions(+), 86 deletions(-) diff --git a/managed/models/agent_model.go b/managed/models/agent_model.go index 8cfb62f231..dbf6e76dc2 100644 --- a/managed/models/agent_model.go +++ b/managed/models/agent_model.go @@ -484,22 +484,20 @@ func (s *Agent) DSN(service *Service, dsnParams DSNParams, tdp *DelimiterPair, p } } - if s.MongoDBOptions != nil { - if s.MongoDBOptions.TLSCertificateKey != "" { - q.Add("tlsCertificateKeyFile", tdp.Left+".TextFiles."+certificateKeyFilePlaceholder+tdp.Right) - } - if s.MongoDBOptions.TLSCertificateKeyFilePassword != "" { - q.Add("tlsCertificateKeyFilePassword", s.MongoDBOptions.TLSCertificateKeyFilePassword) - } - if s.MongoDBOptions.TLSCa != "" { - q.Add("tlsCaFile", tdp.Left+".TextFiles."+caFilePlaceholder+tdp.Right) - } - if s.MongoDBOptions.AuthenticationMechanism != "" { - q.Add("authMechanism", s.MongoDBOptions.AuthenticationMechanism) - } - if s.MongoDBOptions.AuthenticationDatabase != "" { - q.Add("authSource", s.MongoDBOptions.AuthenticationDatabase) - } + if s.MongoDBOptions.TLSCertificateKey != "" { + q.Add("tlsCertificateKeyFile", tdp.Left+".TextFiles."+certificateKeyFilePlaceholder+tdp.Right) + } + if s.MongoDBOptions.TLSCertificateKeyFilePassword != "" { + q.Add("tlsCertificateKeyFilePassword", s.MongoDBOptions.TLSCertificateKeyFilePassword) + } + if s.MongoDBOptions.TLSCa != "" { + q.Add("tlsCaFile", tdp.Left+".TextFiles."+caFilePlaceholder+tdp.Right) + } + if s.MongoDBOptions.AuthenticationMechanism != "" { + q.Add("authMechanism", s.MongoDBOptions.AuthenticationMechanism) + } + if s.MongoDBOptions.AuthenticationDatabase != "" { + q.Add("authSource", s.MongoDBOptions.AuthenticationDatabase) } address := socket @@ -642,10 +640,6 @@ func (s *Agent) IsMySQLTablestatsGroupEnabled() bool { panic(fmt.Errorf("unhandled AgentType %q", s.AgentType)) } - if s.MySQLOptions == nil { - s.MySQLOptions = &MySQLOptions{} - } - switch { case s.MySQLOptions.TableCountTablestatsGroupLimit == 0: // server defined return true @@ -662,58 +656,55 @@ func (s *Agent) IsMySQLTablestatsGroupEnabled() bool { func (s Agent) Files() map[string]string { switch s.AgentType { case MySQLdExporterType, QANMySQLPerfSchemaAgentType, QANMySQLSlowlogAgentType: - if s.MySQLOptions != nil { - files := make(map[string]string) - if s.MySQLOptions.TLSCa != "" { - files["tlsCa"] = s.MySQLOptions.TLSCa - } - if s.MySQLOptions.TLSCert != "" { - files["tlsCert"] = s.MySQLOptions.TLSCert - } - if s.MySQLOptions.TLSKey != "" { - files["tlsKey"] = s.MySQLOptions.TLSKey - } + files := make(map[string]string) + if s.MySQLOptions.TLSCa != "" { + files["tlsCa"] = s.MySQLOptions.TLSCa + } + if s.MySQLOptions.TLSCert != "" { + files["tlsCert"] = s.MySQLOptions.TLSCert + } + if s.MySQLOptions.TLSKey != "" { + files["tlsKey"] = s.MySQLOptions.TLSKey + } - if len(files) != 0 { - return files - } + if len(files) != 0 { + return files } + return nil case ProxySQLExporterType: return nil case QANMongoDBProfilerAgentType, MongoDBExporterType: - if s.MongoDBOptions != nil { - files := make(map[string]string) - if s.MongoDBOptions.TLSCa != "" { - files[caFilePlaceholder] = s.MongoDBOptions.TLSCa - } - if s.MongoDBOptions.TLSCertificateKey != "" { - files[certificateKeyFilePlaceholder] = s.MongoDBOptions.TLSCertificateKey - } + files := make(map[string]string) + if s.MongoDBOptions.TLSCa != "" { + files[caFilePlaceholder] = s.MongoDBOptions.TLSCa + } + if s.MongoDBOptions.TLSCertificateKey != "" { + files[certificateKeyFilePlaceholder] = s.MongoDBOptions.TLSCertificateKey + } - if len(files) != 0 { - return files - } + if len(files) != 0 { + return files } + return nil case PostgresExporterType, QANPostgreSQLPgStatementsAgentType, QANPostgreSQLPgStatMonitorAgentType: - if s.PostgreSQLOptions != nil { - files := make(map[string]string) + files := make(map[string]string) - if s.PostgreSQLOptions.SSLCa != "" { - files[caFilePlaceholder] = s.PostgreSQLOptions.SSLCa - } - if s.PostgreSQLOptions.SSLCert != "" { - files[certificateFilePlaceholder] = s.PostgreSQLOptions.SSLCert - } - if s.PostgreSQLOptions.SSLKey != "" { - files[certificateKeyFilePlaceholder] = s.PostgreSQLOptions.SSLKey - } + if s.PostgreSQLOptions.SSLCa != "" { + files[caFilePlaceholder] = s.PostgreSQLOptions.SSLCa + } + if s.PostgreSQLOptions.SSLCert != "" { + files[certificateFilePlaceholder] = s.PostgreSQLOptions.SSLCert + } + if s.PostgreSQLOptions.SSLKey != "" { + files[certificateKeyFilePlaceholder] = s.PostgreSQLOptions.SSLKey + } - if len(files) != 0 { - return files - } + if len(files) != 0 { + return files } + return nil default: panic(fmt.Errorf("unhandled AgentType %q", s.AgentType)) @@ -722,10 +713,6 @@ func (s Agent) Files() map[string]string { // TemplateDelimiters returns a pair of safe template delimiters that are not present in agent parameters. func (s Agent) TemplateDelimiters(svc *Service) *DelimiterPair { - if s.ExporterOptions == nil { - s.ExporterOptions = &ExporterOptions{} - } - templateParams := []string{ pointer.GetString(svc.Address), pointer.GetString(s.Username), @@ -735,15 +722,15 @@ func (s Agent) TemplateDelimiters(svc *Service) *DelimiterPair { switch svc.ServiceType { case MySQLServiceType: - if s.MySQLOptions != nil && s.MySQLOptions.TLSKey != "" { + if s.MySQLOptions.TLSKey != "" { templateParams = append(templateParams, s.MySQLOptions.TLSKey) } case MongoDBServiceType: - if s.MongoDBOptions != nil && s.MongoDBOptions.TLSCertificateKeyFilePassword != "" { + if s.MongoDBOptions.TLSCertificateKeyFilePassword != "" { templateParams = append(templateParams, s.MongoDBOptions.TLSCertificateKeyFilePassword) } case PostgreSQLServiceType: - if s.PostgreSQLOptions != nil && s.PostgreSQLOptions.SSLKey != "" { + if s.PostgreSQLOptions.SSLKey != "" { templateParams = append(templateParams, s.PostgreSQLOptions.SSLKey) } case ProxySQLServiceType: diff --git a/managed/services/agents/mysql.go b/managed/services/agents/mysql.go index 60e3aec6d0..55ff651b13 100644 --- a/managed/services/agents/mysql.go +++ b/managed/services/agents/mysql.go @@ -104,10 +104,6 @@ func mysqldExporterConfig( args = append(args, tablestatsGroup...) } - if exporter.ExporterOptions == nil { - exporter.ExporterOptions = &models.ExporterOptions{} - } - args = collectors.FilterOutCollectors("--collect.", args, exporter.ExporterOptions.DisabledCollectors) if exporter.ExporterOptions.MetricsPath != nil { @@ -158,9 +154,6 @@ func mysqldExporterConfig( // qanMySQLPerfSchemaAgentConfig returns desired configuration of qan-mysql-perfschema built-in agent. func qanMySQLPerfSchemaAgentConfig(service *models.Service, agent *models.Agent, pmmAgentVersion *version.Parsed) *agentv1.SetStateRequest_BuiltinAgent { tdp := agent.TemplateDelimiters(service) - if agent.QANOptions == nil { - agent.QANOptions = &models.QANOptions{} - } return &agentv1.SetStateRequest_BuiltinAgent{ Type: inventoryv1.AgentType_AGENT_TYPE_QAN_MYSQL_PERFSCHEMA_AGENT, @@ -180,9 +173,6 @@ func qanMySQLPerfSchemaAgentConfig(service *models.Service, agent *models.Agent, // qanMySQLSlowlogAgentConfig returns desired configuration of qan-mysql-slowlog built-in agent. func qanMySQLSlowlogAgentConfig(service *models.Service, agent *models.Agent, pmmAgentVersion *version.Parsed) *agentv1.SetStateRequest_BuiltinAgent { tdp := agent.TemplateDelimiters(service) - if agent.QANOptions == nil { - agent.QANOptions = &models.QANOptions{} - } return &agentv1.SetStateRequest_BuiltinAgent{ Type: inventoryv1.AgentType_AGENT_TYPE_QAN_MYSQL_SLOWLOG_AGENT, diff --git a/managed/services/agents/mysql_test.go b/managed/services/agents/mysql_test.go index 943db762aa..8bd6ca1f4e 100644 --- a/managed/services/agents/mysql_test.go +++ b/managed/services/agents/mysql_test.go @@ -37,11 +37,13 @@ func TestMySQLdExporterConfig(t *testing.T) { Address: "1.2.3.4", } exporter := &models.Agent{ - AgentID: "agent-id", - AgentType: models.MySQLdExporterType, - Username: pointer.ToString("username"), - Password: pointer.ToString("s3cur3 p@$$w0r4."), - AgentPassword: pointer.ToString("agent-password"), + AgentID: "agent-id", + AgentType: models.MySQLdExporterType, + Username: pointer.ToString("username"), + Password: pointer.ToString("s3cur3 p@$$w0r4."), + AgentPassword: pointer.ToString("agent-password"), + ExporterOptions: &models.ExporterOptions{}, + MySQLOptions: &models.MySQLOptions{}, } pmmAgentVersion := version.MustParse("2.21.0") @@ -141,11 +143,12 @@ func TestMySQLdExporterConfigTablestatsGroupDisabled(t *testing.T) { Port: pointer.ToUint16(3306), } exporter := &models.Agent{ - AgentID: "agent-id", - AgentType: models.MySQLdExporterType, - Username: pointer.ToString("username"), - Password: pointer.ToString("s3cur3 p@$$w0r4."), - TLS: true, + AgentID: "agent-id", + AgentType: models.MySQLdExporterType, + Username: pointer.ToString("username"), + Password: pointer.ToString("s3cur3 p@$$w0r4."), + TLS: true, + ExporterOptions: &models.ExporterOptions{}, MySQLOptions: &models.MySQLOptions{ TableCountTablestatsGroupLimit: -1, TLSCa: "content-of-tls-ca", @@ -252,6 +255,7 @@ func TestMySQLdExporterConfigDisabledCollectors(t *testing.T) { ExporterOptions: &models.ExporterOptions{ DisabledCollectors: []string{"heartbeat", "info_schema.clientstats", "perf_schema.eventsstatements", "custom_query.hr"}, }, + MySQLOptions: &models.MySQLOptions{}, } pmmAgentVersion := version.MustParse("2.24.0") From e57bbc9b9f083685157decade6eefebe46e70d2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Tue, 3 Dec 2024 21:50:43 +0100 Subject: [PATCH 41/60] PMM-5086 Fix Node nil handling. --- managed/services/agents/node.go | 3 --- managed/services/agents/node_test.go | 25 +++++++++++++++---------- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/managed/services/agents/node.go b/managed/services/agents/node.go index db8a4d5951..9fe3bb994d 100644 --- a/managed/services/agents/node.go +++ b/managed/services/agents/node.go @@ -36,9 +36,6 @@ var ( func nodeExporterConfig(node *models.Node, exporter *models.Agent, agentVersion *version.Parsed) (*agentv1.SetStateRequest_AgentProcess, error) { listenAddress := getExporterListenAddress(node, exporter) - if exporter.ExporterOptions == nil { - exporter.ExporterOptions = &models.ExporterOptions{} - } tdp := models.TemplateDelimsPair(pointer.GetString(exporter.ExporterOptions.MetricsPath)) args := []string{ "--collector.textfile.directory.lr=" + pathsBase(agentVersion, tdp.Left, tdp.Right) + "/collectors/textfile-collector/low-resolution", diff --git a/managed/services/agents/node_test.go b/managed/services/agents/node_test.go index 3de1512e52..8b70d371d3 100644 --- a/managed/services/agents/node_test.go +++ b/managed/services/agents/node_test.go @@ -34,8 +34,9 @@ func TestAuthWebConfig(t *testing.T) { node := &models.Node{} exporter := &models.Agent{ - AgentID: "agent-id", - AgentType: models.NodeExporterType, + AgentID: "agent-id", + AgentType: models.NodeExporterType, + ExporterOptions: &models.ExporterOptions{}, } agentVersion := version.MustParse("2.26.1") @@ -58,8 +59,9 @@ func TestAuthWebConfig(t *testing.T) { node := &models.Node{} exporter := &models.Agent{ - AgentID: "agent-id", - AgentType: models.NodeExporterType, + AgentID: "agent-id", + AgentType: models.NodeExporterType, + ExporterOptions: &models.ExporterOptions{}, } agentVersion := version.MustParse("2.28.0") @@ -83,8 +85,9 @@ func TestAuthWebConfig(t *testing.T) { node := &models.Node{} exporter := &models.Agent{ - AgentID: "agent-id", - AgentType: models.NodeExporterType, + AgentID: "agent-id", + AgentType: models.NodeExporterType, + ExporterOptions: &models.ExporterOptions{}, } agentVersion := version.MustParse("3.0.0") @@ -114,8 +117,9 @@ func TestNodeExporterConfig(t *testing.T) { Address: "1.2.3.4", } exporter := &models.Agent{ - AgentID: "agent-id", - AgentType: models.NodeExporterType, + AgentID: "agent-id", + AgentType: models.NodeExporterType, + ExporterOptions: &models.ExporterOptions{}, } agentVersion := version.MustParse("2.15.1") @@ -293,8 +297,9 @@ func TestNodeExporterConfig(t *testing.T) { Distro: "darwin", } exporter := &models.Agent{ - AgentID: "agent-id", - AgentType: models.NodeExporterType, + AgentID: "agent-id", + AgentType: models.NodeExporterType, + ExporterOptions: &models.ExporterOptions{}, } agentVersion := version.MustParse("2.15.1") From b5933aee7935e60e539e6e7da39731f4ec263ca1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Tue, 3 Dec 2024 22:07:18 +0100 Subject: [PATCH 42/60] PMM-5086-12634 Fix Postgres nil handling. --- managed/models/agent_helpers.go | 2 +- managed/models/agent_model.go | 2 +- managed/models/agent_model_test.go | 2 +- managed/services/agents/postgresql.go | 29 +++++----------------- managed/services/agents/postgresql_test.go | 21 +++++++++------- managed/services/converters.go | 2 +- managed/services/management/agent.go | 2 +- managed/services/management/rds.go | 2 +- 8 files changed, 24 insertions(+), 38 deletions(-) diff --git a/managed/models/agent_helpers.go b/managed/models/agent_helpers.go index d8e75bcc79..07230f0f17 100644 --- a/managed/models/agent_helpers.go +++ b/managed/models/agent_helpers.go @@ -64,7 +64,7 @@ type PostgreSQLOptionsParams interface { // PostgreSQLExtendedOptionsParams contains extended parameters for PostgreSQL exporter. type PostgreSQLExtendedOptionsParams interface { - GetAutoDiscoveryLimit() int32 + GetAutoDiscoveryLimit() *int32 GetMaxExporterConnections() int32 } diff --git a/managed/models/agent_model.go b/managed/models/agent_model.go index dbf6e76dc2..1c83cf0acc 100644 --- a/managed/models/agent_model.go +++ b/managed/models/agent_model.go @@ -182,7 +182,7 @@ type PostgreSQLOptions struct { SSLCa string `json:"ssl_ca"` SSLCert string `json:"ssl_cert"` SSLKey string `json:"ssl_key"` - AutoDiscoveryLimit int32 `json:"auto_discovery_limit"` + AutoDiscoveryLimit *int32 `json:"auto_discovery_limit"` DatabaseCount int32 `json:"database_count"` PGSMVersion *string `json:"pgsm_version"` MaxExporterConnections int32 `json:"max_exporter_connections"` diff --git a/managed/models/agent_model_test.go b/managed/models/agent_model_test.go index d00bbdd37d..3ca9b8f9fe 100644 --- a/managed/models/agent_model_test.go +++ b/managed/models/agent_model_test.go @@ -260,7 +260,7 @@ func TestPostgresAgentTLS(t *testing.T) { t.Run(fmt.Sprintf("AutodiscoveryLimit set TLS:%v/TLSSkipVerify:%v", testCase.tls, testCase.tlsSkipVerify), func(t *testing.T) { agent.TLS = testCase.tls agent.TLSSkipVerify = testCase.tlsSkipVerify - agent.PostgreSQLOptions = &models.PostgreSQLOptions{AutoDiscoveryLimit: 10} + agent.PostgreSQLOptions = &models.PostgreSQLOptions{AutoDiscoveryLimit: pointer.ToInt32(10)} assert.Equal(t, testCase.expected, agent.DSN(service, models.DSNParams{DialTimeout: time.Second, Database: "database"}, nil, nil)) }) } diff --git a/managed/services/agents/postgresql.go b/managed/services/agents/postgresql.go index ad524a33c8..2e8853b604 100644 --- a/managed/services/agents/postgresql.go +++ b/managed/services/agents/postgresql.go @@ -22,6 +22,7 @@ import ( "strings" "time" + "github.com/AlekSi/pointer" agentv1 "github.com/percona/pmm/api/agent/v1" inventoryv1 "github.com/percona/pmm/api/inventory/v1" "github.com/percona/pmm/managed/models" @@ -84,20 +85,16 @@ func postgresExporterConfig(node *models.Node, service *models.Service, exporter "--web.listen-address=" + listenAddress + ":" + tdp.Left + " .listen_port " + tdp.Right, } - if exporter.PostgreSQLOptions == nil { - exporter.PostgreSQLOptions = &models.PostgreSQLOptions{} - } - autoDiscovery := false if !pmmAgentVersion.Less(postgresExporterAutodiscoveryVersion) { switch { - case exporter.PostgreSQLOptions == nil: + case exporter.PostgreSQLOptions.AutoDiscoveryLimit == nil: autoDiscovery = true - case exporter.PostgreSQLOptions.AutoDiscoveryLimit == 0: // server defined + case pointer.GetInt32(exporter.PostgreSQLOptions.AutoDiscoveryLimit) == 0: // server defined autoDiscovery = exporter.PostgreSQLOptions.DatabaseCount <= defaultAutoDiscoveryDatabaseLimit - case exporter.PostgreSQLOptions.AutoDiscoveryLimit < 0: // always disabled + case pointer.GetInt32(exporter.PostgreSQLOptions.AutoDiscoveryLimit) < 0: // always disabled default: - autoDiscovery = exporter.PostgreSQLOptions.DatabaseCount <= exporter.PostgreSQLOptions.AutoDiscoveryLimit + autoDiscovery = exporter.PostgreSQLOptions.DatabaseCount <= pointer.GetInt32(exporter.PostgreSQLOptions.AutoDiscoveryLimit) } } if autoDiscovery { @@ -107,15 +104,10 @@ func postgresExporterConfig(node *models.Node, service *models.Service, exporter } if !pmmAgentVersion.Less(postgresMaxExporterConnsVersion) && - exporter.PostgreSQLOptions != nil && exporter.PostgreSQLOptions.MaxExporterConnections != 0 { args = append(args, "--max-connections="+strconv.Itoa(int(exporter.PostgreSQLOptions.MaxExporterConnections))) } - if exporter.ExporterOptions == nil { - exporter.ExporterOptions = &models.ExporterOptions{} - } - if exporter.ExporterOptions.MetricsPath != nil { args = append(args, "--web.telemetry-path="+*exporter.ExporterOptions.MetricsPath) } @@ -137,10 +129,7 @@ func postgresExporterConfig(node *models.Node, service *models.Service, exporter PostgreSQLSupportsSSLSNI: !pmmAgentVersion.Less(postgresSSLSniVersion), } - if exporter.AzureOptions == nil { - exporter.AzureOptions = &models.AzureOptions{} - } - if exporter.AzureOptions != nil { + if exporter.AzureOptions.ClientID != "" { dnsParams.DialTimeout = 5 * time.Second } @@ -174,9 +163,6 @@ func qanPostgreSQLPgStatementsAgentConfig(service *models.Service, agent *models Database: service.DatabaseName, PostgreSQLSupportsSSLSNI: !pmmAgentVersion.Less(postgresSSLSniVersion), } - if agent.QANOptions == nil { - agent.QANOptions = &models.QANOptions{} - } return &agentv1.SetStateRequest_BuiltinAgent{ Type: inventoryv1.AgentType_AGENT_TYPE_QAN_POSTGRESQL_PGSTATEMENTS_AGENT, @@ -199,9 +185,6 @@ func qanPostgreSQLPgStatMonitorAgentConfig(service *models.Service, agent *model Database: service.DatabaseName, PostgreSQLSupportsSSLSNI: !pmmAgentVersion.Less(postgresSSLSniVersion), } - if agent.QANOptions == nil { - agent.QANOptions = &models.QANOptions{} - } return &agentv1.SetStateRequest_BuiltinAgent{ Type: inventoryv1.AgentType_AGENT_TYPE_QAN_POSTGRESQL_PGSTATMONITOR_AGENT, diff --git a/managed/services/agents/postgresql_test.go b/managed/services/agents/postgresql_test.go index 62838401ae..e65044e777 100644 --- a/managed/services/agents/postgresql_test.go +++ b/managed/services/agents/postgresql_test.go @@ -196,10 +196,13 @@ func TestAutoDiscovery(t *testing.T) { DatabaseName: "postgres", } exporter := &models.Agent{ - AgentID: "agent-id", - AgentType: models.PostgresExporterType, - Username: pointer.ToString("username"), - Password: pointer.ToString("s3cur3 p@$$w0r4."), + AgentID: "agent-id", + AgentType: models.PostgresExporterType, + Username: pointer.ToString("username"), + Password: pointer.ToString("s3cur3 p@$$w0r4."), + ExporterOptions: &models.ExporterOptions{}, + AzureOptions: &models.AzureOptions{}, + PostgreSQLOptions: &models.PostgreSQLOptions{}, } expected := &agentv1.SetStateRequest_AgentProcess{ @@ -240,7 +243,7 @@ func TestAutoDiscovery(t *testing.T) { t.Run("Database count more than limit - disabled", func(t *testing.T) { exporter.PostgreSQLOptions = &models.PostgreSQLOptions{ - AutoDiscoveryLimit: 5, + AutoDiscoveryLimit: pointer.ToInt32(5), DatabaseCount: 10, } res, err := postgresExporterConfig(node, postgresql, exporter, redactSecrets, pmmAgentVersion) @@ -251,7 +254,7 @@ func TestAutoDiscovery(t *testing.T) { t.Run("Database count equal to limit - enabled", func(t *testing.T) { exporter.PostgreSQLOptions = &models.PostgreSQLOptions{ - AutoDiscoveryLimit: 5, + AutoDiscoveryLimit: pointer.ToInt32(5), DatabaseCount: 5, } res, err := postgresExporterConfig(node, postgresql, exporter, redactSecrets, pmmAgentVersion) @@ -262,7 +265,7 @@ func TestAutoDiscovery(t *testing.T) { t.Run("Database count less than limit - enabled", func(t *testing.T) { exporter.PostgreSQLOptions = &models.PostgreSQLOptions{ - AutoDiscoveryLimit: 5, + AutoDiscoveryLimit: pointer.ToInt32(5), DatabaseCount: 3, } res, err := postgresExporterConfig(node, postgresql, exporter, redactSecrets, pmmAgentVersion) @@ -273,7 +276,7 @@ func TestAutoDiscovery(t *testing.T) { t.Run("Negative limit - disabled", func(t *testing.T) { exporter.PostgreSQLOptions = &models.PostgreSQLOptions{ - AutoDiscoveryLimit: -1, + AutoDiscoveryLimit: pointer.ToInt32(-1), DatabaseCount: 3, } res, err := postgresExporterConfig(node, postgresql, exporter, redactSecrets, pmmAgentVersion) @@ -284,7 +287,7 @@ func TestAutoDiscovery(t *testing.T) { t.Run("Default - enabled", func(t *testing.T) { exporter.PostgreSQLOptions = &models.PostgreSQLOptions{ - AutoDiscoveryLimit: 0, + AutoDiscoveryLimit: pointer.ToInt32(0), DatabaseCount: 3, } res, err := postgresExporterConfig(node, postgresql, exporter, redactSecrets, pmmAgentVersion) diff --git a/managed/services/converters.go b/managed/services/converters.go index c950216a34..e35814a53a 100644 --- a/managed/services/converters.go +++ b/managed/services/converters.go @@ -330,7 +330,7 @@ func ToAPIAgent(q *reform.Querier, agent *models.Agent) (inventoryv1.Agent, erro MetricsResolutions: ConvertMetricsResolutions(agent.ExporterOptions.MetricsResolutions), } - exporter.AutoDiscoveryLimit = agent.PostgreSQLOptions.AutoDiscoveryLimit + exporter.AutoDiscoveryLimit = pointer.GetInt32(agent.PostgreSQLOptions.AutoDiscoveryLimit) exporter.MaxExporterConnections = agent.PostgreSQLOptions.MaxExporterConnections return exporter, nil diff --git a/managed/services/management/agent.go b/managed/services/management/agent.go index f35945031f..23f8f3bf0a 100644 --- a/managed/services/management/agent.go +++ b/managed/services/management/agent.go @@ -203,7 +203,7 @@ func (s *ManagementService) agentToAPI(agent *models.Agent) (*managementv1.Unive if agent.PostgreSQLOptions != nil { ua.PostgresqlOptions = &managementv1.UniversalAgent_PostgreSQLOptions{ IsSslKeySet: agent.PostgreSQLOptions.SSLKey != "", - AutoDiscoveryLimit: agent.PostgreSQLOptions.AutoDiscoveryLimit, + AutoDiscoveryLimit: pointer.GetInt32(agent.PostgreSQLOptions.AutoDiscoveryLimit), MaxExporterConnections: agent.PostgreSQLOptions.MaxExporterConnections, } } diff --git a/managed/services/management/rds.go b/managed/services/management/rds.go index 27f2c96a55..1020b9f72f 100644 --- a/managed/services/management/rds.go +++ b/managed/services/management/rds.go @@ -416,7 +416,7 @@ func (s *ManagementService) addRDS(ctx context.Context, req *managementv1.AddRDS }, PostgreSQLOptions: &models.PostgreSQLOptions{ - AutoDiscoveryLimit: req.AutoDiscoveryLimit, + AutoDiscoveryLimit: pointer.ToInt32(req.AutoDiscoveryLimit), MaxExporterConnections: req.MaxPostgresqlExporterConnections, }, }) From 6c43329ccfd32e7010f01762d9da4eabab29d5d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Tue, 3 Dec 2024 22:08:37 +0100 Subject: [PATCH 43/60] PMM-5086-12634 Fix RDS nil handling. --- managed/services/agents/rds.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/managed/services/agents/rds.go b/managed/services/agents/rds.go index dcd5d1ba70..1be5fe70ee 100644 --- a/managed/services/agents/rds.go +++ b/managed/services/agents/rds.go @@ -77,10 +77,6 @@ func rdsExporterConfig(pairs map[*models.Node]*models.Agent, redactMode redactMo return nil, err } - if exporter.AWSOptions == nil { - exporter.AWSOptions = &models.AWSOptions{} - } - config.Instances = append(config.Instances, rdsInstance{ Region: pointer.GetString(node.Region), Instance: node.Address, From 3042a839eca78b0654fd0fea3f9ff9864f0702b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Tue, 3 Dec 2024 22:30:56 +0100 Subject: [PATCH 44/60] PMM-5086-12634 Fix scraped configs. --- managed/services/agents/mongodb_test.go | 1 + managed/services/victoriametrics/scrape_configs_test.go | 2 ++ 2 files changed, 3 insertions(+) diff --git a/managed/services/agents/mongodb_test.go b/managed/services/agents/mongodb_test.go index c9459b4b99..5ba5e8f11d 100644 --- a/managed/services/agents/mongodb_test.go +++ b/managed/services/agents/mongodb_test.go @@ -114,6 +114,7 @@ func TestMongodbExporterConfig226(t *testing.T) { TemplateLeftDelim: "{{", TemplateRightDelim: "}}", Args: []string{ + "--collector.collstats-limit=0", "--collector.diagnosticdata", "--collector.replicasetstatus", "--compatible-mode", diff --git a/managed/services/victoriametrics/scrape_configs_test.go b/managed/services/victoriametrics/scrape_configs_test.go index 2322427521..711fde6029 100644 --- a/managed/services/victoriametrics/scrape_configs_test.go +++ b/managed/services/victoriametrics/scrape_configs_test.go @@ -261,6 +261,7 @@ func TestScrapeConfig(t *testing.T) { AgentType: models.MySQLdExporterType, CustomLabels: []byte(`{"_some_agent_label": "baz"}`), ListenPort: pointer.ToUint16(12345), + MySQLOptions: &models.MySQLOptions{}, } expected := []*config.ScrapeConfig{{ @@ -418,6 +419,7 @@ func TestScrapeConfig(t *testing.T) { ExporterOptions: &models.ExporterOptions{ DisabledCollectors: []string{"global_status", "info_schema.innodb_cmp", "info_schema.query_response_time", "perf_schema.eventsstatements", "heartbeat"}, }, + MySQLOptions: &models.MySQLOptions{}, } expected := []*config.ScrapeConfig{{ From 35aed540ef698de593e7369088a78b29aebb0225 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Tue, 3 Dec 2024 22:41:22 +0100 Subject: [PATCH 45/60] PMM-5086-12634 Lint. --- managed/services/agents/mongodb.go | 2 +- managed/services/agents/postgresql.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/managed/services/agents/mongodb.go b/managed/services/agents/mongodb.go index c3a8bb6ebe..ff188d68cc 100644 --- a/managed/services/agents/mongodb.go +++ b/managed/services/agents/mongodb.go @@ -50,7 +50,7 @@ func mongodbExporterConfig(node *models.Node, service *models.Service, exporter args := getArgs(exporter, tdp, listenAddress, pmmAgentVersion) if pointer.GetString(exporter.ExporterOptions.MetricsPath) != "" { - args = append(args, "--web.telemetry-path="+*exporter.ExporterOptions.MetricsPath) //nolint:goconst + args = append(args, "--web.telemetry-path="+*exporter.ExporterOptions.MetricsPath) } args = withLogLevel(args, exporter.LogLevel, pmmAgentVersion, true) diff --git a/managed/services/agents/postgresql.go b/managed/services/agents/postgresql.go index 2e8853b604..a4d0695f08 100644 --- a/managed/services/agents/postgresql.go +++ b/managed/services/agents/postgresql.go @@ -115,7 +115,7 @@ func postgresExporterConfig(node *models.Node, service *models.Service, exporter args = collectors.FilterOutCollectors("--collect.", args, exporter.ExporterOptions.DisabledCollectors) if !pmmAgentVersion.Less(postgresExporterCollectorsVersion) { - disableCollectorArgs := collectors.DisableDefaultEnabledCollectors("--no-collector.", defaultPostgresExporterCollectors, exporter.ExporterOptions.DisabledCollectors) + disableCollectorArgs := collectors.DisableDefaultEnabledCollectors("--no-collector.", defaultPostgresExporterCollectors, exporter.ExporterOptions.DisabledCollectors) //nolint:lll args = append(args, disableCollectorArgs...) } From 52a492bd538a51ffbed9b908398cfb55851252da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Tue, 3 Dec 2024 22:52:18 +0100 Subject: [PATCH 46/60] PMM-5086-12634 Fix another tests. --- managed/models/agent_model_test.go | 37 +++++++++++----- managed/services/agents/mongodb_test.go | 56 +++++++++++++++---------- 2 files changed, 60 insertions(+), 33 deletions(-) diff --git a/managed/models/agent_model_test.go b/managed/models/agent_model_test.go index 3ca9b8f9fe..ddc060d36c 100644 --- a/managed/models/agent_model_test.go +++ b/managed/models/agent_model_test.go @@ -48,8 +48,13 @@ func TestAgent(t *testing.T) { t.Run("DSN", func(t *testing.T) { agent := &models.Agent{ - Username: pointer.ToString("username"), - Password: pointer.ToString("s3cur3 p@$$w0r4."), + Username: pointer.ToString("username"), + Password: pointer.ToString("s3cur3 p@$$w0r4."), + ExporterOptions: &models.ExporterOptions{}, + QANOptions: &models.QANOptions{}, + MongoDBOptions: &models.MongoDBOptions{}, + MySQLOptions: &models.MySQLOptions{}, + PostgreSQLOptions: &models.PostgreSQLOptions{}, } service := &models.Service{ Address: pointer.ToString("1.2.3.4"), @@ -80,8 +85,11 @@ func TestAgent(t *testing.T) { t.Run("DSN socket", func(t *testing.T) { agent := &models.Agent{ - Username: pointer.ToString("username"), - Password: pointer.ToString("s3cur3 p@$$w0r4."), + Username: pointer.ToString("username"), + Password: pointer.ToString("s3cur3 p@$$w0r4."), + ExporterOptions: &models.ExporterOptions{}, + QANOptions: &models.QANOptions{}, + MySQLOptions: &models.MySQLOptions{}, } service := &models.Service{ Socket: pointer.ToString("/var/run/mysqld/mysqld.sock"), @@ -101,8 +109,11 @@ func TestAgent(t *testing.T) { t.Run("DSN timeout", func(t *testing.T) { agent := &models.Agent{ - Username: pointer.ToString("username"), - Password: pointer.ToString("s3cur3 p@$$w0r4."), + Username: pointer.ToString("username"), + Password: pointer.ToString("s3cur3 p@$$w0r4."), + ExporterOptions: &models.ExporterOptions{}, + QANOptions: &models.QANOptions{}, + MongoDBOptions: &models.MongoDBOptions{}, } service := &models.Service{ Socket: pointer.ToString("/var/run/mysqld/mysqld.sock"), @@ -139,6 +150,7 @@ func TestAgent(t *testing.T) { Username: pointer.ToString("username"), Password: pointer.ToString("s3cur3 p@$$w0r4."), TLS: true, + ExporterOptions: &models.ExporterOptions{}, MongoDBOptions: &mongoDBOptions, MySQLOptions: &mysqlOptions, PostgreSQLOptions: &postgresqlOptions, @@ -197,10 +209,15 @@ func TestAgent(t *testing.T) { t.Run("DSN ssl-skip-verify", func(t *testing.T) { agent := &models.Agent{ - Username: pointer.ToString("username"), - Password: pointer.ToString("s3cur3 p@$$w0r4."), - TLS: true, - TLSSkipVerify: true, + Username: pointer.ToString("username"), + Password: pointer.ToString("s3cur3 p@$$w0r4."), + TLS: true, + TLSSkipVerify: true, + ExporterOptions: &models.ExporterOptions{}, + QANOptions: &models.QANOptions{}, + MongoDBOptions: &models.MongoDBOptions{}, + MySQLOptions: &models.MySQLOptions{}, + PostgreSQLOptions: &models.PostgreSQLOptions{}, } service := &models.Service{ Address: pointer.ToString("1.2.3.4"), diff --git a/managed/services/agents/mongodb_test.go b/managed/services/agents/mongodb_test.go index 5ba5e8f11d..202c9b737d 100644 --- a/managed/services/agents/mongodb_test.go +++ b/managed/services/agents/mongodb_test.go @@ -248,11 +248,13 @@ func TestMongodbExporterConfig2411(t *testing.T) { Port: pointer.ToUint16(27017), } exporter := &models.Agent{ - AgentID: "agent-id", - AgentType: models.MongoDBExporterType, - Username: pointer.ToString("username"), - Password: pointer.ToString("s3cur3 p@$$w0r4."), - AgentPassword: pointer.ToString("agent-password"), + AgentID: "agent-id", + AgentType: models.MongoDBExporterType, + Username: pointer.ToString("username"), + Password: pointer.ToString("s3cur3 p@$$w0r4."), + AgentPassword: pointer.ToString("agent-password"), + ExporterOptions: &models.ExporterOptions{}, + MongoDBOptions: &models.MongoDBOptions{}, } actual, err := mongodbExporterConfig(node, mongodb, exporter, redactSecrets, pmmAgentVersion) expected := &agentv1.SetStateRequest_AgentProcess{ @@ -434,11 +436,13 @@ func TestMongodbExporterConfig2430(t *testing.T) { Port: pointer.ToUint16(27017), } exporter := &models.Agent{ - AgentID: "agent-id", - AgentType: models.MongoDBExporterType, - Username: pointer.ToString("username"), - Password: pointer.ToString("s3cur3 p@$$w0r4."), - AgentPassword: pointer.ToString("agent-password"), + AgentID: "agent-id", + AgentType: models.MongoDBExporterType, + Username: pointer.ToString("username"), + Password: pointer.ToString("s3cur3 p@$$w0r4."), + AgentPassword: pointer.ToString("agent-password"), + ExporterOptions: &models.ExporterOptions{}, + MongoDBOptions: &models.MongoDBOptions{}, } actual, err := mongodbExporterConfig(node, mongodb, exporter, redactSecrets, pmmAgentVersion) expected := &agentv1.SetStateRequest_AgentProcess{ @@ -514,11 +518,13 @@ func TestMongodbExporterConfig(t *testing.T) { Port: pointer.ToUint16(27017), } exporter := &models.Agent{ - AgentID: "agent-id", - AgentType: models.MongoDBExporterType, - Username: pointer.ToString("username"), - Password: pointer.ToString("s3cur3 p@$$w0r4."), - AgentPassword: pointer.ToString("agent-password"), + AgentID: "agent-id", + AgentType: models.MongoDBExporterType, + Username: pointer.ToString("username"), + Password: pointer.ToString("s3cur3 p@$$w0r4."), + AgentPassword: pointer.ToString("agent-password"), + ExporterOptions: &models.ExporterOptions{}, + MongoDBOptions: &models.MongoDBOptions{}, } actual, err := mongodbExporterConfig(node, mongodb, exporter, redactSecrets, pmmAgentVersion) expected := &agentv1.SetStateRequest_AgentProcess{ @@ -637,10 +643,12 @@ func TestNewMongodbExporterConfig(t *testing.T) { Port: pointer.ToUint16(27017), } exporter := &models.Agent{ - AgentID: "agent-id", - AgentType: models.MongoDBExporterType, - Username: pointer.ToString("username"), - Password: pointer.ToString("s3cur3 p@$$w0r4."), + AgentID: "agent-id", + AgentType: models.MongoDBExporterType, + Username: pointer.ToString("username"), + Password: pointer.ToString("s3cur3 p@$$w0r4."), + ExporterOptions: &models.ExporterOptions{}, + MongoDBOptions: &models.MongoDBOptions{}, } actual, err := mongodbExporterConfig(node, mongodb, exporter, redactSecrets, pmmAgentVersion) require.NoError(t, err) @@ -693,10 +701,12 @@ func TestMongodbExporterConfig228_WebConfigAuth(t *testing.T) { Port: pointer.ToUint16(27017), } exporter := &models.Agent{ - AgentID: "agent-id", - AgentType: models.MongoDBExporterType, - Username: pointer.ToString("username"), - Password: pointer.ToString("s3cur3 p@$$w0r4."), + AgentID: "agent-id", + AgentType: models.MongoDBExporterType, + Username: pointer.ToString("username"), + Password: pointer.ToString("s3cur3 p@$$w0r4."), + ExporterOptions: &models.ExporterOptions{}, + MongoDBOptions: &models.MongoDBOptions{}, } expectedArgs := []string{ From e7ab84554bdf6a5829d0233d3e3572646216cdfe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Tue, 3 Dec 2024 22:58:30 +0100 Subject: [PATCH 47/60] PMM-5086-12634 Format. --- managed/services/agents/postgresql.go | 1 + 1 file changed, 1 insertion(+) diff --git a/managed/services/agents/postgresql.go b/managed/services/agents/postgresql.go index a4d0695f08..f20b8cfffc 100644 --- a/managed/services/agents/postgresql.go +++ b/managed/services/agents/postgresql.go @@ -23,6 +23,7 @@ import ( "time" "github.com/AlekSi/pointer" + agentv1 "github.com/percona/pmm/api/agent/v1" inventoryv1 "github.com/percona/pmm/api/inventory/v1" "github.com/percona/pmm/managed/models" From 1a661a85124a30559eaf5efaea54966a83a8c2fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Wed, 4 Dec 2024 10:12:22 +0100 Subject: [PATCH 48/60] PMM-5086-12634 Fix for another tests. --- managed/models/agent_model_test.go | 8 +++++--- managed/services/agents/mongodb_test.go | 25 ++++++++++++++++--------- 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/managed/models/agent_model_test.go b/managed/models/agent_model_test.go index ddc060d36c..909d49b3aa 100644 --- a/managed/models/agent_model_test.go +++ b/managed/models/agent_model_test.go @@ -249,9 +249,11 @@ func TestAgent(t *testing.T) { func TestPostgresAgentTLS(t *testing.T) { agent := &models.Agent{ - Username: pointer.ToString("username"), - Password: pointer.ToString("s3cur3 p@$$w0r4."), - AgentType: models.PostgresExporterType, + Username: pointer.ToString("username"), + Password: pointer.ToString("s3cur3 p@$$w0r4."), + AgentType: models.PostgresExporterType, + ExporterOptions: &models.ExporterOptions{}, + PostgreSQLOptions: &models.PostgreSQLOptions{}, } service := &models.Service{ Address: pointer.ToString("1.2.3.4"), diff --git a/managed/services/agents/mongodb_test.go b/managed/services/agents/mongodb_test.go index 202c9b737d..6cb5f285a2 100644 --- a/managed/services/agents/mongodb_test.go +++ b/managed/services/agents/mongodb_test.go @@ -262,6 +262,7 @@ func TestMongodbExporterConfig2411(t *testing.T) { TemplateLeftDelim: "{{", TemplateRightDelim: "}}", Args: []string{ + "--collector.collstats-limit=0", "--collector.diagnosticdata", "--collector.replicasetstatus", "--compatible-mode", @@ -450,6 +451,7 @@ func TestMongodbExporterConfig2430(t *testing.T) { TemplateLeftDelim: "{{", TemplateRightDelim: "}}", Args: []string{ + "--collector.collstats-limit=0", "--collector.diagnosticdata", "--collector.fcv", "--collector.pbm", @@ -710,6 +712,7 @@ func TestMongodbExporterConfig228_WebConfigAuth(t *testing.T) { } expectedArgs := []string{ + "--collector.collstats-limit=0", "--collector.diagnosticdata", "--collector.replicasetstatus", "--compatible-mode", @@ -727,11 +730,13 @@ func TestMongodbExporterConfig228_WebConfigAuth(t *testing.T) { t.Parallel() localExporter := &models.Agent{ - AgentID: exporter.AgentID, - AgentType: exporter.AgentType, - Username: exporter.Username, - Password: exporter.Password, - AgentPassword: pointer.ToString("agent-custom-password"), + AgentID: exporter.AgentID, + AgentType: exporter.AgentType, + Username: exporter.Username, + Password: exporter.Password, + AgentPassword: pointer.ToString("agent-custom-password"), + ExporterOptions: &models.ExporterOptions{}, + MongoDBOptions: &models.MongoDBOptions{}, } actual, err := mongodbExporterConfig(node, mongodb, localExporter, redactSecrets, pmmAgentVersion) @@ -756,10 +761,12 @@ func TestMongodbExporterConfig228_WebConfigAuth(t *testing.T) { t.Parallel() localExporter := &models.Agent{ - AgentID: exporter.AgentID, - AgentType: exporter.AgentType, - Username: exporter.Username, - Password: exporter.Password, + AgentID: exporter.AgentID, + AgentType: exporter.AgentType, + Username: exporter.Username, + Password: exporter.Password, + ExporterOptions: &models.ExporterOptions{}, + MongoDBOptions: &models.MongoDBOptions{}, } actual, err := mongodbExporterConfig(node, mongodb, localExporter, redactSecrets, pmmAgentVersion) From ca3d1bf9a1f340d6690435657ec02297f6063fe3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Wed, 4 Dec 2024 10:43:09 +0100 Subject: [PATCH 49/60] PMM-5086-12634 CreateAgent options preparation. --- managed/models/agent_helpers.go | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/managed/models/agent_helpers.go b/managed/models/agent_helpers.go index 07230f0f17..30fc3da44a 100644 --- a/managed/models/agent_helpers.go +++ b/managed/models/agent_helpers.go @@ -52,7 +52,7 @@ func MySQLOptionsFromRequest(params MySQLOptionsParams) *MySQLOptions { TLSKey: params.GetTlsKey(), } } - return &MySQLOptions{} + return nil } // PostgreSQLOptionsParams contains methods to create PostgreSQLOptions object. @@ -858,6 +858,32 @@ func compatibleServiceAndAgent(serviceType ServiceType, agentType AgentType) boo return false } +func prepareOptionsParams(params *CreateAgentParams) *CreateAgentParams { + if params.ExporterOptions == nil { + params.ExporterOptions = &ExporterOptions{} + } + if params.QANOptions == nil { + params.QANOptions = &QANOptions{} + } + if params.AWSOptions == nil { + params.AWSOptions = &AWSOptions{} + } + if params.AzureOptions == nil { + params.AzureOptions = &AzureOptions{} + } + if params.MongoDBOptions == nil { + params.MongoDBOptions = &MongoDBOptions{} + } + if params.MySQLOptions == nil { + params.MySQLOptions = &MySQLOptions{} + } + if params.PostgreSQLOptions == nil { + params.PostgreSQLOptions = &PostgreSQLOptions{} + } + + return params +} + // CreateAgent creates Agent with given type. func CreateAgent(q *reform.Querier, agentType AgentType, params *CreateAgentParams) (*Agent, error) { //nolint:unparam id := uuid.New().String() @@ -865,12 +891,14 @@ func CreateAgent(q *reform.Querier, agentType AgentType, params *CreateAgentPara return nil, err } + params = prepareOptionsParams(params) + pmmAgent, err := FindAgentByID(q, params.PMMAgentID) if err != nil { return nil, err } // check version for agent, if it exists. - if params.ExporterOptions != nil && params.ExporterOptions.PushMetrics { + if params.ExporterOptions.PushMetrics { // special case for vmAgent, it always supports push metrics. if agentType != VMAgentType && !IsPushMetricsSupported(pmmAgent.Version) { return nil, status.Errorf(codes.FailedPrecondition, "cannot use push_metrics_enabled with pmm_agent version=%q,"+ From 3b387489848254e4307ddd90540be97753ecf335 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Wed, 4 Dec 2024 10:46:05 +0100 Subject: [PATCH 50/60] PMM-5086-12634 Fix another tests. --- managed/models/agent_model_test.go | 36 ++++++++++++++++++++---------- 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/managed/models/agent_model_test.go b/managed/models/agent_model_test.go index 909d49b3aa..65f71d6ff0 100644 --- a/managed/models/agent_model_test.go +++ b/managed/models/agent_model_test.go @@ -288,10 +288,12 @@ func TestPostgresAgentTLS(t *testing.T) { func TestPostgresWithSocket(t *testing.T) { t.Run("empty-password", func(t *testing.T) { agent := &models.Agent{ - Username: pointer.ToString("username"), - AgentType: models.PostgresExporterType, - TLS: true, - TLSSkipVerify: false, + Username: pointer.ToString("username"), + AgentType: models.PostgresExporterType, + TLS: true, + TLSSkipVerify: false, + ExporterOptions: &models.ExporterOptions{}, + PostgreSQLOptions: &models.PostgreSQLOptions{}, } service := &models.Service{ Socket: pointer.ToString("/var/run/postgres"), @@ -302,7 +304,9 @@ func TestPostgresWithSocket(t *testing.T) { t.Run("empty-user-password", func(t *testing.T) { agent := &models.Agent{ - AgentType: models.PostgresExporterType, + AgentType: models.PostgresExporterType, + ExporterOptions: &models.ExporterOptions{}, + PostgreSQLOptions: &models.PostgreSQLOptions{}, } service := &models.Service{ Socket: pointer.ToString("/var/run/postgres"), @@ -313,7 +317,9 @@ func TestPostgresWithSocket(t *testing.T) { t.Run("dir-with-symbols", func(t *testing.T) { agent := &models.Agent{ - AgentType: models.PostgresExporterType, + AgentType: models.PostgresExporterType, + ExporterOptions: &models.ExporterOptions{}, + PostgreSQLOptions: &models.PostgreSQLOptions{}, } service := &models.Service{ Socket: pointer.ToString(`/tmp/123\ A0m\%\$\@\8\,\+\-`), @@ -326,10 +332,12 @@ func TestPostgresWithSocket(t *testing.T) { func TestMongoWithSocket(t *testing.T) { t.Run("empty-password", func(t *testing.T) { agent := &models.Agent{ - Username: pointer.ToString("username"), - AgentType: models.MongoDBExporterType, - TLS: true, - TLSSkipVerify: false, + Username: pointer.ToString("username"), + AgentType: models.MongoDBExporterType, + TLS: true, + TLSSkipVerify: false, + ExporterOptions: &models.ExporterOptions{}, + MongoDBOptions: &models.MongoDBOptions{}, } service := &models.Service{ Socket: pointer.ToString("/tmp/mongodb-27017.sock"), @@ -340,7 +348,9 @@ func TestMongoWithSocket(t *testing.T) { t.Run("empty-user-password", func(t *testing.T) { agent := &models.Agent{ - AgentType: models.MongoDBExporterType, + AgentType: models.MongoDBExporterType, + ExporterOptions: &models.ExporterOptions{}, + MongoDBOptions: &models.MongoDBOptions{}, } service := &models.Service{ Socket: pointer.ToString("/tmp/mongodb-27017.sock"), @@ -351,7 +361,9 @@ func TestMongoWithSocket(t *testing.T) { t.Run("dir-with-symbols", func(t *testing.T) { agent := &models.Agent{ - AgentType: models.MongoDBExporterType, + AgentType: models.MongoDBExporterType, + ExporterOptions: &models.ExporterOptions{}, + MongoDBOptions: &models.MongoDBOptions{}, } service := &models.Service{ Socket: pointer.ToString(`/tmp/123\ A0m\%\$\@\8\,\+\-/mongodb-27017.sock`), From 0c196b974dbe9af372c086e5afbd901d8d5a175f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Wed, 4 Dec 2024 12:31:41 +0100 Subject: [PATCH 51/60] PMM-5086-12634 PG test. --- managed/services/agents/postgresql_test.go | 54 ++++++++++++++-------- 1 file changed, 35 insertions(+), 19 deletions(-) diff --git a/managed/services/agents/postgresql_test.go b/managed/services/agents/postgresql_test.go index e65044e777..178824f6ea 100644 --- a/managed/services/agents/postgresql_test.go +++ b/managed/services/agents/postgresql_test.go @@ -311,10 +311,12 @@ func TestMaxConnections(t *testing.T) { DatabaseName: "postgres", } exporter := &models.Agent{ - AgentID: "agent-id", - AgentType: models.PostgresExporterType, - Username: pointer.ToString("username"), - Password: pointer.ToString("s3cur3 p@$$w0r4."), + AgentID: "agent-id", + AgentType: models.PostgresExporterType, + Username: pointer.ToString("username"), + Password: pointer.ToString("s3cur3 p@$$w0r4."), + ExporterOptions: &models.ExporterOptions{}, + AzureOptions: &models.AzureOptions{}, PostgreSQLOptions: &models.PostgreSQLOptions{ MaxExporterConnections: 10, }, @@ -384,10 +386,11 @@ func (s *PostgresExporterConfigTestSuite) TestAzureTimeout() { DatabaseName: "postgres", } s.exporter = &models.Agent{ - AgentID: "agent-id", - AgentType: models.PostgresExporterType, - Username: pointer.ToString("username"), - Password: pointer.ToString("s3cur3 p@$$w0r4."), + AgentID: "agent-id", + AgentType: models.PostgresExporterType, + Username: pointer.ToString("username"), + Password: pointer.ToString("s3cur3 p@$$w0r4."), + ExporterOptions: &models.ExporterOptions{}, AzureOptions: &models.AzureOptions{ SubscriptionID: "subscription_id", ClientID: "client_id", @@ -395,6 +398,7 @@ func (s *PostgresExporterConfigTestSuite) TestAzureTimeout() { TenantID: "tenant_id", ResourceGroup: "resource_group", }, + PostgreSQLOptions: &models.PostgreSQLOptions{}, } actual, err := postgresExporterConfig(s.node, s.postgresql, s.exporter, redactSecrets, s.pmmAgentVersion) @@ -436,11 +440,14 @@ func (s *PostgresExporterConfigTestSuite) TestPrometheusWebConfig() { DatabaseName: "postgres", } s.exporter = &models.Agent{ - AgentID: "agent-id", - AgentType: models.PostgresExporterType, - Username: pointer.ToString("username"), - Password: pointer.ToString("s3cur3 p@$$w0r4."), - TLS: true, + AgentID: "agent-id", + AgentType: models.PostgresExporterType, + Username: pointer.ToString("username"), + Password: pointer.ToString("s3cur3 p@$$w0r4."), + TLS: true, + ExporterOptions: &models.ExporterOptions{}, + AzureOptions: &models.AzureOptions{}, + PostgreSQLOptions: &models.PostgreSQLOptions{}, } actual, err := postgresExporterConfig(s.node, s.postgresql, s.exporter, redactSecrets, s.pmmAgentVersion) @@ -485,11 +492,14 @@ func (s *PostgresExporterConfigTestSuite) TestSSLSni() { DatabaseName: "postgres", } s.exporter = &models.Agent{ - AgentID: "agent-id", - AgentType: models.PostgresExporterType, - Username: pointer.ToString("username"), - Password: pointer.ToString("s3cur3 p@$$w0r4."), - TLS: true, + AgentID: "agent-id", + AgentType: models.PostgresExporterType, + Username: pointer.ToString("username"), + Password: pointer.ToString("s3cur3 p@$$w0r4."), + TLS: true, + ExporterOptions: &models.ExporterOptions{}, + AzureOptions: &models.AzureOptions{}, + PostgreSQLOptions: &models.PostgreSQLOptions{}, } actual, err := postgresExporterConfig(s.node, s.postgresql, s.exporter, redactSecrets, s.pmmAgentVersion) @@ -526,5 +536,11 @@ func (s *PostgresExporterConfigTestSuite) TestSSLSni() { } func TestPostgresExporterConfigTestSuite(t *testing.T) { - suite.Run(t, &PostgresExporterConfigTestSuite{}) + suite.Run(t, &PostgresExporterConfigTestSuite{ + exporter: &models.Agent{ + ExporterOptions: &models.ExporterOptions{}, + AzureOptions: &models.AzureOptions{}, + PostgreSQLOptions: &models.PostgreSQLOptions{}, + }, + }) } From 2935856705a417fd1e88d25864d2a3f1703aaa3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Wed, 4 Dec 2024 19:25:17 +0100 Subject: [PATCH 52/60] PMM-5086-12634 Fix PG test suite. --- managed/services/agents/postgresql_test.go | 34 +++++++++++++++++----- 1 file changed, 27 insertions(+), 7 deletions(-) diff --git a/managed/services/agents/postgresql_test.go b/managed/services/agents/postgresql_test.go index 178824f6ea..a1d115f127 100644 --- a/managed/services/agents/postgresql_test.go +++ b/managed/services/agents/postgresql_test.go @@ -78,6 +78,10 @@ func (s *PostgresExporterConfigTestSuite) SetupTest() { } func (s *PostgresExporterConfigTestSuite) TestConfig() { + s.exporter.ExporterOptions = &models.ExporterOptions{} + s.exporter.AzureOptions = &models.AzureOptions{} + s.exporter.PostgreSQLOptions = &models.PostgreSQLOptions{} + actual, err := postgresExporterConfig(s.node, s.postgresql, s.exporter, redactSecrets, s.pmmAgentVersion) s.NoError(err, "Failed to create exporter config") @@ -92,6 +96,10 @@ func (s *PostgresExporterConfigTestSuite) TestDatabaseName() { s.postgresql.DatabaseName = "db1" s.expected.Env[0] = "DATA_SOURCE_NAME=postgres://username:s3cur3%20p%40$$w0r4.@1.2.3.4:5432/db1?connect_timeout=1&sslmode=disable" + s.exporter.ExporterOptions = &models.ExporterOptions{} + s.exporter.AzureOptions = &models.AzureOptions{} + s.exporter.PostgreSQLOptions = &models.PostgreSQLOptions{} + actual, err := postgresExporterConfig(s.node, s.postgresql, s.exporter, redactSecrets, s.pmmAgentVersion) s.NoError(err, "Failed to create exporter config") @@ -102,6 +110,10 @@ func (s *PostgresExporterConfigTestSuite) TestDatabaseName() { s.postgresql.DatabaseName = "" s.Require().PanicsWithValue("database name not set", func() { + s.exporter.ExporterOptions = &models.ExporterOptions{} + s.exporter.AzureOptions = &models.AzureOptions{} + s.exporter.PostgreSQLOptions = &models.PostgreSQLOptions{} + _, err := postgresExporterConfig(s.node, s.postgresql, s.exporter, redactSecrets, s.pmmAgentVersion) s.NoError(err, "Failed to create exporter config") }) @@ -110,6 +122,9 @@ func (s *PostgresExporterConfigTestSuite) TestDatabaseName() { func (s *PostgresExporterConfigTestSuite) TestEmptyPassword() { s.exporter.Password = nil + s.exporter.ExporterOptions = &models.ExporterOptions{} + s.exporter.AzureOptions = &models.AzureOptions{} + s.exporter.PostgreSQLOptions = &models.PostgreSQLOptions{} actual, err := postgresExporterConfig(s.node, s.postgresql, s.exporter, exposeSecrets, s.pmmAgentVersion) s.NoError(err, "Failed to create exporter config") @@ -119,6 +134,9 @@ func (s *PostgresExporterConfigTestSuite) TestEmptyPassword() { func (s *PostgresExporterConfigTestSuite) TestEmptyUsername() { s.exporter.Username = nil + s.exporter.ExporterOptions = &models.ExporterOptions{} + s.exporter.AzureOptions = &models.AzureOptions{} + s.exporter.PostgreSQLOptions = &models.PostgreSQLOptions{} actual, err := postgresExporterConfig(s.node, s.postgresql, s.exporter, exposeSecrets, s.pmmAgentVersion) s.NoError(err, "Failed to create exporter config") @@ -129,6 +147,9 @@ func (s *PostgresExporterConfigTestSuite) TestEmptyUsername() { func (s *PostgresExporterConfigTestSuite) TestEmptyUsernameAndPassword() { s.exporter.Username = nil s.exporter.Password = nil + s.exporter.ExporterOptions = &models.ExporterOptions{} + s.exporter.AzureOptions = &models.AzureOptions{} + s.exporter.PostgreSQLOptions = &models.PostgreSQLOptions{} actual, err := postgresExporterConfig(s.node, s.postgresql, s.exporter, exposeSecrets, s.pmmAgentVersion) s.NoError(err, "Failed to create exporter config") @@ -142,6 +163,9 @@ func (s *PostgresExporterConfigTestSuite) TestSocket() { s.postgresql.Address = nil s.postgresql.Port = nil s.postgresql.Socket = pointer.ToString("/var/run/postgres") + s.exporter.ExporterOptions = &models.ExporterOptions{} + s.exporter.AzureOptions = &models.AzureOptions{} + s.exporter.PostgreSQLOptions = &models.PostgreSQLOptions{} actual, err := postgresExporterConfig(s.node, s.postgresql, s.exporter, exposeSecrets, s.pmmAgentVersion) s.NoError(err, "Failed to create exporter config") @@ -157,6 +181,8 @@ func (s *PostgresExporterConfigTestSuite) TestDisabledCollectors() { s.exporter.ExporterOptions = &models.ExporterOptions{ DisabledCollectors: []string{"custom_query.hr", "custom_query.hr.directory", "locks"}, } + s.exporter.AzureOptions = &models.AzureOptions{} + s.exporter.PostgreSQLOptions = &models.PostgreSQLOptions{} actual, err := postgresExporterConfig(s.node, s.postgresql, s.exporter, exposeSecrets, s.pmmAgentVersion) s.NoError(err, "Failed to create exporter config") @@ -536,11 +562,5 @@ func (s *PostgresExporterConfigTestSuite) TestSSLSni() { } func TestPostgresExporterConfigTestSuite(t *testing.T) { - suite.Run(t, &PostgresExporterConfigTestSuite{ - exporter: &models.Agent{ - ExporterOptions: &models.ExporterOptions{}, - AzureOptions: &models.AzureOptions{}, - PostgreSQLOptions: &models.PostgreSQLOptions{}, - }, - }) + suite.Run(t, &PostgresExporterConfigTestSuite{}) } From 13107af4b525955877e0016431722eea3fc75b40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Wed, 4 Dec 2024 19:40:31 +0100 Subject: [PATCH 53/60] PMM-5086-12634 Fix proxy exporter. --- managed/services/agents/proxysql_test.go | 29 +++++++++++++----------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/managed/services/agents/proxysql_test.go b/managed/services/agents/proxysql_test.go index bd870e76d5..ba33c65943 100644 --- a/managed/services/agents/proxysql_test.go +++ b/managed/services/agents/proxysql_test.go @@ -39,11 +39,12 @@ func TestProxySQLExporterConfig(t *testing.T) { Address: "1.2.3.4", } exporter := &models.Agent{ - AgentID: "agent-id", - AgentType: models.ProxySQLExporterType, - Username: pointer.ToString("username"), - Password: pointer.ToString("s3cur3 p@$$w0r4."), - AgentPassword: pointer.ToString("agent-password"), + AgentID: "agent-id", + AgentType: models.ProxySQLExporterType, + Username: pointer.ToString("username"), + Password: pointer.ToString("s3cur3 p@$$w0r4."), + AgentPassword: pointer.ToString("agent-password"), + ExporterOptions: &models.ExporterOptions{}, } actual := proxysqlExporterConfig(node, proxysql, exporter, redactSecrets, pmmAgentVersion) expected := &agentv1.SetStateRequest_AgentProcess{ @@ -104,10 +105,11 @@ func TestProxySQLExporterConfig(t *testing.T) { Port: pointer.ToUint16(3306), } exporter := &models.Agent{ - AgentID: "agent-id", - AgentType: models.ProxySQLExporterType, - Username: pointer.ToString("username"), - Password: pointer.ToString("s3cur3 p@$$w0r4."), + AgentID: "agent-id", + AgentType: models.ProxySQLExporterType, + Username: pointer.ToString("username"), + Password: pointer.ToString("s3cur3 p@$$w0r4."), + ExporterOptions: &models.ExporterOptions{}, } actual := proxysqlExporterConfig(node, proxysql, exporter, redactSecrets, pmmAgentVersion) expected := &agentv1.SetStateRequest_AgentProcess{ @@ -144,10 +146,11 @@ func TestProxySQLExporterConfig(t *testing.T) { Port: pointer.ToUint16(3306), } exporter := &models.Agent{ - AgentID: "agent-id", - AgentType: models.ProxySQLExporterType, - Username: pointer.ToString("username"), - Password: pointer.ToString("s3cur3 p@$$w0r4."), + AgentID: "agent-id", + AgentType: models.ProxySQLExporterType, + Username: pointer.ToString("username"), + Password: pointer.ToString("s3cur3 p@$$w0r4."), + ExporterOptions: &models.ExporterOptions{}, } actual := proxysqlExporterConfig(node, proxysql, exporter, redactSecrets, pmmAgentVersion) expected := &agentv1.SetStateRequest_AgentProcess{ From 68d7b9fae01c49db5a1c6da74e489d1cda82b305 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Wed, 4 Dec 2024 19:48:45 +0100 Subject: [PATCH 54/60] PMM-5086-12634 Another MySQL fix. --- managed/models/agent_helpers.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/managed/models/agent_helpers.go b/managed/models/agent_helpers.go index 30fc3da44a..d0e5d12fb1 100644 --- a/managed/models/agent_helpers.go +++ b/managed/models/agent_helpers.go @@ -52,7 +52,7 @@ func MySQLOptionsFromRequest(params MySQLOptionsParams) *MySQLOptions { TLSKey: params.GetTlsKey(), } } - return nil + return &MySQLOptions{} } // PostgreSQLOptionsParams contains methods to create PostgreSQLOptions object. From 68ac91ae8e46baa96db93a5d08a30082d1563320 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Wed, 4 Dec 2024 20:12:16 +0100 Subject: [PATCH 55/60] PMM-5086-12634 Cleanup. --- managed/models/agent_helpers.go | 15 +++------------ .../services/victoriametrics/scrape_configs.go | 2 +- 2 files changed, 4 insertions(+), 13 deletions(-) diff --git a/managed/models/agent_helpers.go b/managed/models/agent_helpers.go index d0e5d12fb1..9fe1aa2044 100644 --- a/managed/models/agent_helpers.go +++ b/managed/models/agent_helpers.go @@ -104,7 +104,7 @@ type MongoDBExtendedOptionsParams interface { // MongoDBOptionsFromRequest creates MongoDBOptionsParams object from request. func MongoDBOptionsFromRequest(params MongoDBOptionsParams) *MongoDBOptions { - var mdbOptions *MongoDBOptions + mdbOptions := &MongoDBOptions{} if params.GetTlsCertificateKey() != "" || params.GetTlsCertificateKeyFilePassword() != "" || params.GetTlsCa() != "" { mdbOptions = &MongoDBOptions{} @@ -114,9 +114,6 @@ func MongoDBOptionsFromRequest(params MongoDBOptionsParams) *MongoDBOptions { } if params.GetAuthenticationMechanism() != "" || params.GetAuthenticationDatabase() != "" { - if mdbOptions == nil { - mdbOptions = &MongoDBOptions{} - } mdbOptions.AuthenticationMechanism = params.GetAuthenticationMechanism() mdbOptions.AuthenticationDatabase = params.GetAuthenticationDatabase() } @@ -124,9 +121,6 @@ func MongoDBOptionsFromRequest(params MongoDBOptionsParams) *MongoDBOptions { // MongoDB exporter has these parameters but they are not needed for QAN agent. if extendedOptions, ok := params.(MongoDBExtendedOptionsParams); ok { if extendedOptions != nil { - if mdbOptions == nil { - mdbOptions = &MongoDBOptions{} - } mdbOptions.StatsCollections = extendedOptions.GetStatsCollections() mdbOptions.CollectionsLimit = extendedOptions.GetCollectionsLimit() mdbOptions.EnableAllCollectors = extendedOptions.GetEnableAllCollectors() @@ -1010,9 +1004,6 @@ func ChangeAgent(q *reform.Querier, agentID string, params *ChangeCommonAgentPar } } - if row.ExporterOptions == nil { - row.ExporterOptions = &ExporterOptions{} - } if row.ExporterOptions.MetricsResolutions == nil { row.ExporterOptions.MetricsResolutions = &MetricsResolutions{} } @@ -1080,7 +1071,7 @@ func RemoveAgent(q *reform.Querier, id string, mode RemoveMode) (*Agent, error) // for external exporter, is needed for push_metrics mode. func updateExternalExporterParams(q *reform.Querier, row *Agent) error { // with push metrics, external exporter must have PMMAgent id without RunsOnNodeID - if row.ExporterOptions != nil && row.ExporterOptions.PushMetrics && row.PMMAgentID == nil { + if row.ExporterOptions.PushMetrics && row.PMMAgentID == nil { pmmAgent, err := FindPMMAgentsRunningOnNode(q, pointer.GetString(row.RunsOnNodeID)) if err != nil { return err @@ -1097,7 +1088,7 @@ func updateExternalExporterParams(q *reform.Querier, row *Agent) error { row.PMMAgentID = pointer.ToString(pmmAgent[0].AgentID) } // without push metrics, external exporter must have RunsOnNodeID without PMMAgentID - if row.ExporterOptions != nil && !row.ExporterOptions.PushMetrics && row.RunsOnNodeID == nil { + if !row.ExporterOptions.PushMetrics && row.RunsOnNodeID == nil { pmmAgent, err := FindAgentByID(q, pointer.GetString(row.PMMAgentID)) if err != nil { return err diff --git a/managed/services/victoriametrics/scrape_configs.go b/managed/services/victoriametrics/scrape_configs.go index 1420a09be8..9ff886d673 100644 --- a/managed/services/victoriametrics/scrape_configs.go +++ b/managed/services/victoriametrics/scrape_configs.go @@ -398,7 +398,7 @@ func scrapeConfigsForMongoDBExporter(params *scrapeConfigParams) ([]*config.Scra if hr != nil { r = append(r, hr) } - if params.agent.MongoDBOptions != nil && params.agent.MongoDBOptions.EnableAllCollectors { + if params.agent.MongoDBOptions.EnableAllCollectors { defaultCollectors := []string{ "dbstats", "indexstats", From 7a57b10746065ccd0da927111bf3681009496e12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Thu, 5 Dec 2024 10:27:01 +0100 Subject: [PATCH 56/60] PMM-5086-12634 API tests/logic fix. --- managed/models/agent_helpers.go | 12 +++++------ managed/models/agent_helpers_test.go | 4 ++-- managed/models/agent_model.go | 10 +++++----- managed/models/agent_model_test.go | 20 +++++++++---------- managed/services/agents/mongodb.go | 6 ++---- managed/services/agents/mysql.go | 4 ++-- managed/services/agents/node.go | 8 +++----- managed/services/agents/postgresql.go | 4 ++-- managed/services/agents/proxysql.go | 4 ++-- managed/services/converters.go | 4 ++-- managed/services/management/agent.go | 4 ++-- .../victoriametrics/scrape_configs.go | 8 ++++---- .../victoriametrics/scrape_configs_test.go | 4 ++-- 13 files changed, 43 insertions(+), 49 deletions(-) diff --git a/managed/models/agent_helpers.go b/managed/models/agent_helpers.go index 9fe1aa2044..f588fd4f17 100644 --- a/managed/models/agent_helpers.go +++ b/managed/models/agent_helpers.go @@ -64,7 +64,7 @@ type PostgreSQLOptionsParams interface { // PostgreSQLExtendedOptionsParams contains extended parameters for PostgreSQL exporter. type PostgreSQLExtendedOptionsParams interface { - GetAutoDiscoveryLimit() *int32 + GetAutoDiscoveryLimit() int32 GetMaxExporterConnections() int32 } @@ -79,7 +79,7 @@ func PostgreSQLOptionsFromRequest(params PostgreSQLOptionsParams) *PostgreSQLOpt // PostgreSQL exporter has these parameters but they are not needed for QAN agent. if extendedOptions, ok := params.(PostgreSQLExtendedOptionsParams); ok && extendedOptions != nil { - res.AutoDiscoveryLimit = extendedOptions.GetAutoDiscoveryLimit() + res.AutoDiscoveryLimit = pointer.ToInt32(extendedOptions.GetAutoDiscoveryLimit()) res.MaxExporterConnections = extendedOptions.GetMaxExporterConnections() } @@ -736,8 +736,8 @@ func CreateExternalExporter(q *reform.Querier, params *CreateExternalExporterPar ListenPort: pointer.ToUint16(uint16(params.ListenPort)), ExporterOptions: &ExporterOptions{ PushMetrics: params.PushMetrics, - MetricsPath: pointer.ToStringOrNil(metricsPath), - MetricsScheme: pointer.ToStringOrNil(scheme), + MetricsPath: metricsPath, + MetricsScheme: scheme, }, } if err := row.SetCustomLabels(params.CustomLabels); err != nil { @@ -982,9 +982,7 @@ func ChangeAgent(q *reform.Querier, agentID string, params *ChangeCommonAgentPar } if params.EnablePushMetrics != nil { - row.ExporterOptions = &ExporterOptions{ - PushMetrics: *params.EnablePushMetrics, - } + row.ExporterOptions.PushMetrics = *params.EnablePushMetrics if row.AgentType == ExternalExporterType { if err := updateExternalExporterParams(q, row); err != nil { return nil, errors.Wrap(err, "failed to update External exporterParams for PushMetrics") diff --git a/managed/models/agent_helpers_test.go b/managed/models/agent_helpers_test.go index bff0c90775..b4706fee0e 100644 --- a/managed/models/agent_helpers_test.go +++ b/managed/models/agent_helpers_test.go @@ -546,8 +546,8 @@ func TestAgentHelpers(t *testing.T) { ServiceID: pointer.ToString("S1"), ListenPort: pointer.ToUint16(9104), ExporterOptions: &models.ExporterOptions{ - MetricsPath: pointer.ToString("/metrics"), - MetricsScheme: pointer.ToString("http"), + MetricsPath: "/metrics", + MetricsScheme: "http", }, CreatedAt: now, UpdatedAt: now, diff --git a/managed/models/agent_model.go b/managed/models/agent_model.go index 1c83cf0acc..ea3558ac52 100644 --- a/managed/models/agent_model.go +++ b/managed/models/agent_model.go @@ -84,8 +84,8 @@ type ExporterOptions struct { PushMetrics bool `json:"push_metrics"` DisabledCollectors pq.StringArray `json:"disabled_collectors"` MetricsResolutions *MetricsResolutions `json:"metrics_resolutions"` - MetricsPath *string `json:"metrics_path"` - MetricsScheme *string `json:"metrics_scheme"` + MetricsPath string `json:"metrics_path"` + MetricsScheme string `json:"metrics_scheme"` } // Value implements database/sql/driver.Valuer interface. Should be defined on the value. @@ -595,8 +595,8 @@ func (s *Agent) ExporterURL(q *reform.Querier) (string, error) { s.ExporterOptions = &ExporterOptions{} } - scheme := pointer.GetString(s.ExporterOptions.MetricsScheme) - path := pointer.GetString(s.ExporterOptions.MetricsPath) + scheme := s.ExporterOptions.MetricsScheme + path := s.ExporterOptions.MetricsPath listenPort := int(pointer.GetUint16(s.ListenPort)) username := pointer.GetString(s.Username) password := pointer.GetString(s.Password) @@ -717,7 +717,7 @@ func (s Agent) TemplateDelimiters(svc *Service) *DelimiterPair { pointer.GetString(svc.Address), pointer.GetString(s.Username), pointer.GetString(s.Password), - pointer.GetString(s.ExporterOptions.MetricsPath), + s.ExporterOptions.MetricsPath, } switch svc.ServiceType { diff --git a/managed/models/agent_model_test.go b/managed/models/agent_model_test.go index 65f71d6ff0..1b7ab39dbc 100644 --- a/managed/models/agent_model_test.go +++ b/managed/models/agent_model_test.go @@ -479,8 +479,8 @@ func TestExporterURL(t *testing.T) { ListenPort: pointer.ToUint16(9121), ExporterOptions: &models.ExporterOptions{ PushMetrics: true, - MetricsPath: pointer.ToString("/metrics"), - MetricsScheme: pointer.ToString("http"), + MetricsPath: "/metrics", + MetricsScheme: "http", }, }, @@ -494,8 +494,8 @@ func TestExporterURL(t *testing.T) { Password: pointer.ToString("secret"), ExporterOptions: &models.ExporterOptions{ PushMetrics: false, - MetricsPath: pointer.ToString("/metrics"), - MetricsScheme: pointer.ToString("http"), + MetricsPath: "/metrics", + MetricsScheme: "http", }, }, @@ -509,8 +509,8 @@ func TestExporterURL(t *testing.T) { Password: pointer.ToString("secret"), ExporterOptions: &models.ExporterOptions{ PushMetrics: false, - MetricsPath: pointer.ToString("/metrics"), - MetricsScheme: pointer.ToString("http"), + MetricsPath: "/metrics", + MetricsScheme: "http", }, }, @@ -524,8 +524,8 @@ func TestExporterURL(t *testing.T) { Password: pointer.ToString("secret"), ExporterOptions: &models.ExporterOptions{ PushMetrics: false, - MetricsPath: pointer.ToString("/metrics?format=prometheus&output=json"), - MetricsScheme: pointer.ToString("http"), + MetricsPath: "/metrics?format=prometheus&output=json", + MetricsScheme: "http", }, }, @@ -539,8 +539,8 @@ func TestExporterURL(t *testing.T) { Password: pointer.ToString("secret"), ExporterOptions: &models.ExporterOptions{ PushMetrics: false, - MetricsPath: pointer.ToString("/"), - MetricsScheme: pointer.ToString("http"), + MetricsPath: "/", + MetricsScheme: "http", }, }, } { diff --git a/managed/services/agents/mongodb.go b/managed/services/agents/mongodb.go index ff188d68cc..1fd19b8d60 100644 --- a/managed/services/agents/mongodb.go +++ b/managed/services/agents/mongodb.go @@ -21,8 +21,6 @@ import ( "strings" "time" - "github.com/AlekSi/pointer" - agentv1 "github.com/percona/pmm/api/agent/v1" inventoryv1 "github.com/percona/pmm/api/inventory/v1" "github.com/percona/pmm/managed/models" @@ -49,8 +47,8 @@ func mongodbExporterConfig(node *models.Node, service *models.Service, exporter args := getArgs(exporter, tdp, listenAddress, pmmAgentVersion) - if pointer.GetString(exporter.ExporterOptions.MetricsPath) != "" { - args = append(args, "--web.telemetry-path="+*exporter.ExporterOptions.MetricsPath) + if exporter.ExporterOptions.MetricsPath != "" { + args = append(args, "--web.telemetry-path="+exporter.ExporterOptions.MetricsPath) } args = withLogLevel(args, exporter.LogLevel, pmmAgentVersion, true) diff --git a/managed/services/agents/mysql.go b/managed/services/agents/mysql.go index 55ff651b13..349cfde831 100644 --- a/managed/services/agents/mysql.go +++ b/managed/services/agents/mysql.go @@ -106,8 +106,8 @@ func mysqldExporterConfig( args = collectors.FilterOutCollectors("--collect.", args, exporter.ExporterOptions.DisabledCollectors) - if exporter.ExporterOptions.MetricsPath != nil { - args = append(args, "--web.telemetry-path="+*exporter.ExporterOptions.MetricsPath) + if exporter.ExporterOptions.MetricsPath != "" { + args = append(args, "--web.telemetry-path="+exporter.ExporterOptions.MetricsPath) } files := exporter.Files() diff --git a/managed/services/agents/node.go b/managed/services/agents/node.go index 9fe3bb994d..b81c763ad2 100644 --- a/managed/services/agents/node.go +++ b/managed/services/agents/node.go @@ -18,8 +18,6 @@ package agents import ( "sort" - "github.com/AlekSi/pointer" - agentv1 "github.com/percona/pmm/api/agent/v1" inventoryv1 "github.com/percona/pmm/api/inventory/v1" "github.com/percona/pmm/managed/models" @@ -36,7 +34,7 @@ var ( func nodeExporterConfig(node *models.Node, exporter *models.Agent, agentVersion *version.Parsed) (*agentv1.SetStateRequest_AgentProcess, error) { listenAddress := getExporterListenAddress(node, exporter) - tdp := models.TemplateDelimsPair(pointer.GetString(exporter.ExporterOptions.MetricsPath)) + tdp := models.TemplateDelimsPair(exporter.ExporterOptions.MetricsPath) args := []string{ "--collector.textfile.directory.lr=" + pathsBase(agentVersion, tdp.Left, tdp.Right) + "/collectors/textfile-collector/low-resolution", "--collector.textfile.directory.mr=" + pathsBase(agentVersion, tdp.Left, tdp.Right) + "/collectors/textfile-collector/medium-resolution", @@ -125,8 +123,8 @@ func nodeExporterConfig(node *models.Node, exporter *models.Agent, agentVersion args = collectors.FilterOutCollectors("--collector.", args, exporter.ExporterOptions.DisabledCollectors) - if exporter.ExporterOptions.MetricsPath != nil { - args = append(args, "--web.telemetry-path="+*exporter.ExporterOptions.MetricsPath) + if exporter.ExporterOptions.MetricsPath != "" { + args = append(args, "--web.telemetry-path="+exporter.ExporterOptions.MetricsPath) } args = withLogLevel(args, exporter.LogLevel, agentVersion, false) diff --git a/managed/services/agents/postgresql.go b/managed/services/agents/postgresql.go index f20b8cfffc..37c78edc9d 100644 --- a/managed/services/agents/postgresql.go +++ b/managed/services/agents/postgresql.go @@ -109,8 +109,8 @@ func postgresExporterConfig(node *models.Node, service *models.Service, exporter args = append(args, "--max-connections="+strconv.Itoa(int(exporter.PostgreSQLOptions.MaxExporterConnections))) } - if exporter.ExporterOptions.MetricsPath != nil { - args = append(args, "--web.telemetry-path="+*exporter.ExporterOptions.MetricsPath) + if exporter.ExporterOptions.MetricsPath != "" { + args = append(args, "--web.telemetry-path="+exporter.ExporterOptions.MetricsPath) } args = collectors.FilterOutCollectors("--collect.", args, exporter.ExporterOptions.DisabledCollectors) diff --git a/managed/services/agents/proxysql.go b/managed/services/agents/proxysql.go index 4c36919ca8..c4b3664b02 100644 --- a/managed/services/agents/proxysql.go +++ b/managed/services/agents/proxysql.go @@ -59,8 +59,8 @@ func proxysqlExporterConfig(node *models.Node, service *models.Service, exporter exporter.ExporterOptions = &models.ExporterOptions{} } - if exporter.ExporterOptions.MetricsPath != nil { - args = append(args, "-web.telemetry-path="+*exporter.ExporterOptions.MetricsPath) + if exporter.ExporterOptions.MetricsPath != "" { + args = append(args, "-web.telemetry-path="+exporter.ExporterOptions.MetricsPath) } args = collectors.FilterOutCollectors("-collect.", args, exporter.ExporterOptions.DisabledCollectors) diff --git a/managed/services/converters.go b/managed/services/converters.go index e35814a53a..238b96ff3b 100644 --- a/managed/services/converters.go +++ b/managed/services/converters.go @@ -498,8 +498,8 @@ func ToAPIAgent(q *reform.Querier, agent *models.Agent) (inventoryv1.Agent, erro ServiceId: pointer.GetString(agent.ServiceID), Username: pointer.GetString(agent.Username), Disabled: agent.Disabled, - Scheme: pointer.GetString(agent.ExporterOptions.MetricsScheme), - MetricsPath: pointer.GetString(agent.ExporterOptions.MetricsPath), + Scheme: agent.ExporterOptions.MetricsScheme, + MetricsPath: agent.ExporterOptions.MetricsPath, ListenPort: uint32(pointer.GetUint16(agent.ListenPort)), CustomLabels: labels, PushMetricsEnabled: agent.ExporterOptions.PushMetrics, diff --git a/managed/services/management/agent.go b/managed/services/management/agent.go index 23f8f3bf0a..85077bcc67 100644 --- a/managed/services/management/agent.go +++ b/managed/services/management/agent.go @@ -148,8 +148,8 @@ func (s *ManagementService) agentToAPI(agent *models.Agent) (*managementv1.Unive if agent.ExporterOptions != nil { ua.DisabledCollectors = agent.ExporterOptions.DisabledCollectors - ua.MetricsPath = pointer.GetString(agent.ExporterOptions.MetricsPath) - ua.MetricsScheme = pointer.GetString(agent.ExporterOptions.MetricsScheme) + ua.MetricsPath = agent.ExporterOptions.MetricsPath + ua.MetricsScheme = agent.ExporterOptions.MetricsScheme ua.PushMetrics = agent.ExporterOptions.PushMetrics ua.ExposeExporter = agent.ExporterOptions.ExposeExporter } diff --git a/managed/services/victoriametrics/scrape_configs.go b/managed/services/victoriametrics/scrape_configs.go index 9ff886d673..fb74df5f16 100644 --- a/managed/services/victoriametrics/scrape_configs.go +++ b/managed/services/victoriametrics/scrape_configs.go @@ -560,8 +560,8 @@ func scrapeConfigsForExternalExporter(s *models.MetricsResolutions, params *scra JobName: jobName(params.agent, "mr"), ScrapeInterval: config.Duration(interval), ScrapeTimeout: scrapeTimeout(interval), - Scheme: pointer.GetString(params.agent.ExporterOptions.MetricsScheme), - MetricsPath: pointer.GetString(params.agent.ExporterOptions.MetricsPath), + Scheme: params.agent.ExporterOptions.MetricsScheme, + MetricsPath: params.agent.ExporterOptions.MetricsPath, } if pointer.GetString(params.agent.Username) != "" { @@ -601,8 +601,8 @@ func scrapeConfigsForVMAgent(s *models.MetricsResolutions, params *scrapeConfigP JobName: jobName(params.agent, "mr"), ScrapeInterval: config.Duration(interval), ScrapeTimeout: scrapeTimeout(interval), - Scheme: pointer.GetString(params.agent.ExporterOptions.MetricsScheme), - MetricsPath: pointer.GetString(params.agent.ExporterOptions.MetricsPath), + Scheme: params.agent.ExporterOptions.MetricsScheme, + MetricsPath: params.agent.ExporterOptions.MetricsPath, } if pointer.GetString(params.agent.Username) != "" { diff --git a/managed/services/victoriametrics/scrape_configs_test.go b/managed/services/victoriametrics/scrape_configs_test.go index 711fde6029..47bacb37d3 100644 --- a/managed/services/victoriametrics/scrape_configs_test.go +++ b/managed/services/victoriametrics/scrape_configs_test.go @@ -1234,8 +1234,8 @@ func TestScrapeConfig(t *testing.T) { Password: pointer.ToString("password"), ListenPort: pointer.ToUint16(12345), ExporterOptions: &models.ExporterOptions{ - MetricsPath: pointer.ToString("/some-metric-path"), - MetricsScheme: pointer.ToString("https"), + MetricsPath: "/some-metric-path", + MetricsScheme: "https", }, } From f554014f141b8a23ace1191e8e2541a424d81f30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Thu, 5 Dec 2024 11:16:17 +0100 Subject: [PATCH 57/60] PMM-5086-12634 Another cleanup. --- managed/models/database.go | 26 +------------------------- managed/services/management/rds.go | 1 - 2 files changed, 1 insertion(+), 26 deletions(-) diff --git a/managed/models/database.go b/managed/models/database.go index 977924ced5..9a811da797 100644 --- a/managed/models/database.go +++ b/managed/models/database.go @@ -78,25 +78,6 @@ var DefaultAgentEncryptionColumnsV3 = []encryption.Table{ }, } -// DefaultAgentEncryptionColumns contains all tables and it's columns to be encrypted in PMM Server DB. -var DefaultAgentEncryptionColumns = []encryption.Table{ - { - Name: "agents", - Identifiers: []string{"agent_id"}, - Columns: []encryption.Column{ - {Name: "username"}, - {Name: "password"}, - {Name: "agent_password"}, - {Name: "aws_access_key"}, - {Name: "aws_secret_key"}, - {Name: "azure_options", CustomHandler: EncryptAzureOptionsHandler}, - {Name: "mongo_db_tls_options", CustomHandler: EncryptMongoDBOptionsHandler}, - {Name: "mysql_options", CustomHandler: EncryptMySQLOptionsHandler}, - {Name: "postgresql_options", CustomHandler: EncryptPostgreSQLOptionsHandler}, - }, - }, -} - // databaseSchema maps schema version from schema_migrations table (id column) to a slice of DDL queries. var databaseSchema = [][]string{ 1: { @@ -1428,12 +1409,7 @@ func migrateDB(db *reform.DB, params SetupDBParams) error { } } - itemsToEncrypt := DefaultAgentEncryptionColumnsV3 - if params.MigrationVersion != nil && currentVersion < 107 { - itemsToEncrypt = DefaultAgentEncryptionColumns - } - - err := EncryptDB(tx, params.Name, itemsToEncrypt) + err := EncryptDB(tx, params.Name, DefaultAgentEncryptionColumnsV3) if err != nil { return err } diff --git a/managed/services/management/rds.go b/managed/services/management/rds.go index 1020b9f72f..6e3edfea89 100644 --- a/managed/services/management/rds.go +++ b/managed/services/management/rds.go @@ -410,7 +410,6 @@ func (s *ManagementService) addRDS(ctx context.Context, req *managementv1.AddRDS ExporterOptions: &models.ExporterOptions{ PushMetrics: isPushMode(metricsMode), }, - // TODO? PostgresExporter but MySQL Option? MySQLOptions: &models.MySQLOptions{ TableCountTablestatsGroupLimit: tablestatsGroupTableLimit, }, From 71a7925226027e5a1c23027f8762195fabd58697 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Thu, 5 Dec 2024 11:41:07 +0100 Subject: [PATCH 58/60] PMM-5086-12634 Remove migration test to avoid two encryption models. --- managed/models/database_test.go | 72 --------------------------------- 1 file changed, 72 deletions(-) diff --git a/managed/models/database_test.go b/managed/models/database_test.go index 1a6e790923..42175ac305 100644 --- a/managed/models/database_test.go +++ b/managed/models/database_test.go @@ -17,12 +17,10 @@ package models_test import ( - "context" "database/sql" "fmt" "testing" - "github.com/AlekSi/pointer" "github.com/lib/pq" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -321,73 +319,3 @@ func TestDatabaseChecks(t *testing.T) { }) }) } - -func TestDatabaseMigrations(t *testing.T) { - t.Run("stats_collections field migration: string to string array", func(t *testing.T) { - sqlDB := testdb.Open(t, models.SkipFixtures, pointer.ToInt(57)) - defer sqlDB.Close() //nolint:errcheck - - // Insert dummy node in DB - _, err := sqlDB.ExecContext(context.Background(), - `INSERT INTO - nodes(node_id, node_type, node_name, distro, node_model, az, address, created_at, updated_at) - VALUES - ('node_id', 'node_type', 'node_name', 'distro', 'node_model', 'az', 'address', '03/03/2014 02:03:04', '03/03/2014 02:03:04')`, - ) - require.NoError(t, err) - - // Insert dummy agent in DB - _, err = sqlDB.ExecContext(context.Background(), - `INSERT INTO - agents(mongo_db_tls_options, agent_id, agent_type, runs_on_node_id, created_at, updated_at, disabled, status, tls, tls_skip_verify, query_examples_disabled, max_query_log_size, table_count_tablestats_group_limit, rds_basic_metrics_disabled, rds_enhanced_metrics_disabled, push_metrics) - VALUES - ('{"stats_collections": "db.col1,db.col2,db.col3"}', 'id', 'pmm-agent', 'node_id' , '03/03/2014 02:03:04', '03/03/2014 02:03:04', false, 'alive', false, false, false, 0, 0, false, false, false)`, - ) - require.NoError(t, err) - - // Apply migration - testdb.SetupDB(t, sqlDB, models.SkipFixtures, pointer.ToInt(68)) - - var agentID string - var mongoDBOptions *models.MongoDBOptions - err = sqlDB.QueryRow(`SELECT agent_id, mongo_db_tls_options FROM agents WHERE agent_id = $1`, "id").Scan(&agentID, &mongoDBOptions) - - require.NoError(t, err) - require.Equal(t, "id", agentID) - require.Equal(t, []string{"db.col1", "db.col2", "db.col3"}, mongoDBOptions.StatsCollections) - }) - - t.Run("stats_collections field migration: string array to string array", func(t *testing.T) { - sqlDB := testdb.Open(t, models.SkipFixtures, pointer.ToInt(58)) - defer sqlDB.Close() //nolint:errcheck - - // Insert dummy node in DB - _, err := sqlDB.ExecContext(context.Background(), - `INSERT INTO - nodes(node_id, node_type, node_name, distro, node_model, az, address, created_at, updated_at) - VALUES - ('node_id', 'generic', 'node_name', 'distro', 'node_model', 'az', 'address', '03/03/2014 02:03:04', '03/03/2014 02:03:04')`, - ) - require.NoError(t, err) - - // Insert dummy agent in DB - _, err = sqlDB.ExecContext(context.Background(), - `INSERT INTO - agents(mongo_db_tls_options, agent_id, agent_type, runs_on_node_id, created_at, updated_at, disabled, status, tls, tls_skip_verify, query_examples_disabled, max_query_log_size, table_count_tablestats_group_limit, rds_basic_metrics_disabled, rds_enhanced_metrics_disabled, push_metrics) - VALUES - ('{"stats_collections": ["db.col1", "db.col2", "db.col3"]}', 'id', 'pmm-agent', 'node_id' , '03/03/2014 02:03:04', '03/03/2014 02:03:04', false, 'alive', false, false, false, 0, 0, false, false, false)`, - ) - require.NoError(t, err) - - // Apply migration - testdb.SetupDB(t, sqlDB, models.SkipFixtures, pointer.ToInt(68)) - - var agentID string - var mongoDBOptions *models.MongoDBOptions - err = sqlDB.QueryRow(`SELECT agent_id, mongo_db_tls_options FROM agents WHERE agent_id = $1`, "id").Scan(&agentID, &mongoDBOptions) - - require.NoError(t, err) - require.Equal(t, "id", agentID) - require.Equal(t, []string{"db.col1", "db.col2", "db.col3"}, mongoDBOptions.StatsCollections) - }) -} From 79b011681860a1b9634e689ca3a6518b7c8e6d29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Thu, 5 Dec 2024 12:02:36 +0100 Subject: [PATCH 59/60] PMM-5086-12634 Remove options assignment in case of nil option. --- managed/models/agent_model.go | 4 -- managed/services/agents/proxysql.go | 4 -- .../services/agents/service_info_broker.go | 4 -- managed/services/converters.go | 44 -------------- .../services/victoriametrics/prometheus.go | 3 - .../victoriametrics/scrape_configs.go | 24 -------- .../victoriametrics/scrape_configs_test.go | 58 +++++++++++-------- 7 files changed, 33 insertions(+), 108 deletions(-) diff --git a/managed/models/agent_model.go b/managed/models/agent_model.go index ea3558ac52..5cc5b70ca8 100644 --- a/managed/models/agent_model.go +++ b/managed/models/agent_model.go @@ -591,10 +591,6 @@ func (s *Agent) DSN(service *Service, dsnParams DSNParams, tdp *DelimiterPair, p // ExporterURL composes URL to an external exporter. func (s *Agent) ExporterURL(q *reform.Querier) (string, error) { - if s.ExporterOptions == nil { - s.ExporterOptions = &ExporterOptions{} - } - scheme := s.ExporterOptions.MetricsScheme path := s.ExporterOptions.MetricsPath listenPort := int(pointer.GetUint16(s.ListenPort)) diff --git a/managed/services/agents/proxysql.go b/managed/services/agents/proxysql.go index c4b3664b02..8f786c25e1 100644 --- a/managed/services/agents/proxysql.go +++ b/managed/services/agents/proxysql.go @@ -55,10 +55,6 @@ func proxysqlExporterConfig(node *models.Node, service *models.Service, exporter args = append(args, "-collect.runtime_mysql_servers") } - if exporter.ExporterOptions == nil { - exporter.ExporterOptions = &models.ExporterOptions{} - } - if exporter.ExporterOptions.MetricsPath != "" { args = append(args, "-web.telemetry-path="+exporter.ExporterOptions.MetricsPath) } diff --git a/managed/services/agents/service_info_broker.go b/managed/services/agents/service_info_broker.go index 420615401a..5927fb20ce 100644 --- a/managed/services/agents/service_info_broker.go +++ b/managed/services/agents/service_info_broker.go @@ -189,10 +189,6 @@ func (c *ServiceInfoBroker) GetInfoFromService(ctx context.Context, q *reform.Qu return updateServiceVersion(ctx, q, resp, service) case models.PostgreSQLServiceType: - if agent.PostgreSQLOptions == nil { - agent.PostgreSQLOptions = &models.PostgreSQLOptions{} - } - databaseList := sInfo.DatabaseList databaseCount := len(databaseList) excludedDatabaseList := postgresExcludedDatabases() diff --git a/managed/services/converters.go b/managed/services/converters.go index 238b96ff3b..a83be2abf8 100644 --- a/managed/services/converters.go +++ b/managed/services/converters.go @@ -219,10 +219,6 @@ func ToAPIAgent(q *reform.Querier, agent *models.Agent) (inventoryv1.Agent, erro serviceID = service.ServiceID } - if agent.ExporterOptions == nil { - agent.ExporterOptions = &models.ExporterOptions{} - } - processExecPath := pointer.GetString(agent.ProcessExecPath) switch agent.AgentType { case models.PMMAgentType: @@ -250,10 +246,6 @@ func ToAPIAgent(q *reform.Querier, agent *models.Agent) (inventoryv1.Agent, erro }, nil case models.MySQLdExporterType: - if agent.MySQLOptions == nil { - agent.MySQLOptions = &models.MySQLOptions{} - } - return &inventoryv1.MySQLdExporter{ AgentId: agent.AgentID, PmmAgentId: pointer.GetString(agent.PMMAgentID), @@ -277,10 +269,6 @@ func ToAPIAgent(q *reform.Querier, agent *models.Agent) (inventoryv1.Agent, erro }, nil case models.MongoDBExporterType: - if agent.MongoDBOptions == nil { - agent.MongoDBOptions = &models.MongoDBOptions{} - } - exporter := &inventoryv1.MongoDBExporter{ AgentId: agent.AgentID, PmmAgentId: pointer.GetString(agent.PMMAgentID), @@ -307,10 +295,6 @@ func ToAPIAgent(q *reform.Querier, agent *models.Agent) (inventoryv1.Agent, erro return exporter, nil case models.PostgresExporterType: - if agent.PostgreSQLOptions == nil { - agent.PostgreSQLOptions = &models.PostgreSQLOptions{} - } - exporter := &inventoryv1.PostgresExporter{ AgentId: agent.AgentID, PmmAgentId: pointer.GetString(agent.PMMAgentID), @@ -335,10 +319,6 @@ func ToAPIAgent(q *reform.Querier, agent *models.Agent) (inventoryv1.Agent, erro return exporter, nil case models.QANMySQLPerfSchemaAgentType: - if agent.QANOptions == nil { - agent.QANOptions = &models.QANOptions{} - } - return &inventoryv1.QANMySQLPerfSchemaAgent{ AgentId: agent.AgentID, PmmAgentId: pointer.GetString(agent.PMMAgentID), @@ -357,10 +337,6 @@ func ToAPIAgent(q *reform.Querier, agent *models.Agent) (inventoryv1.Agent, erro }, nil case models.QANMySQLSlowlogAgentType: - if agent.QANOptions == nil { - agent.QANOptions = &models.QANOptions{} - } - return &inventoryv1.QANMySQLSlowlogAgent{ AgentId: agent.AgentID, PmmAgentId: pointer.GetString(agent.PMMAgentID), @@ -379,10 +355,6 @@ func ToAPIAgent(q *reform.Querier, agent *models.Agent) (inventoryv1.Agent, erro }, nil case models.QANMongoDBProfilerAgentType: - if agent.QANOptions == nil { - agent.QANOptions = &models.QANOptions{} - } - return &inventoryv1.QANMongoDBProfilerAgent{ AgentId: agent.AgentID, PmmAgentId: pointer.GetString(agent.PMMAgentID), @@ -420,10 +392,6 @@ func ToAPIAgent(q *reform.Querier, agent *models.Agent) (inventoryv1.Agent, erro }, nil case models.QANPostgreSQLPgStatementsAgentType: - if agent.QANOptions == nil { - agent.QANOptions = &models.QANOptions{} - } - return &inventoryv1.QANPostgreSQLPgStatementsAgent{ AgentId: agent.AgentID, PmmAgentId: pointer.GetString(agent.PMMAgentID), @@ -441,10 +409,6 @@ func ToAPIAgent(q *reform.Querier, agent *models.Agent) (inventoryv1.Agent, erro }, nil case models.QANPostgreSQLPgStatMonitorAgentType: - if agent.QANOptions == nil { - agent.QANOptions = &models.QANOptions{} - } - return &inventoryv1.QANPostgreSQLPgStatMonitorAgent{ AgentId: agent.AgentID, PmmAgentId: pointer.GetString(agent.PMMAgentID), @@ -463,10 +427,6 @@ func ToAPIAgent(q *reform.Querier, agent *models.Agent) (inventoryv1.Agent, erro }, nil case models.RDSExporterType: - if agent.AWSOptions == nil { - agent.AWSOptions = &models.AWSOptions{} - } - return &inventoryv1.RDSExporter{ AgentId: agent.AgentID, PmmAgentId: pointer.GetString(agent.PMMAgentID), @@ -508,10 +468,6 @@ func ToAPIAgent(q *reform.Querier, agent *models.Agent) (inventoryv1.Agent, erro }, nil case models.AzureDatabaseExporterType: - if agent.AzureOptions == nil { - agent.AzureOptions = &models.AzureOptions{} - } - return &inventoryv1.AzureDatabaseExporter{ AgentId: agent.AgentID, PmmAgentId: pointer.GetString(agent.PMMAgentID), diff --git a/managed/services/victoriametrics/prometheus.go b/managed/services/victoriametrics/prometheus.go index 7bde451e07..a4f957cc6c 100644 --- a/managed/services/victoriametrics/prometheus.go +++ b/managed/services/victoriametrics/prometheus.go @@ -108,9 +108,6 @@ func AddScrapeConfigs(l *logrus.Entry, cfg *config.Config, q *reform.Querier, // } mr := *globalResolutions // copy global resolutions - if agent.ExporterOptions == nil { - agent.ExporterOptions = &models.ExporterOptions{} - } if agent.ExporterOptions.MetricsResolutions != nil { if agent.ExporterOptions.MetricsResolutions.MR != 0 { mr.MR = agent.ExporterOptions.MetricsResolutions.MR diff --git a/managed/services/victoriametrics/scrape_configs.go b/managed/services/victoriametrics/scrape_configs.go index fb74df5f16..16504177ef 100644 --- a/managed/services/victoriametrics/scrape_configs.go +++ b/managed/services/victoriametrics/scrape_configs.go @@ -209,10 +209,6 @@ func scrapeConfigsForNodeExporter(params *scrapeConfigParams) ([]*config.ScrapeC var err error var hrCollect []string - if params.agent.ExporterOptions == nil { - params.agent.ExporterOptions = &models.ExporterOptions{} - } - if params.node.Distro != "darwin" { mrCollect := []string{ "hwmon", @@ -290,10 +286,6 @@ func scrapeConfigsForMySQLdExporter(params *scrapeConfigParams) ([]*config.Scrap "standard.process", } - if params.agent.ExporterOptions == nil { - params.agent.ExporterOptions = &models.ExporterOptions{} - } - hrOptions = collectors.FilterOutCollectors("", hrOptions, params.agent.ExporterOptions.DisabledCollectors) hr, err := scrapeConfigForStandardExporter("hr", params.metricsResolution.HR, params, hrOptions) @@ -365,10 +357,6 @@ func scrapeConfigsForMySQLdExporter(params *scrapeConfigParams) ([]*config.Scrap } func scrapeConfigsForMongoDBExporter(params *scrapeConfigParams) ([]*config.ScrapeConfig, error) { - if params.agent.ExporterOptions == nil { - params.agent.ExporterOptions = &models.ExporterOptions{} - } - // Old pmm-agents doesn't have support of multiple resolution, // so requesting mongodb_exporter metrics in two resolutions increases CPU and Memory usage. if params.pmmAgentVersion == nil || params.pmmAgentVersion.Less(version.MustParse("2.26.0-0")) { @@ -439,10 +427,6 @@ func scrapeConfigsForPostgresExporter(params *scrapeConfigParams) ([]*config.Scr "postgres", } - if params.agent.ExporterOptions == nil { - params.agent.ExporterOptions = &models.ExporterOptions{} - } - hrOptions = collectors.FilterOutCollectors("", hrOptions, params.agent.ExporterOptions.DisabledCollectors) hr, err := scrapeConfigForStandardExporter("hr", params.metricsResolution.HR, params, hrOptions) if err != nil { @@ -552,10 +536,6 @@ func scrapeConfigsForExternalExporter(s *models.MetricsResolutions, params *scra } interval := s.MR - if params.agent.ExporterOptions == nil { - params.agent.ExporterOptions = &models.ExporterOptions{} - } - cfg := &config.ScrapeConfig{ JobName: jobName(params.agent, "mr"), ScrapeInterval: config.Duration(interval), @@ -593,10 +573,6 @@ func scrapeConfigsForVMAgent(s *models.MetricsResolutions, params *scrapeConfigP } interval := s.MR - if params.agent.ExporterOptions == nil { - params.agent.ExporterOptions = &models.ExporterOptions{} - } - cfg := &config.ScrapeConfig{ JobName: jobName(params.agent, "mr"), ScrapeInterval: config.Duration(interval), diff --git a/managed/services/victoriametrics/scrape_configs_test.go b/managed/services/victoriametrics/scrape_configs_test.go index 47bacb37d3..28ad444d38 100644 --- a/managed/services/victoriametrics/scrape_configs_test.go +++ b/managed/services/victoriametrics/scrape_configs_test.go @@ -257,11 +257,12 @@ func TestScrapeConfig(t *testing.T) { CustomLabels: []byte(`{"_some_service_label": "bar"}`), } agent := &models.Agent{ - AgentID: "75bb30d3-ef4a-4147-97a8-621a996611dd", - AgentType: models.MySQLdExporterType, - CustomLabels: []byte(`{"_some_agent_label": "baz"}`), - ListenPort: pointer.ToUint16(12345), - MySQLOptions: &models.MySQLOptions{}, + AgentID: "75bb30d3-ef4a-4147-97a8-621a996611dd", + AgentType: models.MySQLdExporterType, + CustomLabels: []byte(`{"_some_agent_label": "baz"}`), + ListenPort: pointer.ToUint16(12345), + ExporterOptions: &models.ExporterOptions{}, + MySQLOptions: &models.MySQLOptions{}, } expected := []*config.ScrapeConfig{{ @@ -563,9 +564,10 @@ func TestScrapeConfig(t *testing.T) { Address: pointer.ToString("5.6.7.8"), } agent := &models.Agent{ - AgentID: "75bb30d3-ef4a-4147-97a8-621a996611dd", - AgentType: models.MySQLdExporterType, - ListenPort: pointer.ToUint16(12345), + AgentID: "75bb30d3-ef4a-4147-97a8-621a996611dd", + AgentType: models.MySQLdExporterType, + ListenPort: pointer.ToUint16(12345), + ExporterOptions: &models.ExporterOptions{}, MySQLOptions: &models.MySQLOptions{ TableCount: pointer.ToInt32(100500), TableCountTablestatsGroupLimit: 1000, @@ -694,8 +696,9 @@ func TestScrapeConfig(t *testing.T) { node := &models.Node{} service := &models.Service{} agent := &models.Agent{ - CustomLabels: []byte("{"), - ListenPort: pointer.ToUint16(12345), + CustomLabels: []byte("{"), + ListenPort: pointer.ToUint16(12345), + ExporterOptions: &models.ExporterOptions{}, } _, err := scrapeConfigsForMySQLdExporter(&scrapeConfigParams{ @@ -724,11 +727,12 @@ func TestScrapeConfig(t *testing.T) { CustomLabels: []byte(`{"_some_service_label": "bar"}`), } agent := &models.Agent{ - AgentID: "75bb30d3-ef4a-4147-97a8-621a996611dd", - AgentType: models.MongoDBExporterType, - CustomLabels: []byte(`{"_some_agent_label": "baz"}`), - ListenPort: pointer.ToUint16(12345), - MongoDBOptions: &models.MongoDBOptions{EnableAllCollectors: true}, + AgentID: "75bb30d3-ef4a-4147-97a8-621a996611dd", + AgentType: models.MongoDBExporterType, + CustomLabels: []byte(`{"_some_agent_label": "baz"}`), + ListenPort: pointer.ToUint16(12345), + ExporterOptions: &models.ExporterOptions{}, + MongoDBOptions: &models.MongoDBOptions{EnableAllCollectors: true}, } expected := []*config.ScrapeConfig{ @@ -814,8 +818,9 @@ func TestScrapeConfig(t *testing.T) { node := &models.Node{} service := &models.Service{} agent := &models.Agent{ - CustomLabels: []byte("{"), - ListenPort: pointer.ToUint16(12345), + CustomLabels: []byte("{"), + ListenPort: pointer.ToUint16(12345), + ExporterOptions: &models.ExporterOptions{}, } _, err := scrapeConfigsForMongoDBExporter(&scrapeConfigParams{ @@ -965,8 +970,9 @@ func TestScrapeConfig(t *testing.T) { node := &models.Node{} service := &models.Service{} agent := &models.Agent{ - CustomLabels: []byte("{"), - ListenPort: pointer.ToUint16(12345), + CustomLabels: []byte("{"), + ListenPort: pointer.ToUint16(12345), + ExporterOptions: &models.ExporterOptions{}, } _, err := scrapeConfigsForPostgresExporter(&scrapeConfigParams{ @@ -1183,10 +1189,11 @@ func TestScrapeConfig(t *testing.T) { } t.Run("Normal", func(t *testing.T) { agent := &models.Agent{ - AgentID: "75bb30d3-ef4a-4147-97a8-621a996611dd", - AgentType: models.ExternalExporterType, - CustomLabels: []byte(`{"_some_agent_label": "baz"}`), - ListenPort: pointer.ToUint16(12345), + AgentID: "75bb30d3-ef4a-4147-97a8-621a996611dd", + AgentType: models.ExternalExporterType, + CustomLabels: []byte(`{"_some_agent_label": "baz"}`), + ListenPort: pointer.ToUint16(12345), + ExporterOptions: &models.ExporterOptions{}, } expected := []*config.ScrapeConfig{{ @@ -1285,8 +1292,9 @@ func TestScrapeConfig(t *testing.T) { t.Run("BadCustomLabels", func(t *testing.T) { agent := &models.Agent{ - CustomLabels: []byte("{"), - ListenPort: pointer.ToUint16(12345), + CustomLabels: []byte("{"), + ListenPort: pointer.ToUint16(12345), + ExporterOptions: &models.ExporterOptions{}, } _, err := scrapeConfigsForMongoDBExporter(&scrapeConfigParams{ From 03ac18e59b88d1ad2975c255a49319a4e192c305 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Tue, 10 Dec 2024 15:42:52 +0100 Subject: [PATCH 60/60] PMM-5086-12634 Remove the forgotten comments. --- managed/models/database.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/managed/models/database.go b/managed/models/database.go index 9a811da797..7126979431 100644 --- a/managed/models/database.go +++ b/managed/models/database.go @@ -1094,7 +1094,6 @@ var databaseSchema = [][]string{ `UPDATE agents SET postgresql_options = '{}'::jsonb WHERE postgresql_options IS NULL`, - // migrate values into new fields in options and drop original columns `UPDATE agents SET exporter_options = jsonb_set(exporter_options, '{expose_exporter}', to_jsonb(expose_exporter));`, `UPDATE agents SET exporter_options = jsonb_set(exporter_options, '{push_metrics}', to_jsonb(push_metrics));`, `UPDATE agents SET exporter_options = jsonb_set(exporter_options, '{disabled_collectors}', to_jsonb(disabled_collectors));`, @@ -1131,7 +1130,6 @@ var databaseSchema = [][]string{ `ALTER TABLE agents DROP COLUMN table_count`, `ALTER TABLE agents DROP COLUMN table_count_tablestats_group_limit`, - // update settings -> encrypted items!! `UPDATE settings SET settings = jsonb_set(settings, '{encrypted_items}', '[ "pmm-managed.agents.username",