From ada5e706b012a0703366fb6410ef06da71bbbc43 Mon Sep 17 00:00:00 2001 From: William Deveaux Date: Thu, 4 Aug 2022 13:52:42 +0200 Subject: [PATCH 01/28] implements metric system using armon gometrics lib implements the metric system following the implementation done in Consul and trying to implement the same metrics as done in the previous metric system (c.f. PR1378: https://github.com/hashicorp/consul-template/pull/1378/files ) --- README.md | 66 ++++++ cli.go | 8 + config/config.go | 20 ++ config/convert.go | 67 ++++++ config/syslog.go | 2 +- config/telemetry.go | 402 ++++++++++++++++++++++++++++++++++++ config/telemetry_test.go | 25 +++ dependency/dependency.go | 8 + dependency/vault_common.go | 5 + go.mod | 16 +- go.sum | 15 +- manager/runner.go | 50 ++++- telemetry/circonus.go | 54 +++++ telemetry/dogstatsd.go | 25 +++ telemetry/metrics.go | 52 +++++ telemetry/prometheus.go | 52 +++++ telemetry/statsd.go | 19 ++ telemetry/statsite.go | 19 ++ telemetry/telemetry.go | 102 +++++++++ telemetry/telemetry_test.go | 127 ++++++++++++ test/helpers.go | 15 +- 21 files changed, 1139 insertions(+), 10 deletions(-) create mode 100644 config/telemetry.go create mode 100644 config/telemetry_test.go create mode 100644 telemetry/circonus.go create mode 100644 telemetry/dogstatsd.go create mode 100644 telemetry/metrics.go create mode 100644 telemetry/prometheus.go create mode 100644 telemetry/statsd.go create mode 100644 telemetry/statsite.go create mode 100644 telemetry/telemetry.go create mode 100644 telemetry/telemetry_test.go diff --git a/README.md b/README.md index 8efe66d58..58dba1880 100644 --- a/README.md +++ b/README.md @@ -55,6 +55,7 @@ this functionality might prove useful. - [Multiple Commands](#multiple-commands) - [Multi-phase Execution](#multi-phase-execution) - [Debugging](#debugging) +- [Telemetry](#telemetry) - [FAQ](#faq) - [Contributing](#contributing) @@ -407,6 +408,71 @@ $ consul-template -log-level debug ... # ... ``` +## Telemetry + +Consul Template uses the [armon/go-metrics](https://github.com/armon/go-metrics) library to implement the Consul Template metric system. It currently supports metrics exported to circonus API, statsd server, statsite server, dogstatsd server, and prometheus endpoint. + +### Key Metrics + +These metrics offer insight into Consul Template and capture subprocess activities. The number of dependencies are aggregated from the configured templates, and metrics are collected around a dependency when it is updated from source. This is useful to correlate any upstream changes to downstream actions originating from Consul Template. + +Metrics are monitored around template rendering and execution of template commands. These +metrics indicate the rendering status of a template and how long commands for a template takes +to provide insight on performance of the templates. + +| Metric Name | Labels | Description | +|-|:-:|-| +| `consul-template.dependencies_received` | type=(consul\|vault\|local), id=dependencyString | A counter of dependencies received from monitoring value changes | +| `consul-template.templates_rendered` | id=templateID, status=(rendered\|would\|quiescence) | A counter of templates rendered | +| `consul-template.runner_actions` | action=(start\|stop\|run) | A count of runner actions | +| `consul-template.commands_exec` | status=(success\|error) | The number of commands executed after rendering templates | + +#### Metrics yet to be implemented + +The current metrics were implemented by takin as reference the [previous metric-related PR](https://github.com/hashicorp/consul-template/pull/1378/files#diff-d980d9aed26114a3414812b58d45770a201c1f29b7f67ddc0ef0891a8f1b7736), but as the `armon/go-metrics` library doesn't implement all types of metrics yet, histogram metrics could not be implemented. + +Said metrics are described as below: + +| Metric Name | Labels | Description | +|-|:-:|-| +| `consul-template.dependencies` | type=(consul\|vault\|local) | The number of dependencies grouped by types | +| `consul-template.templates` | | The number of templates configured | +| `consul-template.commands_exec_time` | id=tmplDestination | The execution time (seconds) of a template command | + + +### Metric Samples + +#### DogStatsD + +``` +2020-05-05 11:57:46.143979 consul-template.runner_actions:1|c|#action:start +consul-template.runner_actions:2|c|#action:run +consul-template.dependencies_received:1|c|#id:kv.block(hello),type:consul +consul-template.templates_rendered:1|c|#id:aadcafd7f28f1d9fc5e76ab2e029f844,status:rendered +consul-template.commands_exec:1|c|#status:success +consul-template.commands_exec:0|c|#status:error +``` + +#### Prometheus + +``` +$ curl localhost:8888/metrics +# HELP consul_template_commands_exec The number of commands executed with labels status=(success|error) +# TYPE consul_template_commands_exec counter +consul_template_commands_exec{status="error"} 0 +consul_template_commands_exec{status="success"} 1 +# HELP consul_template_dependencies_received A counter of dependencies received with labels type=(consul|vault|local) and id=dependencyString +# TYPE consul_template_dependencies_received counter +consul_template_dependencies_received{id="kv.block(hello)",type="consul"} 1 +# HELP consul_template_runner_actions A count of runner actions with labels action=(start|stop|run) +# TYPE consul_template_runner_actions counter +consul_template_runner_actions{action="run"} 2 +consul_template_runner_actions{action="start"} 1 +# HELP consul_template_templates_rendered A counter of templates rendered with labels id=templateID and status=(rendered|would|quiescence) +# TYPE consul_template_templates_rendered counter +consul_template_templates_rendered{id="aadcafd7f28f1d9fc5e76ab2e029f844",status="rendered"} 1 +``` + ## FAQ **Q: How is this different than confd?**
diff --git a/cli.go b/cli.go index 8aeb3692e..1b15fe859 100644 --- a/cli.go +++ b/cli.go @@ -20,6 +20,7 @@ import ( "github.com/hashicorp/consul-template/manager" "github.com/hashicorp/consul-template/service_os" "github.com/hashicorp/consul-template/signals" + "github.com/hashicorp/consul-template/telemetry" "github.com/hashicorp/consul-template/version" ) @@ -132,6 +133,13 @@ func (cli *CLI) Run(args []string) int { } }() + // Initialize telemetry + tel, err := telemetry.Init(config.Telemetry) + if err != nil { + return logError(err, ExitCodeConfigError) + } + defer tel.Stop() + // Initial runner runner, err := manager.NewRunner(config, dry) if err != nil { diff --git a/config/config.go b/config/config.go index c8109a7da..b8d9f9ae2 100644 --- a/config/config.go +++ b/config/config.go @@ -82,6 +82,9 @@ type Config struct { // Syslog is the configuration for syslog. Syslog *SyslogConfig `mapstructure:"syslog"` + // Telemetry is the configuration for collecting and emitting telemetry. + Telemetry *TelemetryConfig `mapstructure:"telemetry"` + // Templates is the list of templates. Templates *TemplateConfigs `mapstructure:"template"` @@ -174,6 +177,10 @@ func (c *Config) Copy() *Config { o.Syslog = c.Syslog.Copy() } + if c.Telemetry != nil { + o.Telemetry = c.Telemetry.Copy() + } + if c.Templates != nil { o.Templates = c.Templates.Copy() } @@ -265,6 +272,10 @@ func (c *Config) Merge(o *Config) *Config { r.Syslog = r.Syslog.Merge(o.Syslog) } + if o.Telemetry != nil { + r.Telemetry = r.Telemetry.Merge(o.Telemetry) + } + if o.Templates != nil { r.Templates = r.Templates.Merge(o.Templates) } @@ -336,6 +347,7 @@ func Parse(s string) (*Config, error) { "nomad.transport", "ssl", "syslog", + "telemetry", "vault", "vault.retry", "vault.ssl", @@ -494,6 +506,7 @@ func (c *Config) GoString() string { "ReloadSignal:%s, "+ "FileLog:%#v, "+ "Syslog:%#v, "+ + "Telemetry:%#v, "+ "Templates:%#v, "+ "TemplateErrFatal:%#v"+ "Vault:%#v, "+ @@ -513,6 +526,7 @@ func (c *Config) GoString() string { SignalGoString(c.ReloadSignal), c.FileLog, c.Syslog, + c.Telemetry.GoString(), c.Templates, c.TemplateErrFatal, c.Vault, @@ -561,6 +575,7 @@ func DefaultConfig() *Config { FileLog: DefaultLogFileConfig(), Nomad: DefaultNomadConfig(), Syslog: DefaultSyslogConfig(), + Telemetry: DefaultTelemetryConfig(), Templates: DefaultTemplateConfigs(), Vault: DefaultVaultConfig(), Wait: DefaultWaitConfig(), @@ -634,6 +649,11 @@ func (c *Config) Finalize() { } c.Syslog.Finalize() + if c.Telemetry == nil { + c.Telemetry = DefaultTelemetryConfig() + } + c.Telemetry.Finalize() + if c.Templates == nil { c.Templates = DefaultTemplateConfigs() } diff --git a/config/convert.go b/config/convert.go index bd1877cd2..74dd5acc3 100644 --- a/config/convert.go +++ b/config/convert.go @@ -25,6 +25,15 @@ func BoolVal(b *bool) bool { return *b } +// BoolCopy returns a copy of the boolean pointer +func BoolCopy(b *bool) *bool { + if b == nil { + return nil + } + + return Bool(*b) +} + // BoolGoString returns the value of the boolean for printing in a string. func BoolGoString(b *bool) string { if b == nil { @@ -115,6 +124,46 @@ func IntPresent(i *int) bool { return *i != 0 } +// Uint returns a pointer to the given uint. +func Uint(i uint) *uint { + return &i +} + +// UintVal returns the value of the uint at the pointer, or 0 if the pointer is +// nil. +func UintVal(i *uint) uint { + if i == nil { + return 0 + } + return *i +} + +// UintCopy returns a copy of the uint pointer +func UintCopy(i *uint) *uint { + if i == nil { + return nil + } + + return Uint(*i) +} + +// UintGoString returns the value of the uint for printing in a string. +func UintGoString(i *uint) string { + if i == nil { + return "(*uint)(nil)" + } + return fmt.Sprintf("%d", *i) +} + +// UintPresent returns a boolean indicating if the pointer is nil, or if the +// pointer is pointing to the zero value. +func UintPresent(i *uint) bool { + if i == nil { + return false + } + return *i != 0 +} + // Signal returns a pointer to the given os.Signal. func Signal(s os.Signal) *os.Signal { return &s @@ -162,6 +211,15 @@ func StringVal(s *string) string { return *s } +// StringCopy returns a copy of the string pointer +func StringCopy(s *string) *string { + if s == nil { + return nil + } + + return String(*s) +} + // StringGoString returns the value of the string for printing in a string. func StringGoString(s *string) string { if s == nil { @@ -192,6 +250,15 @@ func TimeDurationVal(t *time.Duration) time.Duration { return *t } +// TimeDurationCopy returns a copy of the time.Duration pointer +func TimeDurationCopy(t *time.Duration) *time.Duration { + if t == nil { + return nil + } + + return TimeDuration(*t) +} + // TimeDurationGoString returns the value of the time.Duration for printing in a // string. func TimeDurationGoString(t *time.Duration) string { diff --git a/config/syslog.go b/config/syslog.go index dcc1280d9..963ce645d 100644 --- a/config/syslog.go +++ b/config/syslog.go @@ -99,7 +99,7 @@ func (c *SyslogConfig) GoString() string { return fmt.Sprintf("&SyslogConfig{"+ "Enabled:%s, "+ - "Facility:%s"+ + "Facility:%s, "+ "Name:%s"+ "}", BoolGoString(c.Enabled), diff --git a/config/telemetry.go b/config/telemetry.go new file mode 100644 index 000000000..af666d161 --- /dev/null +++ b/config/telemetry.go @@ -0,0 +1,402 @@ +package config + +/* + Config structure based on Consul telemetry config: + https://github.com/hashicorp/consul/blob/main/lib/telemetry.go#L29 +*/ + +import ( + "fmt" + "time" +) + +const ( + defaultMetricsPrefix = "consul_template" +) + +// TelemetryConfig is embedded in config.RuntimeConfig and holds the +// configuration variables for go-metrics. It is a separate struct to allow it +// to be exported as JSON and passed to other process like managed connect +// proxies so they can inherit the agent's telemetry config. +// +// It is in lib package rather than agent/config because we need to use it in +// the shared InitTelemetry functions below, but we can't import agent/config +// due to a dependency cycle. +type TelemetryConfig struct { + // Disable may be set to true to have InitTelemetry to skip initialization + // and return a nil MetricsSink. + Disable bool + + // Circonus*: see https://github.com/circonus-labs/circonus-gometrics + // for more details on the various configuration options. + // Valid configuration combinations: + // - CirconusAPIToken + // metric management enabled (search for existing check or create a new one) + // - CirconusSubmissionUrl + // metric management disabled (use check with specified submission_url, + // broker must be using a public SSL certificate) + // - CirconusAPIToken + CirconusCheckSubmissionURL + // metric management enabled (use check with specified submission_url) + // - CirconusAPIToken + CirconusCheckID + // metric management enabled (use check with specified id) + + // CirconusAPIApp is an app name associated with API token. + // Default: "consul" + // + // hcl: telemetry { circonus_api_app = string } + CirconusAPIApp string `json:"circonus_api_app,omitempty" mapstructure:"circonus_api_app"` + + // CirconusAPIToken is a valid API Token used to create/manage check. If provided, + // metric management is enabled. + // Default: none + // + // hcl: telemetry { circonus_api_token = string } + CirconusAPIToken string `json:"circonus_api_token,omitempty" mapstructure:"circonus_api_token"` + + // CirconusAPIURL is the base URL to use for contacting the Circonus API. + // Default: "https://api.circonus.com/v2" + // + // hcl: telemetry { circonus_api_url = string } + CirconusAPIURL string `json:"circonus_apiurl,omitempty" mapstructure:"circonus_apiurl"` + + // CirconusBrokerID is an explicit broker to use when creating a new check. The numeric portion + // of broker._cid. If metric management is enabled and neither a Submission URL nor Check ID + // is provided, an attempt will be made to search for an existing check using Instance ID and + // Search Tag. If one is not found, a new HTTPTRAP check will be created. + // Default: use Select Tag if provided, otherwise, a random Enterprise Broker associated + // with the specified API token or the default Circonus Broker. + // Default: none + // + // hcl: telemetry { circonus_broker_id = string } + CirconusBrokerID string `json:"circonus_broker_id,omitempty" mapstructure:"circonus_broker_id"` + + // CirconusBrokerSelectTag is a special tag which will be used to select a broker when + // a Broker ID is not provided. The best use of this is to as a hint for which broker + // should be used based on *where* this particular instance is running. + // (e.g. a specific geo location or datacenter, dc:sfo) + // Default: none + // + // hcl: telemetry { circonus_broker_select_tag = string } + CirconusBrokerSelectTag string `json:"circonus_broker_select_tag,omitempty" mapstructure:"circonus_broker_select_tag"` + + // CirconusCheckDisplayName is the name for the check which will be displayed in the Circonus UI. + // Default: value of CirconusCheckInstanceID + // + // hcl: telemetry { circonus_check_display_name = string } + CirconusCheckDisplayName string `json:"circonus_check_display_name,omitempty" mapstructure:"circonus_check_display_name"` + + // CirconusCheckForceMetricActivation will force enabling metrics, as they are encountered, + // if the metric already exists and is NOT active. If check management is enabled, the default + // behavior is to add new metrics as they are encountered. If the metric already exists in the + // check, it will *NOT* be activated. This setting overrides that behavior. + // Default: "false" + // + // hcl: telemetry { circonus_check_metrics_activation = (true|false) + CirconusCheckForceMetricActivation string `json:"circonus_check_force_metric_activation,omitempty" mapstructure:"circonus_check_force_metric_activation"` + + // CirconusCheckID is the check id (not check bundle id) from a previously created + // HTTPTRAP check. The numeric portion of the check._cid field. + // Default: none + // + // hcl: telemetry { circonus_check_id = string } + CirconusCheckID string `json:"circonus_check_id,omitempty" mapstructure:"circonus_check_id"` + + // CirconusCheckInstanceID serves to uniquely identify the metrics coming from this "instance". + // It can be used to maintain metric continuity with transient or ephemeral instances as + // they move around within an infrastructure. + // Default: hostname:app + // + // hcl: telemetry { circonus_check_instance_id = string } + CirconusCheckInstanceID string `json:"circonus_check_instance_id,omitempty" mapstructure:"circonus_check_instance_id"` + + // CirconusCheckSearchTag is a special tag which, when coupled with the instance id, helps to + // narrow down the search results when neither a Submission URL or Check ID is provided. + // Default: service:app (e.g. service:consul) + // + // hcl: telemetry { circonus_check_search_tag = string } + CirconusCheckSearchTag string `json:"circonus_check_search_tag,omitempty" mapstructure:"circonus_check_search_tag"` + + // CirconusCheckSearchTag is a special tag which, when coupled with the instance id, helps to + // narrow down the search results when neither a Submission URL or Check ID is provided. + // Default: service:app (e.g. service:consul) + // + // hcl: telemetry { circonus_check_tags = string } + CirconusCheckTags string `json:"circonus_check_tags,omitempty" mapstructure:"circonus_check_tags"` + + // CirconusSubmissionInterval is the interval at which metrics are submitted to Circonus. + // Default: 10s + // + // hcl: telemetry { circonus_submission_interval = "duration" } + CirconusSubmissionInterval string `json:"circonus_submission_interval,omitempty" mapstructure:"circonus_submission_interval"` + + // CirconusCheckSubmissionURL is the check.config.submission_url field from a + // previously created HTTPTRAP check. + // Default: none + // + // hcl: telemetry { circonus_submission_url = string } + CirconusSubmissionURL string `json:"circonus_submission_url,omitempty" mapstructure:"circonus_submission_url"` + + // DisableHostname will disable hostname prefixing for all metrics. + // + // hcl: telemetry { disable_hostname = (true|false) + DisableHostname bool `json:"disable_hostname,omitempty" mapstructure:"disable_hostname"` + + // DogStatsdAddr is the address of a dogstatsd instance. If provided, + // metrics will be sent to that instance + // + // hcl: telemetry { dogstatsd_addr = string } + DogstatsdAddr string `json:"dogstatsd_addr,omitempty" mapstructure:"dogstatsd_addr"` + + // DogStatsdTags are the global tags that should be sent with each packet to dogstatsd + // It is a list of strings, where each string looks like "my_tag_name:my_tag_value" + // + // hcl: telemetry { dogstatsd_tags = []string } + DogstatsdTags []string `json:"dogstatsd_tags,omitempty" mapstructure:"dogstatsd_tags"` + + // FilterDefault is the default for whether to allow a metric that's not + // covered by the filter. + // + // hcl: telemetry { filter_default = (true|false) } + FilterDefault bool `json:"filter_default,omitempty" mapstructure:"filter_default"` + + // AllowedPrefixes is a list of filter rules to apply for allowing metrics + // by prefix. Use the 'prefix_filter' option and prefix rules with '+' to be + // included. + // + // hcl: telemetry { prefix_filter = []string{"+", "+", ...} } + AllowedPrefixes []string `json:"allowed_prefixes,omitempty" mapstructure:"allowed_prefixes"` + + // BlockedPrefixes is a list of filter rules to apply for blocking metrics + // by prefix. Use the 'prefix_filter' option and prefix rules with '-' to be + // excluded. + // + // hcl: telemetry { prefix_filter = []string{"-", "-", ...} } + BlockedPrefixes []string `json:"blocked_prefixes,omitempty" mapstructure:"blocked_prefixes"` + + // MetricsPrefix is the prefix used to write stats values to. + // Default: "consul_template." + // + // hcl: telemetry { metrics_prefix = string } + MetricsPrefix string `json:"metrics_prefix,omitempty" mapstructure:"metrics_prefix"` + + // StatsdAddr is the address of a statsd instance. If provided, + // metrics will be sent to that instance. + // + // hcl: telemetry { statsd_address = string } + StatsdAddr string `json:"statsd_address,omitempty" mapstructure:"statsd_address"` + + // StatsiteAddr is the address of a statsite instance. If provided, + // metrics will be streamed to that instance. + // + // hcl: telemetry { statsite_address = string } + StatsiteAddr string `json:"statsite_address,omitempty" mapstructure:"statsite_address"` + + // PrometheusRetentionTime is the time before a prometheus metric expires. + // + // hcl: telemetry { prometheus_retention_time = "duration" } + PrometheusRetentionTime time.Duration `json:"prometheus_retention_time,omitempty" mapstructure:"prometheus_retention_time"` + + // PrometheusPort is the REST port under which the metrics can be queried. + // + // hcl: telemetry { prometheus_port = int } + PrometheusPort int `json:"prometheus_port,omitempty" mapstructure:"prometheus_port"` +} + +func DefaultTelemetryConfig() *TelemetryConfig { + return &TelemetryConfig{} +} + +func (c *TelemetryConfig) Copy() *TelemetryConfig { + if c == nil { + return nil + } + + return &TelemetryConfig{ + Disable: c.Disable, + CirconusAPIApp: c.CirconusAPIApp, + CirconusAPIToken: c.CirconusAPIToken, + CirconusAPIURL: c.CirconusAPIURL, + CirconusBrokerID: c.CirconusBrokerID, + CirconusBrokerSelectTag: c.CirconusBrokerSelectTag, + CirconusCheckDisplayName: c.CirconusCheckDisplayName, + CirconusCheckForceMetricActivation: c.CirconusCheckForceMetricActivation, + CirconusCheckID: c.CirconusCheckID, + CirconusCheckInstanceID: c.CirconusCheckInstanceID, + CirconusCheckSearchTag: c.CirconusCheckSearchTag, + CirconusCheckTags: c.CirconusCheckTags, + CirconusSubmissionInterval: c.CirconusSubmissionInterval, + CirconusSubmissionURL: c.CirconusSubmissionURL, + DisableHostname: c.DisableHostname, + DogstatsdAddr: c.DogstatsdAddr, + DogstatsdTags: c.DogstatsdTags, + FilterDefault: c.FilterDefault, + AllowedPrefixes: c.AllowedPrefixes, + BlockedPrefixes: c.BlockedPrefixes, + MetricsPrefix: c.MetricsPrefix, + StatsdAddr: c.StatsdAddr, + StatsiteAddr: c.StatsiteAddr, + PrometheusPort: c.PrometheusPort, + PrometheusRetentionTime: c.PrometheusRetentionTime, + } +} + +func (c *TelemetryConfig) Merge(o *TelemetryConfig) *TelemetryConfig { + if c == nil { + if o == nil { + return nil + } + return o.Copy() + } + if o == nil { + return c.Copy() + } + + r := c.Copy() + + r.Disable = o.Disable + + if o.CirconusAPIApp != "" { + r.CirconusAPIApp = o.CirconusAPIApp + } + if o.CirconusAPIToken != "" { + r.CirconusAPIToken = o.CirconusAPIToken + } + if o.CirconusAPIURL != "" { + r.CirconusAPIURL = o.CirconusAPIURL + } + if o.CirconusBrokerID != "" { + r.CirconusBrokerID = o.CirconusBrokerID + } + if o.CirconusBrokerSelectTag != "" { + r.CirconusBrokerSelectTag = o.CirconusBrokerSelectTag + } + if o.CirconusCheckDisplayName != "" { + r.CirconusCheckDisplayName = o.CirconusCheckDisplayName + } + if o.CirconusCheckForceMetricActivation != "" { + r.CirconusCheckForceMetricActivation = o.CirconusCheckForceMetricActivation + } + if o.CirconusCheckID != "" { + r.CirconusCheckID = o.CirconusCheckID + } + if o.CirconusCheckInstanceID != "" { + r.CirconusCheckInstanceID = o.CirconusCheckInstanceID + } + if o.CirconusCheckSearchTag != "" { + r.CirconusCheckSearchTag = o.CirconusCheckSearchTag + } + if o.CirconusCheckTags != "" { + r.CirconusCheckTags = o.CirconusCheckTags + } + if o.CirconusSubmissionInterval != "" { + r.CirconusSubmissionInterval = o.CirconusSubmissionInterval + } + if o.CirconusSubmissionURL != "" { + r.CirconusSubmissionURL = o.CirconusSubmissionURL + } + r.DisableHostname = o.DisableHostname + if o.DogstatsdAddr != "" { + r.DogstatsdAddr = o.DogstatsdAddr + } + if len(o.DogstatsdTags) != 0 { + r.DogstatsdTags = o.DogstatsdTags + } + r.FilterDefault = o.FilterDefault + if len(o.AllowedPrefixes) != 0 { + r.AllowedPrefixes = o.AllowedPrefixes + } + if len(o.BlockedPrefixes) != 0 { + r.BlockedPrefixes = o.BlockedPrefixes + } + if o.MetricsPrefix != "" { + r.MetricsPrefix = o.MetricsPrefix + } + if o.StatsdAddr != "" { + r.StatsdAddr = o.StatsdAddr + } + if o.StatsiteAddr != "" { + r.StatsiteAddr = o.StatsiteAddr + } + + if o.PrometheusRetentionTime.Nanoseconds() > 0 { + r.PrometheusRetentionTime = o.PrometheusRetentionTime + } + if o.PrometheusPort != 0 { + r.PrometheusPort = o.PrometheusPort + } + + return r +} + +func (c *TelemetryConfig) GoString() string { + if c == nil { + return "(*TelemetryConfig)(nil)" + } + return fmt.Sprintf("&TelemetryConfig{"+ + "Disable:%v, "+ + "CirconusAPIApp:%s, "+ + "CirconusAPIToken:%s, "+ + "CirconusAPIURL:%s, "+ + "CirconusBrokerID:%s, "+ + "CirconusBrokerSelectTag:%s, "+ + "CirconusCheckDisplayName:%s, "+ + "CirconusCheckForceMetricActivation:%s, "+ + "CirconusCheckID:%s, "+ + "CirconusCheckInstanceID:%s, "+ + "CirconusCheckSearchTag:%s, "+ + "CirconusCheckTags:%s, "+ + "CirconusSubmissionInterval:%s, "+ + "CirconusSubmissionURL:%s, "+ + "DisableHostname:%v, "+ + "DogstatsdAddr:%s, "+ + "DogstatsdTags:%v, "+ + "FilterDefault:%v, "+ + "AllowedPrefixes:%s, "+ + "BlockedPrefixes:%s, "+ + "MetricsPrefix:%s, "+ + "StatsdAddr:%s, "+ + "StatsiteAddr:%s, "+ + "PrometheusPort:%d, "+ + "PrometheusRetentionTime:%s}", + c.Disable, + c.CirconusAPIApp, + c.CirconusAPIToken, + c.CirconusAPIURL, + c.CirconusBrokerID, + c.CirconusBrokerSelectTag, + c.CirconusCheckDisplayName, + c.CirconusCheckForceMetricActivation, + c.CirconusCheckID, + c.CirconusCheckInstanceID, + c.CirconusCheckSearchTag, + c.CirconusCheckTags, + c.CirconusSubmissionInterval, + c.CirconusSubmissionURL, + c.DisableHostname, + c.DogstatsdAddr, + c.DogstatsdTags, + c.FilterDefault, + c.AllowedPrefixes, + c.BlockedPrefixes, + c.MetricsPrefix, + c.StatsdAddr, + c.StatsiteAddr, + c.PrometheusPort, + c.PrometheusRetentionTime, + ) +} + +func (c *TelemetryConfig) Finalize() { + if c == nil { + return + } + if c.MetricsPrefix == "" { + c.MetricsPrefix = defaultMetricsPrefix + } + + c.AllowedPrefixes = append(c.AllowedPrefixes, c.MetricsPrefix) + + if c.PrometheusRetentionTime.Nanoseconds() < 1 { + c.PrometheusRetentionTime = 60 * time.Second + } +} diff --git a/config/telemetry_test.go b/config/telemetry_test.go new file mode 100644 index 000000000..fecaae746 --- /dev/null +++ b/config/telemetry_test.go @@ -0,0 +1,25 @@ +package config + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestPromConfigParsing(t *testing.T) { + configStr := "telemetry {" + + "prometheus_port = 9110" + + "prometheus_retention_time = \"120s\"" + + "}" + + config, err := Parse(configStr) + require.NoError(t, err) + + require.Equal(t, 9110, config.Telemetry.PrometheusPort) + require.Equal(t, 120*time.Second, config.Telemetry.PrometheusRetentionTime) + + config.Finalize() + require.Equal(t, 9110, config.Telemetry.PrometheusPort) + require.Equal(t, 120*time.Second, config.Telemetry.PrometheusRetentionTime) +} diff --git a/dependency/dependency.go b/dependency/dependency.go index 29ed744ad..b443835a0 100644 --- a/dependency/dependency.go +++ b/dependency/dependency.go @@ -49,6 +49,14 @@ const ( DefaultContextTimeout = 60 * time.Second ) +func (t Type) String() string { + if t > 2 { + return "unknown" + } + + return []string{"consul", "vault", "local"}[t] +} + // Dependency is an interface for a dependency that Consul Template is capable // of watching. type Dependency interface { diff --git a/dependency/vault_common.go b/dependency/vault_common.go index b28a98214..553d33562 100644 --- a/dependency/vault_common.go +++ b/dependency/vault_common.go @@ -96,6 +96,9 @@ func renewSecret(clients *ClientSet, d renewer) error { case err := <-renewer.DoneCh(): if err != nil { log.Printf("[WARN] %s: failed to renew: %s", d, err) + //d.recordCounter("status", "stopped") + } else { + //d.recordCounter("status", "expired") } log.Printf("[WARN] %s: renewer done (maybe the lease expired)", d) return nil @@ -103,7 +106,9 @@ func renewSecret(clients *ClientSet, d renewer) error { log.Printf("[TRACE] %s: successfully renewed", d) printVaultWarnings(d, renewal.Secret.Warnings) updateSecret(secret, renewal.Secret) + //d.recordCounter("status", "renewed") case <-d.stopChan(): + //d.recordCounter("status", "stopped") return ErrStopped } } diff --git a/go.mod b/go.mod index 28da4b3fd..43ec0ef98 100644 --- a/go.mod +++ b/go.mod @@ -33,19 +33,28 @@ require ( require ( dario.cat/mergo v1.0.0 github.com/Masterminds/sprig/v3 v3.2.3 + github.com/armon/go-metrics v0.4.1 github.com/hashicorp/vault/api/auth/kubernetes v0.5.0 + github.com/prometheus/client_golang v1.5.0 golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 golang.org/x/text v0.14.0 ) +require google.golang.org/protobuf v1.33.0 // indirect + require ( + github.com/DataDog/datadog-go v3.2.0+incompatible // indirect github.com/Masterminds/goutils v1.1.1 // indirect github.com/Masterminds/semver/v3 v3.2.0 // indirect - github.com/armon/go-metrics v0.4.1 // indirect + github.com/beorn7/perks v1.0.1 // indirect github.com/cenkalti/backoff/v3 v3.2.2 // indirect + github.com/cespare/xxhash/v2 v2.1.1 // indirect + github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible // indirect + github.com/circonus-labs/circonusllhist v0.1.3 // indirect github.com/coreos/go-systemd/v22 v22.5.0 github.com/fatih/color v1.17.0 // indirect github.com/go-jose/go-jose/v3 v3.0.3 // indirect + github.com/golang/protobuf v1.5.4 // indirect github.com/google/uuid v1.3.0 // indirect github.com/gorilla/websocket v1.5.0 // indirect github.com/hashicorp/cronexpr v1.1.1 // indirect @@ -62,13 +71,18 @@ require ( github.com/imdario/mergo v0.3.11 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect github.com/miekg/dns v1.1.50 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/prometheus/client_model v0.2.0 // indirect + github.com/prometheus/common v0.9.1 // indirect + github.com/prometheus/procfs v0.0.10 // indirect github.com/ryanuber/go-glob v1.0.0 // indirect github.com/shopspring/decimal v1.3.1 // indirect github.com/spf13/cast v1.5.0 // indirect + github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926 // indirect golang.org/x/net v0.24.0 // indirect golang.org/x/time v0.3.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index c3a330640..1ee936c98 100644 --- a/go.sum +++ b/go.sum @@ -2,6 +2,7 @@ dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/DataDog/datadog-go v3.2.0+incompatible h1:qSG2N4FghB1He/r2mFrWKCaL7dXCilEuNEeAn20fdD4= github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= @@ -21,13 +22,17 @@ github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/cenkalti/backoff/v3 v3.0.0/go.mod h1:cIeZDE3IrqwwJl6VUwCN6trj1oXrTS4rc0ij+ULvLYs= github.com/cenkalti/backoff/v3 v3.2.2 h1:cfUAAO3yvKMYKPrvhDuHSwQnhZNk/RMHKdZqKTxfm6M= github.com/cenkalti/backoff/v3 v3.2.2/go.mod h1:cIeZDE3IrqwwJl6VUwCN6trj1oXrTS4rc0ij+ULvLYs= +github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible h1:C29Ae4G5GtYyYMm1aztcyj/J5ckgJm2zwdDajFbx1NY= github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= +github.com/circonus-labs/circonusllhist v0.1.3 h1:TJH+oke8D16535+jHExHj4nQvzlZrj7ug5D7I/orNUA= github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= @@ -110,8 +115,6 @@ github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+l github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= github.com/hashicorp/go-retryablehttp v0.6.6/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= -github.com/hashicorp/go-retryablehttp v0.7.2 h1:AcYqCvkpalPnPF2pn0KamgwamS42TqUDDYFRKq/RAd0= -github.com/hashicorp/go-retryablehttp v0.7.2/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8= github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU= github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk= github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5Oi2viEzc= @@ -185,6 +188,7 @@ github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27k github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= @@ -229,14 +233,20 @@ github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSg github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= +github.com/prometheus/client_golang v1.5.0 h1:Ctq0iGpCmr3jeP77kbF2UxgvRwzWWz+4Bh9/vJTyg1A= +github.com/prometheus/client_golang v1.5.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.9.1 h1:KOMtN28tlbam3/7ZKEYKHhKoJZYYj3gMH4uc62x7X7U= github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= +github.com/prometheus/procfs v0.0.10 h1:QJQN3jYQhkamO4mhfUWqdDH2asK7ONOI9MTWjyAxNKM= +github.com/prometheus/procfs v0.0.10/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= @@ -268,6 +278,7 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926 h1:G3dpKMzFDjgEh2q1Z7zUUtKa8ViPtH+ocF0bE0g00O8= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= diff --git a/manager/runner.go b/manager/runner.go index 8c16c727b..748ad1d67 100644 --- a/manager/runner.go +++ b/manager/runner.go @@ -20,6 +20,7 @@ import ( "github.com/hashicorp/consul-template/config" dep "github.com/hashicorp/consul-template/dependency" "github.com/hashicorp/consul-template/renderer" + "github.com/hashicorp/consul-template/telemetry" "github.com/hashicorp/consul-template/template" "github.com/hashicorp/consul-template/watch" @@ -256,6 +257,8 @@ func NewRunner(config *config.Config, dry bool) (*Runner, error) { func (r *Runner) Start() { log.Printf("[INFO] (runner) starting") + telemetry.CounterActions.Add(1, telemetry.NewLabel("action", "start")) + // Create the pid before doing anything. if err := r.storePid(); err != nil { r.ErrCh <- err @@ -540,6 +543,9 @@ func (r *Runner) internalStop(immediately bool) { } log.Printf("[INFO] (runner) stopping") + + telemetry.CounterActions.Add(1, telemetry.NewLabel("action", "stop")) + r.stopDedup() r.stopWatchers() r.stopChild(immediately) @@ -608,9 +614,16 @@ func (r *Runner) Receive(d dep.Dependency, data interface{}) { // https://github.com/hashicorp/consul-template/issues/198 // // and by "little" bug, I mean really big bug. - if _, ok := r.dependencies[d.String()]; ok { + depID := d.String() + if _, ok := r.dependencies[depID]; ok { log.Printf("[DEBUG] (runner) receiving dependency %s", d) r.brain.Remember(d, data) + + telemetry.CounterDependenciesReceived.Add( + 1, + telemetry.NewLabel("id", depID), + telemetry.NewLabel("type", d.Type().String()), + ) } } @@ -634,6 +647,7 @@ func (r *Runner) Signal(s os.Signal) error { // executed. func (r *Runner) Run() error { log.Printf("[DEBUG] (runner) initiating run") + telemetry.CounterActions.Add(1, telemetry.NewLabel("action", "run")) var newRenderEvent, wouldRenderAny, renderedAny bool runCtx := &templateRunCtx{ @@ -655,14 +669,31 @@ func (r *Runner) Run() error { // Record that there is at least one new render event newRenderEvent = true + var label string + // Record that at least one template would have been rendered. if event.WouldRender { wouldRenderAny = true + label = "would" } // Record that at least one template was rendered. if event.DidRender { renderedAny = true + label = "rendered" + } + + if event.ForQuiescence { + label = "quiescence" + } + + // Report the template render event + if label != "" { + telemetry.CounterTemplatesRendered.Add( + 1, + telemetry.NewLabel("id", tmpl.ID()), + telemetry.NewLabel("status", label), + ) } } } @@ -670,6 +701,10 @@ func (r *Runner) Run() error { // Perform the diff and update the known dependencies. r.diffAndUpdateDeps(runCtx.depsMap) + // Record dependency counts on runCtx instead of runner.dependencies + // to avoid blocking the locks + recordDependencyCounts(runCtx.depsMap) + // Execute each command in sequence, collecting any errors that occur - this // ensures all commands execute at least once. var errs []error @@ -696,6 +731,12 @@ func (r *Runner) Run() error { } } + // Report on number of commands executed and their statuses + numCommands := len(runCtx.commands) + failedCommands := len(errs) + telemetry.CounterCommandExecs.Add(float32(numCommands-failedCommands), telemetry.NewLabel("status", "success")) + telemetry.CounterCommandExecs.Add(float32(failedCommands), telemetry.NewLabel("status", "error")) + // Check if we need to deliver any rendered signals if wouldRenderAny || renderedAny { // Send the signal that a template got rendered @@ -1478,3 +1519,10 @@ func newWatcher(c *config.Config, clients *dep.ClientSet) *watch.Watcher { RetryFuncNomad: watch.RetryFunc(c.Nomad.Retry.RetryFunc()), }) } + +func recordDependencyCounts(deps map[string]dep.Dependency) { + types := make(map[dep.Type]float32) + for _, dep := range deps { + types[dep.Type()]++ + } +} diff --git a/telemetry/circonus.go b/telemetry/circonus.go new file mode 100644 index 000000000..427dc9dbb --- /dev/null +++ b/telemetry/circonus.go @@ -0,0 +1,54 @@ +package telemetry + +import ( + "github.com/armon/go-metrics" + "github.com/armon/go-metrics/circonus" + "github.com/hashicorp/consul-template/config" +) + +/* + methods extracted from Consul telemetry: + https://github.com/hashicorp/consul/blob/main/lib/telemetry.go#L274 +*/ + +func circonusSink(cfg *config.TelemetryConfig, hostname string) (metrics.MetricSink, error) { + token := cfg.CirconusAPIToken + url := cfg.CirconusSubmissionURL + if token == "" && url == "" { + return nil, nil + } + + conf := &circonus.Config{} + conf.Interval = cfg.CirconusSubmissionInterval + conf.CheckManager.API.TokenKey = token + conf.CheckManager.API.TokenApp = cfg.CirconusAPIApp + conf.CheckManager.API.URL = cfg.CirconusAPIURL + conf.CheckManager.Check.SubmissionURL = url + conf.CheckManager.Check.ID = cfg.CirconusCheckID + conf.CheckManager.Check.ForceMetricActivation = cfg.CirconusCheckForceMetricActivation + conf.CheckManager.Check.InstanceID = cfg.CirconusCheckInstanceID + conf.CheckManager.Check.SearchTag = cfg.CirconusCheckSearchTag + conf.CheckManager.Check.DisplayName = cfg.CirconusCheckDisplayName + conf.CheckManager.Check.Tags = cfg.CirconusCheckTags + conf.CheckManager.Broker.ID = cfg.CirconusBrokerID + conf.CheckManager.Broker.SelectTag = cfg.CirconusBrokerSelectTag + + if conf.CheckManager.Check.DisplayName == "" { + conf.CheckManager.Check.DisplayName = "Consul" + } + + if conf.CheckManager.API.TokenApp == "" { + conf.CheckManager.API.TokenApp = "consul" + } + + if conf.CheckManager.Check.SearchTag == "" { + conf.CheckManager.Check.SearchTag = "service:consul" + } + + sink, err := circonus.NewCirconusSink(conf) + if err != nil { + return nil, err + } + sink.Start() + return sink, nil +} diff --git a/telemetry/dogstatsd.go b/telemetry/dogstatsd.go new file mode 100644 index 000000000..d175fc1ba --- /dev/null +++ b/telemetry/dogstatsd.go @@ -0,0 +1,25 @@ +package telemetry + +import ( + "github.com/armon/go-metrics" + "github.com/armon/go-metrics/datadog" + "github.com/hashicorp/consul-template/config" +) + +/* + methods extracted from Consul telemetry: + https://github.com/hashicorp/consul/blob/main/lib/telemetry.go#L248 +*/ + +func dogstatdSink(cfg *config.TelemetryConfig, hostname string) (metrics.MetricSink, error) { + addr := cfg.DogstatsdAddr + if addr == "" { + return nil, nil + } + sink, err := datadog.NewDogStatsdSink(addr, hostname) + if err != nil { + return nil, err + } + sink.SetTags(cfg.DogstatsdTags) + return sink, nil +} diff --git a/telemetry/metrics.go b/telemetry/metrics.go new file mode 100644 index 000000000..153d4e272 --- /dev/null +++ b/telemetry/metrics.go @@ -0,0 +1,52 @@ +package telemetry + +import ( + "github.com/armon/go-metrics" +) + +type CounterMetric struct { + Names []string + Description string + ConstLabels []metrics.Label +} + +func (m *CounterMetric) Add(val float32, labels ...metrics.Label) { + if len(labels) == 0 { + metrics.IncrCounter(m.Names, val) + } else { + metrics.IncrCounterWithLabels(m.Names, val, labels) + } +} + +// Counters +var CounterDependenciesReceived = CounterMetric{ + Names: []string{"dependencies_received"}, + ConstLabels: []metrics.Label{}, + Description: "A counter of dependencies received with labels " + + "type=(consul|vault|local) and id=dependencyString", +} +var CounterTemplatesRendered = CounterMetric{ + Names: []string{"templates_rendered"}, + ConstLabels: []metrics.Label{}, + Description: "A counter of templates rendered with labels " + + "id=templateID and status=(rendered|would|quiescence)", +} + +var CounterActions = CounterMetric{ + Names: []string{"runner_actions"}, + ConstLabels: []metrics.Label{}, + Description: "A count of runner actions with labels action=(start|stop|run)", +} +var CounterCommandExecs = CounterMetric{ + Names: []string{"commands_exec"}, + ConstLabels: []metrics.Label{}, + Description: "The number of commands executed with labels status=(success|error)", +} + +func NewLabel(name string, value string) metrics.Label { + return metrics.Label{Name: name, Value: value} +} + +func InitMetrics() { + CounterActions.Add(0) +} diff --git a/telemetry/prometheus.go b/telemetry/prometheus.go new file mode 100644 index 000000000..367e1b1a9 --- /dev/null +++ b/telemetry/prometheus.go @@ -0,0 +1,52 @@ +package telemetry + +import ( + "fmt" + "log" + "net/http" + + "github.com/armon/go-metrics" + "github.com/armon/go-metrics/prometheus" + "github.com/hashicorp/consul-template/config" + prom "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" +) + +/* + methods based on Consul telemetry: + https://github.com/hashicorp/consul/blob/main/lib/telemetry.go#L261 +*/ + +func PrometheusSink(cfg *config.TelemetryConfig, hostname string) (metrics.MetricSink, error) { + if cfg.PrometheusPort == 0 { + return nil, nil + } + + sink, err := prometheus.NewPrometheusSinkFrom(prometheus.PrometheusOpts{ + Expiration: cfg.PrometheusRetentionTime, + }) + + if err != nil { + return nil, err + } + + runPrometheusMetricServer(cfg.PrometheusPort) + + return sink, nil +} + +func runPrometheusMetricServer(prometheusPort int) { + handlerOptions := promhttp.HandlerOpts{ + ErrorHandling: promhttp.ContinueOnError, + } + + go func() { + log.Println("[INFO] (prometheus) running prom server") + handler := promhttp.HandlerFor(prom.DefaultGatherer, handlerOptions) + http.Handle("/metrics", handler) + err := http.ListenAndServe(fmt.Sprintf(":%d", prometheusPort), nil) + if err != nil { + log.Printf("[ERROR] (prometheus) error thrown by the metric server: %v", err) + } + }() +} diff --git a/telemetry/statsd.go b/telemetry/statsd.go new file mode 100644 index 000000000..8058f26a6 --- /dev/null +++ b/telemetry/statsd.go @@ -0,0 +1,19 @@ +package telemetry + +import ( + "github.com/armon/go-metrics" + "github.com/hashicorp/consul-template/config" +) + +/* + methods extracted from Consul telemetry: + https://github.com/hashicorp/consul/blob/main/lib/telemetry.go#L240 +*/ + +func statsdSink(cfg *config.TelemetryConfig, hostname string) (metrics.MetricSink, error) { + addr := cfg.StatsdAddr + if addr == "" { + return nil, nil + } + return metrics.NewStatsdSink(addr) +} diff --git a/telemetry/statsite.go b/telemetry/statsite.go new file mode 100644 index 000000000..0fda210dc --- /dev/null +++ b/telemetry/statsite.go @@ -0,0 +1,19 @@ +package telemetry + +import ( + "github.com/armon/go-metrics" + "github.com/hashicorp/consul-template/config" +) + +/* + methods extracted from Consul telemetry: + https://github.com/hashicorp/consul/blob/main/lib/telemetry.go#L232 +*/ + +func statsiteSink(cfg *config.TelemetryConfig, hostname string) (metrics.MetricSink, error) { + addr := cfg.StatsiteAddr + if addr == "" { + return nil, nil + } + return metrics.NewStatsiteSink(addr) +} diff --git a/telemetry/telemetry.go b/telemetry/telemetry.go new file mode 100644 index 000000000..8d8766f08 --- /dev/null +++ b/telemetry/telemetry.go @@ -0,0 +1,102 @@ +package telemetry + +/* + methods based on Consul telemetry: + https://github.com/hashicorp/consul/blob/main/lib/telemetry.go +*/ + +import ( + "context" + "net/http" + "sync" + "time" + + "github.com/armon/go-metrics" + "github.com/hashicorp/go-multierror" + + "github.com/hashicorp/consul-template/config" +) + +// MetricsHandler provides an http.Handler for displaying metrics. +type MetricsHandler interface { + DisplayMetrics(resp http.ResponseWriter, req *http.Request) (interface{}, error) + Stream(ctx context.Context, encoder metrics.Encoder) +} + +type Telemetry struct { + Handler MetricsHandler + mu sync.Mutex + cancelFn context.CancelFunc +} + +func (tel *Telemetry) Stop() { + tel.mu.Lock() + defer tel.mu.Unlock() + + if tel.cancelFn != nil { + tel.cancelFn() + } +} + +func ConfigureSinks(cfg *config.TelemetryConfig, memSink metrics.MetricSink) (metrics.FanoutSink, error) { + metricsConf := metrics.DefaultConfig(cfg.MetricsPrefix) + metricsConf.EnableHostname = !cfg.DisableHostname + metricsConf.FilterDefault = cfg.FilterDefault + metricsConf.AllowedPrefixes = cfg.AllowedPrefixes + metricsConf.BlockedPrefixes = cfg.BlockedPrefixes + + var sinks metrics.FanoutSink + var errors error + addSink := func(fn func(*config.TelemetryConfig, string) (metrics.MetricSink, error)) { + s, err := fn(cfg, metricsConf.HostName) + if err != nil { + errors = multierror.Append(errors, err) + return + } + if s != nil { + sinks = append(sinks, s) + } + } + + addSink(statsiteSink) + addSink(statsdSink) + addSink(dogstatdSink) + addSink(circonusSink) + addSink(circonusSink) + addSink(PrometheusSink) + + if len(sinks) > 0 { + sinks = append(sinks, memSink) + _, err := metrics.NewGlobal(metricsConf, sinks) + errors = multierror.Append(errors, err) + } else { + metricsConf.EnableHostname = false + _, err := metrics.NewGlobal(metricsConf, memSink) + errors = multierror.Append(errors, err) + } + return sinks, errors +} + +// Init configures go-metrics based on map of telemetry config +// values as returned by Runtimecfg.Config(). +// Init retries configurating the sinks in case error is retriable +// and retry_failed_connection is set to true. +func Init(cfg *config.TelemetryConfig) (*Telemetry, error) { + if cfg.Disable { + return &Telemetry{}, nil + } + + memSink := metrics.NewInmemSink(10*time.Second, time.Minute) + metrics.DefaultInmemSignal(memSink) + + telemetry := &Telemetry{ + Handler: memSink, + } + + _, errs := ConfigureSinks(cfg, memSink) + if errs != nil { + return nil, errs + } + + return telemetry, nil +} diff --git a/telemetry/telemetry_test.go b/telemetry/telemetry_test.go new file mode 100644 index 000000000..9b312992b --- /dev/null +++ b/telemetry/telemetry_test.go @@ -0,0 +1,127 @@ +package telemetry + +import ( + "fmt" + "io" + "net" + "net/http" + "strconv" + "strings" + "testing" + "time" + + "github.com/hashicorp/consul-template/config" + "github.com/stretchr/testify/require" +) + +func newCfg() *config.TelemetryConfig { + return &config.TelemetryConfig{ + StatsdAddr: "statsd.host:1234", + StatsiteAddr: "statsite.host:1234", + DogstatsdAddr: "mydog.host:8125", + } +} + +func TestConfigureSinks(t *testing.T) { + cfg := newCfg() + sinks, err := ConfigureSinks(cfg, nil) + require.Error(t, err) + // 3 sinks: statsd, statsite, inmem + require.Equal(t, 3, len(sinks)) + + cfg = &config.TelemetryConfig{ + DogstatsdAddr: "", + } + _, err = ConfigureSinks(cfg, nil) + require.NoError(t, err) + +} + +func TestPrometheusMetrics(t *testing.T) { + + // expected metrics based on the first metric-related PR: + // https://github.com/hashicorp/consul-template/pull/1378/files#diff-b335630551682c19a781afebcf4d07bf978fb1f8ac04c6bf87428ed5106870f5R2680 + expectedMetrics := []string{ + "# HELP consul_template_commands_exec The number of commands executed with labels status=(success|error)", + "# TYPE consul_template_commands_exec counter", + "consul_template_commands_exec{status=\"error\"} 0", + "consul_template_commands_exec{status=\"success\"} 1", + "# HELP consul_template_dependencies_received A counter of dependencies received with labels type=(consul|vault|local) and id=dependencyString", + "# TYPE consul_template_dependencies_received counter", + "consul_template_dependencies_received{id=\"kv.block(hello)\",type=\"consul\"} 1", + "# HELP consul_template_runner_actions A count of runner actions with labels action=(start|stop|run)", + "# TYPE consul_template_runner_actions counter", + "consul_template_runner_actions{action=\"run\"} 2", + "consul_template_runner_actions{action=\"start\"} 1", + "# HELP consul_template_templates_rendered A counter of templates rendered with labels id=templateID and status=(rendered|would|quiescence)", + "# TYPE consul_template_templates_rendered counter", + "consul_template_templates_rendered{id=\"aadcafd7f28f1d9fc5e76ab2e029f844\",status=\"rendered\"} 1", + } + + l, err := net.Listen("tcp", ":0") + require.NoError(t, err) + + address := l.Addr().String() + portStr := address[strings.LastIndex(address, ":")+1:] + + err = l.Close() + require.NoError(t, err) + + port, err := strconv.Atoi(portStr) + require.NoError(t, err) + + cfg := config.TelemetryConfig{ + PrometheusRetentionTime: 60 * time.Second, + PrometheusPort: port, + } + + _, err = Init(&cfg) + require.NoError(t, err) + + CounterCommandExecs.Add(0, NewLabel("status", "error")) + CounterCommandExecs.Add(1, NewLabel("status", "success")) + CounterDependenciesReceived.Add(1, NewLabel("id", "kv.block(hello)"), NewLabel("type", "consul")) + CounterActions.Add(2, NewLabel("action", "run")) + CounterActions.Add(1, NewLabel("action", "start")) + CounterTemplatesRendered.Add(1, NewLabel("id", "aadcafd7f28f1d9fc5e76ab2e029f844"), NewLabel("status", "rendered")) + + httpClient := http.DefaultClient + + resp, err := httpClient.Get(fmt.Sprintf("http://localhost:%d/metrics", port)) + require.NoError(t, err) + + b, err := io.ReadAll(resp.Body) + require.NoError(t, err) + + actualMetrics := strings.Split(string(b), "\n") + + missingActualMetrics := []string{} + + for _, actualMetric := range actualMetrics { + if strings.HasPrefix(actualMetric, "# HELP go_") || + strings.HasPrefix(actualMetric, "# TYPE go_") || + strings.HasPrefix(actualMetric, "go_") || + strings.HasPrefix(actualMetric, "# HELP process_") || + strings.HasPrefix(actualMetric, "# TYPE process_") || + strings.HasPrefix(actualMetric, "process_") || + actualMetric == "" { + continue + } + + contained := false + for _, expectedMetric := range expectedMetrics { + if actualMetric == expectedMetric { + contained = true + break + } + } + + if !contained { + missingActualMetrics = append(missingActualMetrics, actualMetric) + } + } + + t.Log(len(missingActualMetrics)) + require.Emptyf(t, missingActualMetrics, "The following metrics are missing:\n - %s", strings.Join(missingActualMetrics, "\n - ")) + +} diff --git a/test/helpers.go b/test/helpers.go index f270a4d49..32b16b6cc 100644 --- a/test/helpers.go +++ b/test/helpers.go @@ -4,6 +4,7 @@ package test import ( + "log" "os" "strings" "sync" @@ -97,11 +98,14 @@ func (t *TestingTB) DoCleanup() { t.cleanup() } -func (*TestingTB) Failed() bool { return false } -func (*TestingTB) Logf(string, ...interface{}) {} -func (*TestingTB) Fatalf(string, ...interface{}) {} -func (*TestingTB) Name() string { return "TestingTB" } -func (*TestingTB) Helper() {} +func (*TestingTB) Failed() bool { return false } +func (*TestingTB) Logf(format string, v ...interface{}) { + log.Printf(format, v...) +} +func (*TestingTB) Fatalf(format string, v ...interface{}) { + log.Fatalf(format, v...) +} +func (*TestingTB) Name() string { return "TestingTB" } func (t *TestingTB) Cleanup(f func()) { t.Lock() defer t.Unlock() @@ -121,3 +125,4 @@ func (*TestingTB) Fatal(...any) {} func (*TestingTB) Log(...any) {} func (*TestingTB) Setenv(string, string) {} func (*TestingTB) TempDir() string { return "" } +func (*TestingTB) Helper() {} From 639548218305e2cafd10785bac6db48972583b99 Mon Sep 17 00:00:00 2001 From: William Deveaux Date: Tue, 9 Aug 2022 10:20:49 +0200 Subject: [PATCH 02/28] Corrects the telemetry config comments --- config/telemetry.go | 4 ++-- config/telemetry_test.go | 7 +++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/config/telemetry.go b/config/telemetry.go index af666d161..527cd0a4d 100644 --- a/config/telemetry.go +++ b/config/telemetry.go @@ -163,14 +163,14 @@ type TelemetryConfig struct { // by prefix. Use the 'prefix_filter' option and prefix rules with '+' to be // included. // - // hcl: telemetry { prefix_filter = []string{"+", "+", ...} } + // hcl: telemetry { allowed_prefixes = ["", "", ...] } AllowedPrefixes []string `json:"allowed_prefixes,omitempty" mapstructure:"allowed_prefixes"` // BlockedPrefixes is a list of filter rules to apply for blocking metrics // by prefix. Use the 'prefix_filter' option and prefix rules with '-' to be // excluded. // - // hcl: telemetry { prefix_filter = []string{"-", "-", ...} } + // hcl: telemetry { blocked_prefixes = ["", "", ...] } BlockedPrefixes []string `json:"blocked_prefixes,omitempty" mapstructure:"blocked_prefixes"` // MetricsPrefix is the prefix used to write stats values to. diff --git a/config/telemetry_test.go b/config/telemetry_test.go index fecaae746..8256cd2e2 100644 --- a/config/telemetry_test.go +++ b/config/telemetry_test.go @@ -11,6 +11,9 @@ func TestPromConfigParsing(t *testing.T) { configStr := "telemetry {" + "prometheus_port = 9110" + "prometheus_retention_time = \"120s\"" + + "allowed_prefixes = [\"keep\"]" + + "blocked_prefixes = [\"dont_keep\"]" + + "metrics_prefix = \"consul_template\"" + "}" config, err := Parse(configStr) @@ -18,6 +21,10 @@ func TestPromConfigParsing(t *testing.T) { require.Equal(t, 9110, config.Telemetry.PrometheusPort) require.Equal(t, 120*time.Second, config.Telemetry.PrometheusRetentionTime) + require.Equal(t, "consul_template", config.Telemetry.MetricsPrefix) + require.Equal(t, "consul_template", config.Telemetry.MetricsPrefix) + require.ElementsMatch(t, []string{"keep"}, config.Telemetry.AllowedPrefixes) + require.ElementsMatch(t, []string{"dont_keep"}, config.Telemetry.BlockedPrefixes) config.Finalize() require.Equal(t, 9110, config.Telemetry.PrometheusPort) From 2dce549140d9975232becd75f485ec8a4e5f16e7 Mon Sep 17 00:00:00 2001 From: William Deveaux Date: Tue, 9 Aug 2022 10:56:10 +0200 Subject: [PATCH 03/28] prevents crashes due to empty Telemetry config --- telemetry/telemetry.go | 18 ++++++++++++++++-- telemetry/telemetry_test.go | 8 ++++++++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/telemetry/telemetry.go b/telemetry/telemetry.go index 8d8766f08..a8b728d3c 100644 --- a/telemetry/telemetry.go +++ b/telemetry/telemetry.go @@ -7,6 +7,7 @@ package telemetry import ( "context" + "log" "net/http" "sync" "time" @@ -50,6 +51,8 @@ func ConfigureSinks(cfg *config.TelemetryConfig, memSink metrics.MetricSink) (me addSink := func(fn func(*config.TelemetryConfig, string) (metrics.MetricSink, error)) { s, err := fn(cfg, metricsConf.HostName) if err != nil { + log.Println("error!!!") + log.Println(err) errors = multierror.Append(errors, err) return } @@ -68,12 +71,22 @@ func ConfigureSinks(cfg *config.TelemetryConfig, memSink metrics.MetricSink) (me if len(sinks) > 0 { sinks = append(sinks, memSink) _, err := metrics.NewGlobal(metricsConf, sinks) - errors = multierror.Append(errors, err) + if err != nil { + errors = multierror.Append(errors, err) + } } else { metricsConf.EnableHostname = false _, err := metrics.NewGlobal(metricsConf, memSink) - errors = multierror.Append(errors, err) + if err != nil { + errors = multierror.Append(errors, err) + } } + + // if no errors where collected, the method should not return + /*if len(errors.Errors) == 0 { + errors = nil + }*/ + return sinks, errors } @@ -94,6 +107,7 @@ func Init(cfg *config.TelemetryConfig) (*Telemetry, error) { } _, errs := ConfigureSinks(cfg, memSink) + if errs != nil { return nil, errs } diff --git a/telemetry/telemetry_test.go b/telemetry/telemetry_test.go index 9b312992b..fd30d199e 100644 --- a/telemetry/telemetry_test.go +++ b/telemetry/telemetry_test.go @@ -125,3 +125,11 @@ func TestPrometheusMetrics(t *testing.T) { require.Emptyf(t, missingActualMetrics, "The following metrics are missing:\n - %s", strings.Join(missingActualMetrics, "\n - ")) } + +func TestInitWithEmptyConfig(t *testing.T) { + cfg := &config.Config{} + cfg.Finalize() + _, err := Init(cfg.Telemetry) + require.NoError(t, err) + +} From 1c1fc8759b6c0d776f21f985e081e49e428490e9 Mon Sep 17 00:00:00 2001 From: John Eikenberry Date: Mon, 9 Jan 2023 15:07:55 -0800 Subject: [PATCH 04/28] unmaintained badge --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 58dba1880..8d7336cc3 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,7 @@ --- # Consul Template +[![Project unmaintained](https://img.shields.io/badge/project-unmaintained-red.svg)](https://github.com/hashicorp/consul-template/issues/1698) [![build](https://github.com/hashicorp/consul-template/actions/workflows/build.yml/badge.svg)](https://github.com/hashicorp/consul-template/actions/workflows/build.yml) [![ci](https://github.com/hashicorp/consul-template/actions/workflows/ci.yml/badge.svg)](https://github.com/hashicorp/consul-template/actions/workflows/ci.yml) [![Go Documentation](http://img.shields.io/badge/go-documentation-%2300acd7)](https://godoc.org/github.com/hashicorp/consul-template) From a6b7020718cfd7d1e12921380ce33bd23ed91024 Mon Sep 17 00:00:00 2001 From: John Eikenberry Date: Tue, 10 Jan 2023 13:37:46 -0800 Subject: [PATCH 05/28] we're trying to figure out a maintanence story... stay tuned --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 8d7336cc3..58dba1880 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,6 @@ --- # Consul Template -[![Project unmaintained](https://img.shields.io/badge/project-unmaintained-red.svg)](https://github.com/hashicorp/consul-template/issues/1698) [![build](https://github.com/hashicorp/consul-template/actions/workflows/build.yml/badge.svg)](https://github.com/hashicorp/consul-template/actions/workflows/build.yml) [![ci](https://github.com/hashicorp/consul-template/actions/workflows/ci.yml/badge.svg)](https://github.com/hashicorp/consul-template/actions/workflows/ci.yml) [![Go Documentation](http://img.shields.io/badge/go-documentation-%2300acd7)](https://godoc.org/github.com/hashicorp/consul-template) From c2060d9f0dc5147ea10ff989cc9014f6b34ad849 Mon Sep 17 00:00:00 2001 From: William Deveaux Date: Thu, 22 Jun 2023 10:43:27 +0200 Subject: [PATCH 06/28] fixes README typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 58dba1880..680e66b58 100644 --- a/README.md +++ b/README.md @@ -410,7 +410,7 @@ $ consul-template -log-level debug ... ## Telemetry -Consul Template uses the [armon/go-metrics](https://github.com/armon/go-metrics) library to implement the Consul Template metric system. It currently supports metrics exported to circonus API, statsd server, statsite server, dogstatsd server, and prometheus endpoint. +Consul Template uses the [armon/go-metrics](https://github.com/armon/go-metrics) library to implement the Consul Template metrics system. It currently supports metrics exported to circonus API, statsd server, statsite server, dogstatsd server, and prometheus endpoint. ### Key Metrics From 20aae40e88c3092667d7c20de787a20c9c8552b2 Mon Sep 17 00:00:00 2001 From: William Deveaux Date: Thu, 22 Jun 2023 11:59:44 +0200 Subject: [PATCH 07/28] removes unused methods from config.convert --- config/convert.go | 67 ----------------------------------------------- 1 file changed, 67 deletions(-) diff --git a/config/convert.go b/config/convert.go index 74dd5acc3..bd1877cd2 100644 --- a/config/convert.go +++ b/config/convert.go @@ -25,15 +25,6 @@ func BoolVal(b *bool) bool { return *b } -// BoolCopy returns a copy of the boolean pointer -func BoolCopy(b *bool) *bool { - if b == nil { - return nil - } - - return Bool(*b) -} - // BoolGoString returns the value of the boolean for printing in a string. func BoolGoString(b *bool) string { if b == nil { @@ -124,46 +115,6 @@ func IntPresent(i *int) bool { return *i != 0 } -// Uint returns a pointer to the given uint. -func Uint(i uint) *uint { - return &i -} - -// UintVal returns the value of the uint at the pointer, or 0 if the pointer is -// nil. -func UintVal(i *uint) uint { - if i == nil { - return 0 - } - return *i -} - -// UintCopy returns a copy of the uint pointer -func UintCopy(i *uint) *uint { - if i == nil { - return nil - } - - return Uint(*i) -} - -// UintGoString returns the value of the uint for printing in a string. -func UintGoString(i *uint) string { - if i == nil { - return "(*uint)(nil)" - } - return fmt.Sprintf("%d", *i) -} - -// UintPresent returns a boolean indicating if the pointer is nil, or if the -// pointer is pointing to the zero value. -func UintPresent(i *uint) bool { - if i == nil { - return false - } - return *i != 0 -} - // Signal returns a pointer to the given os.Signal. func Signal(s os.Signal) *os.Signal { return &s @@ -211,15 +162,6 @@ func StringVal(s *string) string { return *s } -// StringCopy returns a copy of the string pointer -func StringCopy(s *string) *string { - if s == nil { - return nil - } - - return String(*s) -} - // StringGoString returns the value of the string for printing in a string. func StringGoString(s *string) string { if s == nil { @@ -250,15 +192,6 @@ func TimeDurationVal(t *time.Duration) time.Duration { return *t } -// TimeDurationCopy returns a copy of the time.Duration pointer -func TimeDurationCopy(t *time.Duration) *time.Duration { - if t == nil { - return nil - } - - return TimeDuration(*t) -} - // TimeDurationGoString returns the value of the time.Duration for printing in a // string. func TimeDurationGoString(t *time.Duration) string { From b6e1ca44cd02be2aac75eb9aed452713f400e817 Mon Sep 17 00:00:00 2001 From: William Deveaux Date: Thu, 22 Jun 2023 12:00:43 +0200 Subject: [PATCH 08/28] provides context for method TelemetryConfig.Merge --- config/telemetry.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/config/telemetry.go b/config/telemetry.go index 527cd0a4d..19b638fd7 100644 --- a/config/telemetry.go +++ b/config/telemetry.go @@ -240,6 +240,9 @@ func (c *TelemetryConfig) Copy() *TelemetryConfig { } } +// Merge combines all values in this configuration with the values in the other +// configuration, with values in the other configuration taking precedence. +// Maps and slices are merged, most other values are overwritten. func (c *TelemetryConfig) Merge(o *TelemetryConfig) *TelemetryConfig { if c == nil { if o == nil { From fbdfc8f4ba3da459f36112b9b6af860d08d0ddb2 Mon Sep 17 00:00:00 2001 From: William Deveaux Date: Thu, 22 Jun 2023 12:01:44 +0200 Subject: [PATCH 09/28] reformats metrics label computing in runner following @roncodingenthusiast's suggestion --- manager/runner.go | 40 ++++++++++++++++++++++++++-------------- 1 file changed, 26 insertions(+), 14 deletions(-) diff --git a/manager/runner.go b/manager/runner.go index 748ad1d67..7bd0f2d3e 100644 --- a/manager/runner.go +++ b/manager/runner.go @@ -669,32 +669,24 @@ func (r *Runner) Run() error { // Record that there is at least one new render event newRenderEvent = true - var label string - // Record that at least one template would have been rendered. if event.WouldRender { wouldRenderAny = true - label = "would" } // Record that at least one template was rendered. if event.DidRender { renderedAny = true - label = "rendered" } - if event.ForQuiescence { - label = "quiescence" - } + label := getTelemetryLabel(event) // Report the template render event - if label != "" { - telemetry.CounterTemplatesRendered.Add( - 1, - telemetry.NewLabel("id", tmpl.ID()), - telemetry.NewLabel("status", label), - ) - } + telemetry.CounterTemplatesRendered.Add( + 1, + telemetry.NewLabel("id", tmpl.ID()), + telemetry.NewLabel("status", label), + ) } } @@ -1526,3 +1518,23 @@ func recordDependencyCounts(deps map[string]dep.Dependency) { types[dep.Type()]++ } } + +func getTelemetryLabel(event *RenderEvent) string { + var label string + + // Record that at least one template would have been rendered. + if event.WouldRender { + label = "would" + } + + // Record that at least one template was rendered. + if event.DidRender { + label = "rendered" + } + + if event.ForQuiescence { + label = "quiescence" + } + + return label +} From b4445cecc69a2901412df61ce307baa061c2ade8 Mon Sep 17 00:00:00 2001 From: William Deveaux Date: Thu, 22 Jun 2023 12:02:26 +0200 Subject: [PATCH 10/28] simplifies method CounterMetric.Add --- telemetry/metrics.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/telemetry/metrics.go b/telemetry/metrics.go index 153d4e272..a893e6cd5 100644 --- a/telemetry/metrics.go +++ b/telemetry/metrics.go @@ -11,11 +11,7 @@ type CounterMetric struct { } func (m *CounterMetric) Add(val float32, labels ...metrics.Label) { - if len(labels) == 0 { - metrics.IncrCounter(m.Names, val) - } else { - metrics.IncrCounterWithLabels(m.Names, val, labels) - } + metrics.IncrCounterWithLabels(m.Names, val, labels) } // Counters From 30d31a33e9a34a93b91d30293bef05a8291f4232 Mon Sep 17 00:00:00 2001 From: William Deveaux Date: Thu, 22 Jun 2023 12:02:57 +0200 Subject: [PATCH 11/28] initializes all counters --- telemetry/metrics.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/telemetry/metrics.go b/telemetry/metrics.go index a893e6cd5..ae5899ec8 100644 --- a/telemetry/metrics.go +++ b/telemetry/metrics.go @@ -44,5 +44,8 @@ func NewLabel(name string, value string) metrics.Label { } func InitMetrics() { + CounterDependenciesReceived.Add(0) + CounterTemplatesRendered.Add(0) CounterActions.Add(0) + CounterCommandExecs.Add(0) } From 36b28196d214b7df53da64f25119c942cef4518d Mon Sep 17 00:00:00 2001 From: William Deveaux Date: Thu, 22 Jun 2023 12:03:33 +0200 Subject: [PATCH 12/28] removes debug logging --- telemetry/telemetry.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/telemetry/telemetry.go b/telemetry/telemetry.go index a8b728d3c..d8d6fc02d 100644 --- a/telemetry/telemetry.go +++ b/telemetry/telemetry.go @@ -7,7 +7,6 @@ package telemetry import ( "context" - "log" "net/http" "sync" "time" @@ -51,8 +50,6 @@ func ConfigureSinks(cfg *config.TelemetryConfig, memSink metrics.MetricSink) (me addSink := func(fn func(*config.TelemetryConfig, string) (metrics.MetricSink, error)) { s, err := fn(cfg, metricsConf.HostName) if err != nil { - log.Println("error!!!") - log.Println(err) errors = multierror.Append(errors, err) return } From 5cdbb6db1f9f6782d398fba2deceb687a242e378 Mon Sep 17 00:00:00 2001 From: William Deveaux Date: Thu, 22 Jun 2023 12:04:22 +0200 Subject: [PATCH 13/28] uses multierror for metrics sink configuration following @roncodingenthusiast's suggestion --- telemetry/telemetry.go | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/telemetry/telemetry.go b/telemetry/telemetry.go index d8d6fc02d..5aaa6dd56 100644 --- a/telemetry/telemetry.go +++ b/telemetry/telemetry.go @@ -46,7 +46,7 @@ func ConfigureSinks(cfg *config.TelemetryConfig, memSink metrics.MetricSink) (me metricsConf.BlockedPrefixes = cfg.BlockedPrefixes var sinks metrics.FanoutSink - var errors error + var errors *multierror.Error addSink := func(fn func(*config.TelemetryConfig, string) (metrics.MetricSink, error)) { s, err := fn(cfg, metricsConf.HostName) if err != nil { @@ -79,12 +79,7 @@ func ConfigureSinks(cfg *config.TelemetryConfig, memSink metrics.MetricSink) (me } } - // if no errors where collected, the method should not return - /*if len(errors.Errors) == 0 { - errors = nil - }*/ - - return sinks, errors + return sinks, errors.ErrorOrNil() } // Init configures go-metrics based on map of telemetry config From 4742f4b54d0df20ba9b23313cc1e96db7d6ab3fb Mon Sep 17 00:00:00 2001 From: William Deveaux Date: Thu, 22 Jun 2023 12:04:51 +0200 Subject: [PATCH 14/28] removes duplicate metrics sink addition --- telemetry/telemetry.go | 1 - 1 file changed, 1 deletion(-) diff --git a/telemetry/telemetry.go b/telemetry/telemetry.go index 5aaa6dd56..623f824c7 100644 --- a/telemetry/telemetry.go +++ b/telemetry/telemetry.go @@ -62,7 +62,6 @@ func ConfigureSinks(cfg *config.TelemetryConfig, memSink metrics.MetricSink) (me addSink(statsdSink) addSink(dogstatdSink) addSink(circonusSink) - addSink(circonusSink) addSink(PrometheusSink) if len(sinks) > 0 { From 4caed4fe8437373eacccbc049fc9d7f331103b5d Mon Sep 17 00:00:00 2001 From: William Deveaux Date: Fri, 23 Jun 2023 14:31:00 +0200 Subject: [PATCH 15/28] removes commented metrics call --- dependency/vault_common.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/dependency/vault_common.go b/dependency/vault_common.go index 553d33562..b28a98214 100644 --- a/dependency/vault_common.go +++ b/dependency/vault_common.go @@ -96,9 +96,6 @@ func renewSecret(clients *ClientSet, d renewer) error { case err := <-renewer.DoneCh(): if err != nil { log.Printf("[WARN] %s: failed to renew: %s", d, err) - //d.recordCounter("status", "stopped") - } else { - //d.recordCounter("status", "expired") } log.Printf("[WARN] %s: renewer done (maybe the lease expired)", d) return nil @@ -106,9 +103,7 @@ func renewSecret(clients *ClientSet, d renewer) error { log.Printf("[TRACE] %s: successfully renewed", d) printVaultWarnings(d, renewal.Secret.Warnings) updateSecret(secret, renewal.Secret) - //d.recordCounter("status", "renewed") case <-d.stopChan(): - //d.recordCounter("status", "stopped") return ErrStopped } } From 95f391de5ee39dbe98651d3b0a99ca05d79be7d8 Mon Sep 17 00:00:00 2001 From: William Deveaux Date: Mon, 26 Aug 2024 11:20:12 +0200 Subject: [PATCH 16/28] adds armon/go-metrics to allowed depencies for golantci-linter --- .golangci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.golangci.yml b/.golangci.yml index a46841f56..a16095ff0 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -63,6 +63,7 @@ linters-settings: # List of allowed packages. allow: - $gostd + - github.com/armon/go-metrics - github.com/BurntSushi/toml - github.com/Masterminds/sprig/v3 - github.com/davecgh/go-spew/spew @@ -90,4 +91,4 @@ linters-settings: run: timeout: 10m - concurrency: 4 \ No newline at end of file + concurrency: 4 From 5e7bb541afed7ead1485b3820e68f57fe3c768f2 Mon Sep 17 00:00:00 2001 From: William Deveaux Date: Mon, 26 Aug 2024 11:34:16 +0200 Subject: [PATCH 17/28] adds promclient dependency to golangci-lint --- .golangci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.golangci.yml b/.golangci.yml index a16095ff0..208a73c41 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -85,6 +85,7 @@ linters-settings: - github.com/mitchellh/hashstructure - github.com/mitchellh/mapstructure - github.com/pkg/errors + - github.com/prometheus/client_golang/prometheus - github.com/stretchr/testify/assert - github.com/stretchr/testify/require - github.com/coreos/go-systemd From 61e9218f5c48e5b3351f65b5baa63fc2f07bf99c Mon Sep 17 00:00:00 2001 From: William Deveaux Date: Fri, 11 Oct 2024 10:53:27 +0200 Subject: [PATCH 18/28] fix(config/telemetry): prevents displaying the circonus API token --- config/telemetry.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/config/telemetry.go b/config/telemetry.go index 19b638fd7..c228fffdf 100644 --- a/config/telemetry.go +++ b/config/telemetry.go @@ -335,6 +335,12 @@ func (c *TelemetryConfig) GoString() string { if c == nil { return "(*TelemetryConfig)(nil)" } + + circonusAPITokenState := "" + if c.CirconusAPIToken != "" { + circonusAPITokenState = "" + } + return fmt.Sprintf("&TelemetryConfig{"+ "Disable:%v, "+ "CirconusAPIApp:%s, "+ @@ -363,7 +369,7 @@ func (c *TelemetryConfig) GoString() string { "PrometheusRetentionTime:%s}", c.Disable, c.CirconusAPIApp, - c.CirconusAPIToken, + circonusAPITokenState, c.CirconusAPIURL, c.CirconusBrokerID, c.CirconusBrokerSelectTag, From 552ffaa222207dc779e6ef0250034c2d28d2078b Mon Sep 17 00:00:00 2001 From: William Deveaux Date: Fri, 11 Oct 2024 11:05:24 +0200 Subject: [PATCH 19/28] fix(config/telemetry): uses the correct type for 'CirconusCheckForceMetricActivation' --- config/telemetry.go | 13 ++++++++----- telemetry/circonus.go | 2 +- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/config/telemetry.go b/config/telemetry.go index c228fffdf..53b27ab5e 100644 --- a/config/telemetry.go +++ b/config/telemetry.go @@ -7,6 +7,7 @@ package config import ( "fmt" + "strconv" "time" ) @@ -92,7 +93,7 @@ type TelemetryConfig struct { // Default: "false" // // hcl: telemetry { circonus_check_metrics_activation = (true|false) - CirconusCheckForceMetricActivation string `json:"circonus_check_force_metric_activation,omitempty" mapstructure:"circonus_check_force_metric_activation"` + CirconusCheckForceMetricActivation bool `json:"circonus_check_force_metric_activation,omitempty" mapstructure:"circonus_check_force_metric_activation"` // CirconusCheckID is the check id (not check bundle id) from a previously created // HTTPTRAP check. The numeric portion of the check._cid field. @@ -276,9 +277,7 @@ func (c *TelemetryConfig) Merge(o *TelemetryConfig) *TelemetryConfig { if o.CirconusCheckDisplayName != "" { r.CirconusCheckDisplayName = o.CirconusCheckDisplayName } - if o.CirconusCheckForceMetricActivation != "" { - r.CirconusCheckForceMetricActivation = o.CirconusCheckForceMetricActivation - } + r.CirconusCheckForceMetricActivation = o.CirconusCheckForceMetricActivation if o.CirconusCheckID != "" { r.CirconusCheckID = o.CirconusCheckID } @@ -374,7 +373,7 @@ func (c *TelemetryConfig) GoString() string { c.CirconusBrokerID, c.CirconusBrokerSelectTag, c.CirconusCheckDisplayName, - c.CirconusCheckForceMetricActivation, + c.GetCirconusCheckForceMetricActivation(), c.CirconusCheckID, c.CirconusCheckInstanceID, c.CirconusCheckSearchTag, @@ -409,3 +408,7 @@ func (c *TelemetryConfig) Finalize() { c.PrometheusRetentionTime = 60 * time.Second } } + +func (c *TelemetryConfig) GetCirconusCheckForceMetricActivation() string { + return strconv.FormatBool(c.CirconusCheckForceMetricActivation) +} diff --git a/telemetry/circonus.go b/telemetry/circonus.go index 427dc9dbb..38a62b8f6 100644 --- a/telemetry/circonus.go +++ b/telemetry/circonus.go @@ -25,7 +25,7 @@ func circonusSink(cfg *config.TelemetryConfig, hostname string) (metrics.MetricS conf.CheckManager.API.URL = cfg.CirconusAPIURL conf.CheckManager.Check.SubmissionURL = url conf.CheckManager.Check.ID = cfg.CirconusCheckID - conf.CheckManager.Check.ForceMetricActivation = cfg.CirconusCheckForceMetricActivation + conf.CheckManager.Check.ForceMetricActivation = cfg.GetCirconusCheckForceMetricActivation() conf.CheckManager.Check.InstanceID = cfg.CirconusCheckInstanceID conf.CheckManager.Check.SearchTag = cfg.CirconusCheckSearchTag conf.CheckManager.Check.DisplayName = cfg.CirconusCheckDisplayName From 80287f937750313208602f6929e2714be6d45b90 Mon Sep 17 00:00:00 2001 From: William Deveaux Date: Fri, 11 Oct 2024 11:23:15 +0200 Subject: [PATCH 20/28] feat(config/telemetry): adds config merge tests --- config/telemetry_test.go | 95 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 95 insertions(+) diff --git a/config/telemetry_test.go b/config/telemetry_test.go index 8256cd2e2..4466bb724 100644 --- a/config/telemetry_test.go +++ b/config/telemetry_test.go @@ -4,6 +4,7 @@ import ( "testing" "time" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -30,3 +31,97 @@ func TestPromConfigParsing(t *testing.T) { require.Equal(t, 9110, config.Telemetry.PrometheusPort) require.Equal(t, 120*time.Second, config.Telemetry.PrometheusRetentionTime) } + +func TestTelemetryNilEmptyConfigMerge(t *testing.T) { + var nilConfig *TelemetryConfig + require.Nil(t, nilConfig.Merge(nil)) + + emptyConfig := &TelemetryConfig{} + require.Equal(t, emptyConfig, nilConfig.Merge(emptyConfig)) + require.Equal(t, emptyConfig, emptyConfig.Merge(nil)) +} + +func TestTelemetryPartialConfigMerge(t *testing.T) { + // Partial configuration merge test + partialConfigA := &TelemetryConfig{ + MetricsPrefix: "prefix", + Disable: true, + AllowedPrefixes: []string{"allowedPrefixA"}, + StatsdAddr: "statsA", + } + + partialConfigB := &TelemetryConfig{ + MetricsPrefix: "new_prefix", + Disable: false, + BlockedPrefixes: []string{"prefix"}, + } + + configC := partialConfigA.Merge(partialConfigB) + require.NotEqual(t, configC, partialConfigB) + + require.Equal(t, "new_prefix", configC.MetricsPrefix) + require.False(t, configC.Disable) + require.Equal(t, []string{"allowedPrefixA"}, configC.AllowedPrefixes) + require.Equal(t, []string{"prefix"}, configC.BlockedPrefixes) + require.Equal(t, "statsA", configC.StatsdAddr) +} + +func TestTelemetryFullConfigMerge(t *testing.T) { + configA := &TelemetryConfig{ + Disable: false, + CirconusAPIApp: "appA", + CirconusAPIToken: "tokenA", + CirconusAPIURL: "apiUrlA", + CirconusBrokerID: "brokerA", + CirconusBrokerSelectTag: "brokerTagA", + CirconusCheckDisplayName: "A", + CirconusCheckForceMetricActivation: false, + CirconusCheckID: "idA", + CirconusCheckInstanceID: "instanceA", + CirconusCheckSearchTag: "searchTagA", + CirconusCheckTags: "tagA", + CirconusSubmissionInterval: "1ms", + CirconusSubmissionURL: "urlA", + DisableHostname: false, + DogstatsdAddr: "addrA", + DogstatsdTags: []string{"dsTagA1", "dsTagA2"}, + FilterDefault: false, + AllowedPrefixes: []string{"allowedPrefixA"}, + BlockedPrefixes: []string{"blockedPrefixA"}, + MetricsPrefix: "prefixA", + StatsdAddr: "statsA", + StatsiteAddr: "statsiteA", + PrometheusPort: 8080, + PrometheusRetentionTime: 2 * time.Hour, + } + + configB := &TelemetryConfig{ + Disable: true, + CirconusAPIApp: "appB", + CirconusAPIToken: "tokenB", + CirconusAPIURL: "apiUrlB", + CirconusBrokerID: "brokerB", + CirconusBrokerSelectTag: "brokerTagB", + CirconusCheckDisplayName: "B", + CirconusCheckForceMetricActivation: true, + CirconusCheckID: "idB", + CirconusCheckInstanceID: "instanceB", + CirconusCheckSearchTag: "searchTagB", + CirconusCheckTags: "tagB", + CirconusSubmissionInterval: "1ms", + CirconusSubmissionURL: "urlB", + DisableHostname: true, + DogstatsdAddr: "addrB", + DogstatsdTags: []string{"dsTagB3"}, + FilterDefault: true, + AllowedPrefixes: []string{"allowedPrefixB"}, + BlockedPrefixes: []string{"blockedPrefixB"}, + MetricsPrefix: "prefixB", + StatsdAddr: "statsB", + StatsiteAddr: "statsiteB", + PrometheusPort: 9090, + PrometheusRetentionTime: 10 * time.Minute, + } + + assert.Equal(t, configB, configA.Merge(configB)) +} From 77625108caaea483fd1ccb2c1637c482e8cb43cc Mon Sep 17 00:00:00 2001 From: William Deveaux Date: Fri, 11 Oct 2024 11:25:55 +0200 Subject: [PATCH 21/28] feat(config/telemetry): adds gostring test for telemetry config --- config/telemetry_test.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/config/telemetry_test.go b/config/telemetry_test.go index 4466bb724..ab7701985 100644 --- a/config/telemetry_test.go +++ b/config/telemetry_test.go @@ -125,3 +125,12 @@ func TestTelemetryFullConfigMerge(t *testing.T) { assert.Equal(t, configB, configA.Merge(configB)) } + +func TestTelemetryConfigGoString(t *testing.T) { + config := &TelemetryConfig{ + PrometheusRetentionTime: 1 * time.Minute, + } + expected := "&TelemetryConfig{Disable:false, CirconusAPIApp:, CirconusAPIToken:, CirconusAPIURL:, CirconusBrokerID:, CirconusBrokerSelectTag:, CirconusCheckDisplayName:, CirconusCheckForceMetricActivation:false, CirconusCheckID:, CirconusCheckInstanceID:, CirconusCheckSearchTag:, CirconusCheckTags:, CirconusSubmissionInterval:, CirconusSubmissionURL:, DisableHostname:false, DogstatsdAddr:, DogstatsdTags:[], FilterDefault:false, AllowedPrefixes:[], BlockedPrefixes:[], MetricsPrefix:, StatsdAddr:, StatsiteAddr:, PrometheusPort:0, PrometheusRetentionTime:1m0s}" + + assert.Equal(t, expected, config.GoString()) +} From 37fc559eefd3a748496877e3278a7c91f99af36f Mon Sep 17 00:00:00 2001 From: William Deveaux Date: Fri, 11 Oct 2024 11:39:25 +0200 Subject: [PATCH 22/28] fix(dependency/type): makes use of switch-case for Type's String method --- dependency/dependency.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/dependency/dependency.go b/dependency/dependency.go index b443835a0..385012173 100644 --- a/dependency/dependency.go +++ b/dependency/dependency.go @@ -50,11 +50,18 @@ const ( ) func (t Type) String() string { - if t > 2 { + switch t { + case TypeConsul: + return "consul" + case TypeVault: + return "vault" + case TypeLocal: + return "local" + case TypeNomad: + return "nomad" + default: return "unknown" } - - return []string{"consul", "vault", "local"}[t] } // Dependency is an interface for a dependency that Consul Template is capable From 81d87ce1ad46e70672d60266cc96435d219f19f4 Mon Sep 17 00:00:00 2001 From: William Deveaux Date: Fri, 11 Oct 2024 11:46:42 +0200 Subject: [PATCH 23/28] fix(manager/runner): removes unneeded recordDependencyCounts method --- manager/runner.go | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/manager/runner.go b/manager/runner.go index 7bd0f2d3e..51b06ed22 100644 --- a/manager/runner.go +++ b/manager/runner.go @@ -693,10 +693,6 @@ func (r *Runner) Run() error { // Perform the diff and update the known dependencies. r.diffAndUpdateDeps(runCtx.depsMap) - // Record dependency counts on runCtx instead of runner.dependencies - // to avoid blocking the locks - recordDependencyCounts(runCtx.depsMap) - // Execute each command in sequence, collecting any errors that occur - this // ensures all commands execute at least once. var errs []error @@ -1512,13 +1508,6 @@ func newWatcher(c *config.Config, clients *dep.ClientSet) *watch.Watcher { }) } -func recordDependencyCounts(deps map[string]dep.Dependency) { - types := make(map[dep.Type]float32) - for _, dep := range deps { - types[dep.Type()]++ - } -} - func getTelemetryLabel(event *RenderEvent) string { var label string From c7b4679250e2def8e5f7c17ba142603ca6ed8a1d Mon Sep 17 00:00:00 2001 From: William Deveaux Date: Fri, 11 Oct 2024 11:49:10 +0200 Subject: [PATCH 24/28] fix(telemetry_test/formatting): applies @jm96441n 's recommendations regarding prefix check on array --- telemetry/telemetry_test.go | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/telemetry/telemetry_test.go b/telemetry/telemetry_test.go index fd30d199e..fd4f2a016 100644 --- a/telemetry/telemetry_test.go +++ b/telemetry/telemetry_test.go @@ -5,6 +5,7 @@ import ( "io" "net" "net/http" + "slices" "strconv" "strings" "testing" @@ -97,13 +98,9 @@ func TestPrometheusMetrics(t *testing.T) { missingActualMetrics := []string{} + prefixes := []string{"# HELP go_", "#TYPE go", "go_", "# HELP process_", "# TYPE process_", "process_"} for _, actualMetric := range actualMetrics { - if strings.HasPrefix(actualMetric, "# HELP go_") || - strings.HasPrefix(actualMetric, "# TYPE go_") || - strings.HasPrefix(actualMetric, "go_") || - strings.HasPrefix(actualMetric, "# HELP process_") || - strings.HasPrefix(actualMetric, "# TYPE process_") || - strings.HasPrefix(actualMetric, "process_") || + if slices.ContainsFunc(prefixes, func(p string) bool { return strings.HasPrefix(actualMetric, p) }) || actualMetric == "" { continue } From 1b4810c7269f305f58cda5e9e0df4764229a66b3 Mon Sep 17 00:00:00 2001 From: William Deveaux Date: Fri, 11 Oct 2024 12:16:22 +0200 Subject: [PATCH 25/28] fix(config/telemetry): corrects some circonus config json field --- config/telemetry.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/telemetry.go b/config/telemetry.go index 53b27ab5e..9dc32f43c 100644 --- a/config/telemetry.go +++ b/config/telemetry.go @@ -58,7 +58,7 @@ type TelemetryConfig struct { // Default: "https://api.circonus.com/v2" // // hcl: telemetry { circonus_api_url = string } - CirconusAPIURL string `json:"circonus_apiurl,omitempty" mapstructure:"circonus_apiurl"` + CirconusAPIURL string `json:"circonus_api_url,omitempty" mapstructure:"circonus_api_url"` // CirconusBrokerID is an explicit broker to use when creating a new check. The numeric portion // of broker._cid. If metric management is enabled and neither a Submission URL nor Check ID From 6dd0f566c5b0528ba52f91b7d02c2e509627e7f1 Mon Sep 17 00:00:00 2001 From: William Deveaux Date: Fri, 11 Oct 2024 12:17:04 +0200 Subject: [PATCH 26/28] feat(telemetry/doc): adds a telemetry config example in the configuration doc --- docs/configuration.md | 49 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/docs/configuration.md b/docs/configuration.md index 3f9092c20..2842a0865 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -735,3 +735,52 @@ exec { [consul-kv]: https://www.consul.io/docs/agent/kv.html "Consul KV" [nomad]: https://www.nomadproject.io/ "Nomad by HashiCorp" [vault]: https://www.vaultproject.io/ "Vault by HashiCorp" + +## Telemetry + +This block is an HCL mapping to OpenTelemetry configurations for +various exporters. Configuring telemetry is only supported in +configuration files and not as CLI flags. Only one metric provider can +be used at a given time. More details on the metrics collected can be found +in the Telemetry section. + +```hcl +# These are example configurations for monitoring Consul Template metrics. +telemetry { + # General configuration + disable_hostname = true + filter_default = true + allowed_prefixes = ["consul_template."] + blocked_prefixes = ["go."] + metrics_prefix = "consul_template" + + # If you use Circonus + circonus_api_app = "consul" + circonus_api_token = "Your token" + circonus_api_url = "https://api.circonus.com/v2" + circonus_broker_id = "Your broker ID" + circonus_broker_select_tag = "dc:sfo" + circonus_check_display_name = "Your chosen display name" + circonus_check_force_metric_activation = false + circonus_check_id = "Your check ID" + circonus_check_instance_id = "hostname:app" + circonus_check_search_tag = "service:consul" + circonus_check_tags = "service:consul" + circonus_submission_interval = "10s" + circonus_submission_url = "https://a.circonus.submission.url.com" + + # If you use DogStatsD + dogstatsd_addr = "udp://127.0.0.1:8125" + dogstatsd_tags = ["my_tag_name:my_tag_value"] + + # If you use statsd + statsd_address = "statsd.company.local:8125" + + # If you use statsite + statsite_address = "statsite.company.local:8125" + + # If you use Prometheus + prometheus_retention_time = "" + prometheus_port = 8888 +} +``` From 4ae8d022da221e2338ef9789f32235b1d006725a Mon Sep 17 00:00:00 2001 From: William Deveaux Date: Mon, 14 Oct 2024 17:34:17 +0200 Subject: [PATCH 27/28] fix(telemetry/init): extracts the Metrics server init in order to reuse it's shutdown function --- telemetry/circonus.go | 2 +- telemetry/prometheus.go | 2 +- telemetry/statsd.go | 2 +- telemetry/statsite.go | 2 +- telemetry/telemetry.go | 54 ++++++++++++++++--------------------- telemetry/telemetry_test.go | 4 +-- 6 files changed, 29 insertions(+), 37 deletions(-) diff --git a/telemetry/circonus.go b/telemetry/circonus.go index 38a62b8f6..a889678f7 100644 --- a/telemetry/circonus.go +++ b/telemetry/circonus.go @@ -11,7 +11,7 @@ import ( https://github.com/hashicorp/consul/blob/main/lib/telemetry.go#L274 */ -func circonusSink(cfg *config.TelemetryConfig, hostname string) (metrics.MetricSink, error) { +func circonusSink(cfg *config.TelemetryConfig, _ string) (metrics.MetricSink, error) { token := cfg.CirconusAPIToken url := cfg.CirconusSubmissionURL if token == "" && url == "" { diff --git a/telemetry/prometheus.go b/telemetry/prometheus.go index 367e1b1a9..10a9997e6 100644 --- a/telemetry/prometheus.go +++ b/telemetry/prometheus.go @@ -17,7 +17,7 @@ import ( https://github.com/hashicorp/consul/blob/main/lib/telemetry.go#L261 */ -func PrometheusSink(cfg *config.TelemetryConfig, hostname string) (metrics.MetricSink, error) { +func PrometheusSink(cfg *config.TelemetryConfig, _ string) (metrics.MetricSink, error) { if cfg.PrometheusPort == 0 { return nil, nil } diff --git a/telemetry/statsd.go b/telemetry/statsd.go index 8058f26a6..bac0d929b 100644 --- a/telemetry/statsd.go +++ b/telemetry/statsd.go @@ -10,7 +10,7 @@ import ( https://github.com/hashicorp/consul/blob/main/lib/telemetry.go#L240 */ -func statsdSink(cfg *config.TelemetryConfig, hostname string) (metrics.MetricSink, error) { +func statsdSink(cfg *config.TelemetryConfig, _ string) (metrics.MetricSink, error) { addr := cfg.StatsdAddr if addr == "" { return nil, nil diff --git a/telemetry/statsite.go b/telemetry/statsite.go index 0fda210dc..0088bdd0a 100644 --- a/telemetry/statsite.go +++ b/telemetry/statsite.go @@ -10,7 +10,7 @@ import ( https://github.com/hashicorp/consul/blob/main/lib/telemetry.go#L232 */ -func statsiteSink(cfg *config.TelemetryConfig, hostname string) (metrics.MetricSink, error) { +func statsiteSink(cfg *config.TelemetryConfig, _ string) (metrics.MetricSink, error) { addr := cfg.StatsiteAddr if addr == "" { return nil, nil diff --git a/telemetry/telemetry.go b/telemetry/telemetry.go index 623f824c7..99a933ac3 100644 --- a/telemetry/telemetry.go +++ b/telemetry/telemetry.go @@ -8,7 +8,6 @@ package telemetry import ( "context" "net/http" - "sync" "time" "github.com/armon/go-metrics" @@ -25,30 +24,29 @@ type MetricsHandler interface { type Telemetry struct { Handler MetricsHandler - mu sync.Mutex cancelFn context.CancelFunc } func (tel *Telemetry) Stop() { - tel.mu.Lock() - defer tel.mu.Unlock() - if tel.cancelFn != nil { tel.cancelFn() } } -func ConfigureSinks(cfg *config.TelemetryConfig, memSink metrics.MetricSink) (metrics.FanoutSink, error) { - metricsConf := metrics.DefaultConfig(cfg.MetricsPrefix) - metricsConf.EnableHostname = !cfg.DisableHostname - metricsConf.FilterDefault = cfg.FilterDefault - metricsConf.AllowedPrefixes = cfg.AllowedPrefixes - metricsConf.BlockedPrefixes = cfg.BlockedPrefixes +func computeMetricsConfig(telemetryConf *config.TelemetryConfig) *metrics.Config { + metricsConf := metrics.DefaultConfig(telemetryConf.MetricsPrefix) + metricsConf.EnableHostname = !telemetryConf.DisableHostname + metricsConf.FilterDefault = telemetryConf.FilterDefault + metricsConf.AllowedPrefixes = telemetryConf.AllowedPrefixes + metricsConf.BlockedPrefixes = telemetryConf.BlockedPrefixes + return metricsConf +} +func setupSinks(telemetryConf *config.TelemetryConfig, hostname string) (metrics.FanoutSink, error) { var sinks metrics.FanoutSink var errors *multierror.Error addSink := func(fn func(*config.TelemetryConfig, string) (metrics.MetricSink, error)) { - s, err := fn(cfg, metricsConf.HostName) + s, err := fn(telemetryConf, hostname) if err != nil { errors = multierror.Append(errors, err) return @@ -64,20 +62,6 @@ func ConfigureSinks(cfg *config.TelemetryConfig, memSink metrics.MetricSink) (me addSink(circonusSink) addSink(PrometheusSink) - if len(sinks) > 0 { - sinks = append(sinks, memSink) - _, err := metrics.NewGlobal(metricsConf, sinks) - if err != nil { - errors = multierror.Append(errors, err) - } - } else { - metricsConf.EnableHostname = false - _, err := metrics.NewGlobal(metricsConf, memSink) - if err != nil { - errors = multierror.Append(errors, err) - } - } - return sinks, errors.ErrorOrNil() } @@ -93,15 +77,23 @@ func Init(cfg *config.TelemetryConfig) (*Telemetry, error) { memSink := metrics.NewInmemSink(10*time.Second, time.Minute) metrics.DefaultInmemSignal(memSink) - telemetry := &Telemetry{ - Handler: memSink, - } - - _, errs := ConfigureSinks(cfg, memSink) + metricsConf := computeMetricsConfig(cfg) + sinks, errs := setupSinks(cfg, metricsConf.HostName) if errs != nil { return nil, errs } + sinks = append(sinks, memSink) + + metricsServer, err := metrics.NewGlobal(metricsConf, sinks) + if err != nil { + return nil, err + } + + telemetry := &Telemetry{ + Handler: memSink, + cancelFn: metricsServer.Shutdown, + } return telemetry, nil } diff --git a/telemetry/telemetry_test.go b/telemetry/telemetry_test.go index fd4f2a016..859ff207c 100644 --- a/telemetry/telemetry_test.go +++ b/telemetry/telemetry_test.go @@ -25,7 +25,7 @@ func newCfg() *config.TelemetryConfig { func TestConfigureSinks(t *testing.T) { cfg := newCfg() - sinks, err := ConfigureSinks(cfg, nil) + sinks, err := setupSinks(cfg, "") require.Error(t, err) // 3 sinks: statsd, statsite, inmem require.Equal(t, 3, len(sinks)) @@ -33,7 +33,7 @@ func TestConfigureSinks(t *testing.T) { cfg = &config.TelemetryConfig{ DogstatsdAddr: "", } - _, err = ConfigureSinks(cfg, nil) + _, err = setupSinks(cfg, "") require.NoError(t, err) } From c8f4f0fea934466563c05d07ba624009e4b71d92 Mon Sep 17 00:00:00 2001 From: William Deveaux Date: Mon, 14 Oct 2024 17:41:42 +0200 Subject: [PATCH 28/28] fix(telemetry/test): fixes typo in go metrics prefixes --- telemetry/telemetry_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/telemetry/telemetry_test.go b/telemetry/telemetry_test.go index 859ff207c..65f797785 100644 --- a/telemetry/telemetry_test.go +++ b/telemetry/telemetry_test.go @@ -28,7 +28,7 @@ func TestConfigureSinks(t *testing.T) { sinks, err := setupSinks(cfg, "") require.Error(t, err) // 3 sinks: statsd, statsite, inmem - require.Equal(t, 3, len(sinks)) + require.Equal(t, 2, len(sinks)) cfg = &config.TelemetryConfig{ DogstatsdAddr: "", @@ -98,7 +98,7 @@ func TestPrometheusMetrics(t *testing.T) { missingActualMetrics := []string{} - prefixes := []string{"# HELP go_", "#TYPE go", "go_", "# HELP process_", "# TYPE process_", "process_"} + prefixes := []string{"# HELP go_", "# TYPE go_", "go_", "# HELP process_", "# TYPE process_", "process_"} for _, actualMetric := range actualMetrics { if slices.ContainsFunc(prefixes, func(p string) bool { return strings.HasPrefix(actualMetric, p) }) || actualMetric == "" {