Skip to content

Commit

Permalink
feat(altermanager): auto generate alertmanager config and reload if c…
Browse files Browse the repository at this point in the history
…hanged
  • Loading branch information
rmrf committed Jan 6, 2024
1 parent ccbbf11 commit 3d53134
Show file tree
Hide file tree
Showing 10 changed files with 231 additions and 117 deletions.
13 changes: 11 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,21 +6,30 @@ Simple SSL Certs Expiration Check
- programe will check these hosts ssl certs regulaly
- then expose the ssl certs expiration date as prometheus metrics
- all metrics with alert email as label
- auto generate alertmanager config file base on configuration
- docker-compose will start Prometheus/Alertmanager/Grafana for check and alert

## Building
## Building Binary

make build
./ssl-certs-check -config configurations/config.toml

### Docker
### Docker build

modify `docker-compose.yaml` ssl-certs-check env `ENV_GOPROXY`, then

docker-compose build

## Usage

docker-compose up

Then access:

- [alertmanager](http://localhost:9093/)
- [prometheus](http://localhost:9090/)
- [grafana](http://localhost:3000/)

## Metrics

| Metric | Meaning | Labels |
Expand Down
126 changes: 126 additions & 0 deletions config.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,126 @@
package main

import (
"os"

"github.com/BurntSushi/toml"
"go.uber.org/zap"
)

type Host struct {
Address string `toml:"address"`
AlertEmails []string `toml:"alert-emails"`
}
type Config struct {
ListenAddress string `toml:"listen-address"`
RefreshIntervalSecond int `toml:"refresh-interval-second"`
Concurrency int `toml:"concurrency"`
AlertManager AlertManager `toml:"alertmanager"`
Hosts []Host `toml:"hosts"`
}
type AlertManager struct {
ReloadURL string `toml:"reload-url"`
ConfigPath string `toml:"config-path"`
SMTPSmarthost string `toml:"smtp-smarthost"`
SMTPFrom string `toml:"smtp-from"`
SMTPUsername string `toml:"smtp-username"`
SMTPPassword string `toml:"smtp-password"`
}

type AlertManagerYaml struct {
Global struct {
SMTPSmarthost string `yaml:"smtp_smarthost"`
SMTPFrom string `yaml:"smtp_from"`
SMTPAuthUsername string `yaml:"smtp_auth_username"`
SMTPAuthPassword string `yaml:"smtp_auth_password"`
ResolveTimeout string `yaml:"resolve_timeout"`
} `yaml:"global"`
Templates []string `yaml:"templates"`
Route struct {
GroupBy []string `yaml:"group_by"`
GroupWait string `yaml:"group_wait"`
GroupInterval string `yaml:"group_interval"`
RepeatInterval string `yaml:"repeat_interval"`
Receiver string `yaml:"receiver"`
Routes []struct {
Matchers []string `yaml:"matchers"`
Receiver string `yaml:"receiver"`
} `yaml:"routes"`
} `yaml:"route"`
Receivers []struct {
Name string `yaml:"name"`
EmailConfigs []struct {
To string `yaml:"to"`
} `yaml:"email_configs,omitempty"`
} `yaml:"receivers"`
}

func parseConfig() Config {
var c Config
_, err := toml.DecodeFile(*configFile, &c)
if err != nil {
logger.Error("Could not Decode toml configuration file", zap.Error(err))
os.Exit(1)
}
return c
}

func genAlertManagerYaml(c Config) AlertManagerYaml {
var emailCount = make(map[string]int)

var a AlertManagerYaml
a.Global.ResolveTimeout = "1m"
a.Global.SMTPAuthPassword = c.AlertManager.SMTPPassword
a.Global.SMTPAuthUsername = c.AlertManager.SMTPUsername
a.Global.SMTPFrom = c.AlertManager.SMTPFrom
a.Global.SMTPSmarthost = c.AlertManager.SMTPSmarthost

a.Templates = []string{"/etc/alertmanager/template/*.tmpl"}

a.Route.GroupBy = []string{"alert_email"}
a.Route.GroupWait = "30s"
a.Route.GroupInterval = "5m"
a.Route.RepeatInterval = "24h"
a.Route.Receiver = "default"

for _, host := range c.Hosts {
for _, e := range host.AlertEmails {
emailCount[e]++
}
}

for email := range emailCount {
a.Route.Routes = append(a.Route.Routes, struct {
Matchers []string `yaml:"matchers"`
Receiver string `yaml:"receiver"`
}{
Matchers: []string{"alert_email=" + email},
Receiver: email,
})

a.Receivers = append(a.Receivers, struct {
Name string `yaml:"name"`
EmailConfigs []struct {
To string `yaml:"to"`
} `yaml:"email_configs,omitempty"`
}{
Name: email,
EmailConfigs: []struct {
To string `yaml:"to"`
}{
{
To: email,
},
},
})

}
a.Receivers = append(a.Receivers, struct {
Name string `yaml:"name"`
EmailConfigs []struct {
To string `yaml:"to"`
} `yaml:"email_configs,omitempty"`
}{Name: "default"})

return a
}
4 changes: 2 additions & 2 deletions configurations/alert_rules.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,9 @@ groups:
- name: 'ssl-certs-check-alert'
rules:
- alert: SSLCertsNearlyExpiration
expr: (exporter_cert_not_after{} - time())/3600/24 < 25
expr: round((exporter_cert_not_after{} - time())/3600/24) < 75
annotations:
title: 'SSL Certs Will expire after 25 days'
title: 'SSL Certs Will expire after {{ $value }} days'
description: ' Please kindly renew'
labels:
severity: 'critical'
60 changes: 0 additions & 60 deletions configurations/alertmanager.conf.example

This file was deleted.

8 changes: 7 additions & 1 deletion configurations/config-example.toml
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,13 @@ refresh-interval-second=3600

# how many workers to check host ssl certs
concurrency=5

[alertmanager]
reload-url="http://alertmanager:9093/-/reload"
config-path="configurations/alertmanager.conf"
smtp-smarthost=''
smtp-from=''
smtp-username=''
smtp-password=''

[[hosts]]
address = "www.supertechfans.com"
Expand Down
11 changes: 9 additions & 2 deletions docker-compose.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,9 @@ services:
image: local/ssl-certs-check
container_name: ssl-certs-check
volumes:
- ./config.toml:/app/config.toml
- ./configurations:/app/configurations
command:
- '-config=/app/config.toml'
- '-config=/app/configurations/config.toml'
ports:
- 8080:8080

Expand All @@ -29,6 +29,8 @@ services:
- '--config.file=/etc/alertmanager/alertmanager.conf'
ports:
- 9093:9093
depends_on:
- ssl-certs-check

prometheus:
container_name: prometheus
Expand All @@ -40,10 +42,13 @@ services:
- prometheus_data:/prometheus
command:
- '--config.file=/etc/prometheus/prometheus.yml'
- '--web.enable-lifecycle'
links:
- alertmanager:alertmanager
ports:
- 9090:9090
depends_on:
- ssl-certs-check

grafana:
container_name: grafana
Expand All @@ -55,3 +60,5 @@ services:
- grafana_data:/var/lib/grafana
ports:
- 3000:3000
depends_on:
- ssl-certs-check
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ require (
github.com/BurntSushi/toml v1.3.2
github.com/prometheus/client_golang v1.18.0
go.uber.org/zap v1.26.0
gopkg.in/yaml.v3 v3.0.1
)

require (
Expand Down
9 changes: 9 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,10 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg=
github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
Expand All @@ -22,6 +26,8 @@ github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lne
github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY=
github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo=
github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo=
github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk=
Expand All @@ -36,5 +42,8 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8=
google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
50 changes: 0 additions & 50 deletions main.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ import (
"syscall"
"time"

"github.com/BurntSushi/toml"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"
"go.uber.org/zap"
Expand All @@ -28,17 +27,6 @@ func init() {
logger, _ = zap.NewProduction()
}

type Host struct {
Address string `toml:"address"`
AlertEmails []string `toml:"alert-emails"`
}
type Config struct {
ListenAddress string `toml:"listen-address"`
RefreshIntervalSecond int `toml:"refresh-interval-second"`
Concurrency int `toml:"concurrency"`
Hosts []Host `toml:"hosts"`
}

var configFile = flag.String("config", "config.toml", "配置文件")

func NewGauge(name, help string, labels []string) *prometheus.GaugeVec {
Expand Down Expand Up @@ -86,41 +74,3 @@ func main() {

logger.Info("Bye")
}

func parseConfig() Config {
var c Config
_, err := toml.DecodeFile(*configFile, &c)
if err != nil {
logger.Error("Could not Decode toml configuration file", zap.Error(err))
os.Exit(1)
}
return c
}

func collectHosts(ctx context.Context) {
c := parseConfig()

for _, host := range c.Hosts {
hostQueue <- host
logger.Debug("collectHosts", zap.String("address", host.Address),
zap.Strings("emails", host.AlertEmails))
}
}

func runCollectHosts(ctx context.Context, interval time.Duration) {
ticker := time.NewTicker(interval)
defer ticker.Stop()

// start first, then will run with Ticker
go collectHosts(ctx)

for {
select {
case <-ticker.C:
go collectHosts(ctx)
case <-ctx.Done():
logger.Info("collectHosts ctx done")
return
}
}
}
Loading

0 comments on commit 3d53134

Please sign in to comment.