From 3cbaa0ee15167d9fc54734ad04db1f6ada1db135 Mon Sep 17 00:00:00 2001 From: Nate Gay Date: Mon, 15 Jul 2024 21:05:17 -0500 Subject: [PATCH] Improve logging and add systemd healthchecks --- go.mod | 5 +- go.sum | 3 ++ internal/certmanager/certmanager.go | 16 +++++- internal/pikvm/fs.go | 5 ++ internal/pikvm/nginx.go | 3 ++ internal/pikvm/sslconf.go | 37 +++++++------ internal/sslpaths/sslpaths.go | 29 +++++++--- main.go | 79 ++++++++++++++++++++++------ pikvm-tailscale-cert-renewer.service | 3 ++ 9 files changed, 141 insertions(+), 39 deletions(-) diff --git a/go.mod b/go.mod index 0a0e40d..dc0fb9d 100644 --- a/go.mod +++ b/go.mod @@ -2,7 +2,10 @@ module github.com/nateinaction/pikvm-tailscale-cert-renewer go 1.22.5 -require tailscale.com v1.68.2 +require ( + github.com/coreos/go-systemd/v22 v22.5.0 + tailscale.com v1.68.2 +) require ( filippo.io/edwards25519 v1.1.0 // indirect diff --git a/go.sum b/go.sum index 041a4ad..cd7ca41 100644 --- a/go.sum +++ b/go.sum @@ -6,12 +6,15 @@ github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa h1:LHTHcTQiSGT7V github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4= github.com/cilium/ebpf v0.12.3 h1:8ht6F9MquybnY97at+VDZb3eQQr8ev79RueWeVaEcG4= github.com/cilium/ebpf v0.12.3/go.mod h1:TctK1ivibvI3znr66ljgi4hqOT8EYQjz1KWBfb1UVgM= +github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= +github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/dblohm7/wingoes v0.0.0-20240119213807-a09d6be7affa h1:h8TfIT1xc8FWbwwpmHn1J5i43Y0uZP97GqasGCzSRJk= github.com/dblohm7/wingoes v0.0.0-20240119213807-a09d6be7affa/go.mod h1:Nx87SkVqTKd8UtT+xu7sM/l+LgXs6c0aHrlKusR+2EQ= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fxamacker/cbor/v2 v2.5.0 h1:oHsG0V/Q6E/wqTS2O1Cozzsy69nqCiguo5Q1a1ADivE= github.com/fxamacker/cbor/v2 v2.5.0/go.mod h1:TA1xS00nchWmaBnEIxPSE5oHLuJBAVvqrtAnWBwBCVo= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/hdevalence/ed25519consensus v0.2.0 h1:37ICyZqdyj0lAZ8P4D1d1id3HqbbG1N3iBb1Tb4rdcU= diff --git a/internal/certmanager/certmanager.go b/internal/certmanager/certmanager.go index 9327cf6..d0be70b 100644 --- a/internal/certmanager/certmanager.go +++ b/internal/certmanager/certmanager.go @@ -38,10 +38,14 @@ var ( // CheckCert checks the cert and key files to see if they exist and match the tailscale cert func (c *CertManager) CheckCert(ctx context.Context) error { if _, err := os.Stat(c.ssl.GetCertPath()); os.IsNotExist(err) { + slog.Warn("cert file does not exist", "path", c.ssl.GetCertPath()) + return ErrCertDoesNotExist } if _, err := os.Stat(c.ssl.GetKeyPath()); os.IsNotExist(err) { + slog.Warn("key file does not exist", "path", c.ssl.GetKeyPath()) + return ErrKeyDoesNotExist } @@ -60,11 +64,15 @@ func (c *CertManager) CheckCert(ctx context.Context) error { return fmt.Errorf("failed to read key file: %w", err) } - if reflect.DeepEqual(tsCert, fsCert) { + if !reflect.DeepEqual(tsCert, fsCert) { + slog.Warn("tailscale and filesystem certs do not match", "path", c.ssl.GetCertPath()) + return ErrCertDoesNotMatch } - if reflect.DeepEqual(tsKey, fsKey) { + if !reflect.DeepEqual(tsKey, fsKey) { + slog.Warn("tailscale and filesystem keys do not match", "path", c.ssl.GetCertPath()) + return ErrKeyDoesNotMatch } @@ -98,9 +106,13 @@ func (c *CertManager) GenerateCert(ctx context.Context) error { return fmt.Errorf("failed to write cert file: %w", err) } + slog.Info("wrote cert file", "path", c.ssl.GetCertPath()) + if err := os.WriteFile(c.ssl.GetKeyPath(), key, certFilePerms); err != nil { return fmt.Errorf("failed to write key file: %w", err) } + slog.Info("wrote key file", "path", c.ssl.GetKeyPath()) + return nil } diff --git a/internal/pikvm/fs.go b/internal/pikvm/fs.go index bb9706a..14474d8 100644 --- a/internal/pikvm/fs.go +++ b/internal/pikvm/fs.go @@ -2,6 +2,7 @@ package pikvm import ( "fmt" + "log/slog" "os/exec" ) @@ -12,6 +13,8 @@ func SetFSReadOnly() error { return fmt.Errorf("failed enable read-only mode with output: %s: %w", out, err) } + slog.Info("filesystem mode changed to read-only") + return nil } @@ -22,5 +25,7 @@ func SetFSReadWrite() error { return fmt.Errorf("failed enable read/write mode with output: %s: %w", out, err) } + slog.Info("filesystem mode changed to read/write") + return nil } diff --git a/internal/pikvm/nginx.go b/internal/pikvm/nginx.go index 911f0a9..0b6c6c3 100644 --- a/internal/pikvm/nginx.go +++ b/internal/pikvm/nginx.go @@ -2,6 +2,7 @@ package pikvm import ( "fmt" + "log/slog" "os/exec" ) @@ -12,5 +13,7 @@ func RestartNginx() error { return fmt.Errorf("failed restart kvmd-nginx with output: %s: %w", out, err) } + slog.Info("kvmd-nginx restarted") + return nil } diff --git a/internal/pikvm/sslconf.go b/internal/pikvm/sslconf.go index 026d56a..21db446 100644 --- a/internal/pikvm/sslconf.go +++ b/internal/pikvm/sslconf.go @@ -14,6 +14,8 @@ import ( var ( certlineRegex = regexp.MustCompile(`^ssl_certificate\s+.*`) keylineRegex = regexp.MustCompile(`^ssl_certificate_key\s+.*`) + + ErrNginxConfigMissingSSLDetails = fmt.Errorf("nginx config missing ssl details") ) const ( @@ -21,11 +23,8 @@ const ( nginxSSLConfPerms = 0o644 ) -// SetCertsInNginxConfig sets the certificates in the nginx config file -func SetCertsInNginxConfig(ssl *sslpaths.SSLPaths) error { - certLine := fmt.Sprintf("ssl_certificate %s;", ssl.GetCertPath()) - keyLine := fmt.Sprintf("ssl_certificate_key %s;", ssl.GetKeyPath()) - +// CheckNginxConfig checks the nginx config for the cert and key lines +func CheckNginxConfig(ssl *sslpaths.SSLPaths) error { b, err := os.ReadFile(nginxSSLConf) if err != nil { return fmt.Errorf("failed to read ssl.conf: %w", err) @@ -33,21 +32,27 @@ func SetCertsInNginxConfig(ssl *sslpaths.SSLPaths) error { lines := strings.Split(string(b), "\n") - if !slices.Contains(lines, certLine) || !slices.Contains(lines, keyLine) { - slog.Warn("cert or key line not found in nginx config, adding") + if !slices.Contains(lines, ssl.GetNginxConfigCertLine()) || + !slices.Contains(lines, ssl.GetNginxConfigKeyLine()) { + slog.Warn("cert or key line not found in nginx config", "path", nginxSSLConf) - if err := writeNginxConfig(lines, certLine, keyLine); err != nil { - return fmt.Errorf("failed to write nginx config: %w", err) - } + return ErrNginxConfigMissingSSLDetails } return nil } -// writeNginxConfig writes the cert and key lines to the nginx config -func writeNginxConfig(lines []string, certLine string, keyLine string) error { - lines = setLine(lines, certlineRegex, certLine) - lines = setLine(lines, keylineRegex, keyLine) +// WriteNginxConfig writes the cert and key lines to the nginx config +func WriteNginxConfig(ssl *sslpaths.SSLPaths) error { + b, err := os.ReadFile(nginxSSLConf) + if err != nil { + return fmt.Errorf("failed to read ssl.conf: %w", err) + } + + lines := strings.Split(string(b), "\n") + + lines = setLine(lines, certlineRegex, ssl.GetNginxConfigCertLine()) + lines = setLine(lines, keylineRegex, ssl.GetNginxConfigKeyLine()) if err := SetFSReadWrite(); err != nil { return fmt.Errorf("failed filesystem mode change: %w", err) @@ -60,9 +65,11 @@ func writeNginxConfig(lines []string, certLine string, keyLine string) error { }() if err := os.WriteFile(nginxSSLConf, []byte(strings.Join(lines, "\n")), nginxSSLConfPerms); err != nil { - return fmt.Errorf("failed to write ssl.conf: %w", err) + return fmt.Errorf("failed to write to nginx ssl config at %s: %w", nginxSSLConf, err) } + slog.Info("wrote to nginx ssl config", "path", nginxSSLConf) + return nil } diff --git a/internal/sslpaths/sslpaths.go b/internal/sslpaths/sslpaths.go index 045369a..df022ae 100644 --- a/internal/sslpaths/sslpaths.go +++ b/internal/sslpaths/sslpaths.go @@ -1,22 +1,31 @@ package sslpaths -import "path" +import ( + "fmt" + "path" +) // SSLPaths is a struct that holds the paths to the SSL certificate and key type SSLPaths struct { - cert string - dir string - domain string - key string + cert string + dir string + domain string + key string + nginxConfigCertLine string + nginxConfigKeyLine string } func NewSSLPaths(dir, domain string) *SSLPaths { - return &SSLPaths{ + sslP := &SSLPaths{ cert: path.Join(dir, domain+".crt"), dir: dir, domain: domain, key: path.Join(dir, domain+".key"), } + sslP.nginxConfigCertLine = fmt.Sprintf("ssl_certificate %s;", sslP.GetCertPath()) + sslP.nginxConfigKeyLine = fmt.Sprintf("ssl_certificate_key %s;", sslP.GetKeyPath()) + + return sslP } func (c *SSLPaths) GetCertPath() string { @@ -34,3 +43,11 @@ func (c *SSLPaths) GetDir() string { func (c *SSLPaths) GetDomain() string { return c.domain } + +func (c *SSLPaths) GetNginxConfigCertLine() string { + return c.nginxConfigCertLine +} + +func (c *SSLPaths) GetNginxConfigKeyLine() string { + return c.nginxConfigKeyLine +} diff --git a/main.go b/main.go index ccae71a..f9fa809 100644 --- a/main.go +++ b/main.go @@ -5,8 +5,10 @@ import ( "errors" "fmt" "log/slog" + "os" "time" + systemd "github.com/coreos/go-systemd/v22/daemon" "github.com/nateinaction/pikvm-tailscale-cert-renewer/internal/certmanager" "github.com/nateinaction/pikvm-tailscale-cert-renewer/internal/pikvm" "github.com/nateinaction/pikvm-tailscale-cert-renewer/internal/sslpaths" @@ -20,12 +22,31 @@ const ( func main() { ctx := context.Background() + slog.Info("starting tailscale cert renewer") + + systemdOk, err := systemd.SdNotify(false, systemd.SdNotifyReady) + if !systemdOk { + if err != nil { + slog.Error("failed to notify systemd, notify socket is unset") + os.Exit(1) + } + + slog.Error("failed to notify systemd", "error", err) + os.Exit(1) + } + for { + systemdOk, err := systemd.SdNotify(false, systemd.SdNotifyWatchdog) + if !systemdOk { + if err != nil { + slog.Error("failed to notify systemd, notify socket is unset") + } + + slog.Error("failed to notify systemd", "error", err) + } + if err := doCertCheckAndRenewal(ctx); err != nil { slog.Error("failed to check or renew cert", "time_until_retry", timeToSleep, "error", err) - time.Sleep(timeToSleep) - - continue } time.Sleep(timeToSleep) @@ -42,24 +63,52 @@ func doCertCheckAndRenewal(ctx context.Context) error { certManager := certmanager.NewCertManager(ssl) - if err := certManager.CheckCert(ctx); !errors.Is(err, certmanager.ErrCertDoesNotExist) && - !errors.Is(err, certmanager.ErrKeyDoesNotExist) && - !errors.Is(err, certmanager.ErrCertDoesNotMatch) && - !errors.Is(err, certmanager.ErrKeyDoesNotMatch) { - return fmt.Errorf("failed to check cert: %w", err) + sslCertsChanged, err := changedSSLCerts(ctx, certManager) + if err != nil { + return fmt.Errorf("failed to check ssl certs: %w", err) + } + + nginxConfigChanged, err := changedNginxConfig(ssl) + if err != nil { + return fmt.Errorf("failed to check nginx config: %w", err) } - if err := certManager.GenerateCert(ctx); err != nil { - return fmt.Errorf("failed to generate cert: %w", err) + if sslCertsChanged || nginxConfigChanged { + if err := pikvm.RestartNginx(); err != nil { + return fmt.Errorf("failed to restart nginx: %w", err) + } } - if err := pikvm.SetCertsInNginxConfig(ssl); err != nil { - return fmt.Errorf("failed to set certs in nginx config: %w", err) + return nil +} + +func changedSSLCerts(ctx context.Context, certManager *certmanager.CertManager) (bool, error) { + if err := certManager.CheckCert(ctx); errors.Is(err, certmanager.ErrCertDoesNotExist) || + errors.Is(err, certmanager.ErrKeyDoesNotExist) || + errors.Is(err, certmanager.ErrCertDoesNotMatch) || + errors.Is(err, certmanager.ErrKeyDoesNotMatch) { + if err := certManager.GenerateCert(ctx); err != nil { + return false, fmt.Errorf("failed to generate cert: %w", err) + } + + return true, nil + } else if err != nil { + return false, fmt.Errorf("failed to check cert: %w", err) } - if err := pikvm.RestartNginx(); err != nil { - return fmt.Errorf("failed to restart nginx: %w", err) + return false, nil +} + +func changedNginxConfig(ssl *sslpaths.SSLPaths) (bool, error) { + if err := pikvm.CheckNginxConfig(ssl); errors.Is(err, pikvm.ErrNginxConfigMissingSSLDetails) { + if err := pikvm.WriteNginxConfig(ssl); err != nil { + return false, fmt.Errorf("failed to write nginx config: %w", err) + } + + return true, nil + } else if err != nil { + return false, fmt.Errorf("failed to set certs in nginx config: %w", err) } - return nil + return false, nil } diff --git a/pikvm-tailscale-cert-renewer.service b/pikvm-tailscale-cert-renewer.service index 564ca3c..8685c40 100644 --- a/pikvm-tailscale-cert-renewer.service +++ b/pikvm-tailscale-cert-renewer.service @@ -3,9 +3,12 @@ Description=PiKVM Tailscale Certificate Renewer After=network.target [Service] +Type=notify ExecStart=/usr/local/bin/pikvm-tailscale-cert-renewer Restart=always RestartSec=3 +TimeoutSec=5 +WatchdogSec=120 User=root Group=root