Improve logging and add systemd healthchecks

go.mod

@@ -2,7 +2,10 @@ module github.com/nateinaction/pikvm-tailscale-cert-renewer
 
 go 1.22.5
 
-require tailscale.com v1.68.2
+require (
+	github.com/coreos/go-systemd/v22 v22.5.0
+	tailscale.com v1.68.2
+)
 
 require (
 	filippo.io/edwards25519 v1.1.0 // indirect

go.sum

@@ -6,12 +6,15 @@ github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa h1:LHTHcTQiSGT7V
 github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4=
 github.com/cilium/ebpf v0.12.3 h1:8ht6F9MquybnY97at+VDZb3eQQr8ev79RueWeVaEcG4=
 github.com/cilium/ebpf v0.12.3/go.mod h1:TctK1ivibvI3znr66ljgi4hqOT8EYQjz1KWBfb1UVgM=
+github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs=
+github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
 github.com/dblohm7/wingoes v0.0.0-20240119213807-a09d6be7affa h1:h8TfIT1xc8FWbwwpmHn1J5i43Y0uZP97GqasGCzSRJk=
 github.com/dblohm7/wingoes v0.0.0-20240119213807-a09d6be7affa/go.mod h1:Nx87SkVqTKd8UtT+xu7sM/l+LgXs6c0aHrlKusR+2EQ=
 github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
 github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
 github.com/fxamacker/cbor/v2 v2.5.0 h1:oHsG0V/Q6E/wqTS2O1Cozzsy69nqCiguo5Q1a1ADivE=
 github.com/fxamacker/cbor/v2 v2.5.0/go.mod h1:TA1xS00nchWmaBnEIxPSE5oHLuJBAVvqrtAnWBwBCVo=
+github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/hdevalence/ed25519consensus v0.2.0 h1:37ICyZqdyj0lAZ8P4D1d1id3HqbbG1N3iBb1Tb4rdcU=

internal/certmanager/certmanager.go

@@ -38,10 +38,14 @@ var (
 // CheckCert checks the cert and key files to see if they exist and match the tailscale cert
 func (c *CertManager) CheckCert(ctx context.Context) error {
 	if _, err := os.Stat(c.ssl.GetCertPath()); os.IsNotExist(err) {
+		slog.Warn("cert file does not exist", "path", c.ssl.GetCertPath())
+
 		return ErrCertDoesNotExist
 	}
 
 	if _, err := os.Stat(c.ssl.GetKeyPath()); os.IsNotExist(err) {
+		slog.Warn("key file does not exist", "path", c.ssl.GetKeyPath())
+
 		return ErrKeyDoesNotExist
 	}
 
@@ -60,11 +64,15 @@ func (c *CertManager) CheckCert(ctx context.Context) error {
 		return fmt.Errorf("failed to read key file: %w", err)
 	}
 
-	if reflect.DeepEqual(tsCert, fsCert) {
+	if !reflect.DeepEqual(tsCert, fsCert) {
+		slog.Warn("tailscale and filesystem certs do not match", "path", c.ssl.GetCertPath())
+
 		return ErrCertDoesNotMatch
 	}
 
-	if reflect.DeepEqual(tsKey, fsKey) {
+	if !reflect.DeepEqual(tsKey, fsKey) {
+		slog.Warn("tailscale and filesystem keys do not match", "path", c.ssl.GetCertPath())
+
 		return ErrKeyDoesNotMatch
 	}
 
@@ -98,9 +106,13 @@ func (c *CertManager) GenerateCert(ctx context.Context) error {
 		return fmt.Errorf("failed to write cert file: %w", err)
 	}
 
+	slog.Info("wrote cert file", "path", c.ssl.GetCertPath())
+
 	if err := os.WriteFile(c.ssl.GetKeyPath(), key, certFilePerms); err != nil {
 		return fmt.Errorf("failed to write key file: %w", err)
 	}
 
+	slog.Info("wrote key file", "path", c.ssl.GetKeyPath())
+
 	return nil
 }

internal/pikvm/fs.go

@@ -2,6 +2,7 @@ package pikvm
 
 import (
 	"fmt"
+	"log/slog"
 	"os/exec"
 )
 
@@ -12,6 +13,8 @@ func SetFSReadOnly() error {
 		return fmt.Errorf("failed enable read-only mode with output: %s: %w", out, err)
 	}
 
+	slog.Info("filesystem mode changed to read-only")
+
 	return nil
 }
 
@@ -22,5 +25,7 @@ func SetFSReadWrite() error {
 		return fmt.Errorf("failed enable read/write mode with output: %s: %w", out, err)
 	}
 
+	slog.Info("filesystem mode changed to read/write")
+
 	return nil
 }

internal/pikvm/nginx.go

@@ -2,6 +2,7 @@ package pikvm
 
 import (
 	"fmt"
+	"log/slog"
 	"os/exec"
 )
 
@@ -12,5 +13,7 @@ func RestartNginx() error {
 		return fmt.Errorf("failed restart kvmd-nginx with output: %s: %w", out, err)
 	}
 
+	slog.Info("kvmd-nginx restarted")
+
 	return nil
 }

internal/pikvm/sslconf.go

@@ -14,40 +14,45 @@ import (
 var (
 	certlineRegex = regexp.MustCompile(`^ssl_certificate\s+.*`)
 	keylineRegex  = regexp.MustCompile(`^ssl_certificate_key\s+.*`)
+
+	ErrNginxConfigMissingSSLDetails = fmt.Errorf("nginx config missing ssl details")
 )
 
 const (
 	nginxSSLConf      = "/etc/kvmd/nginx/ssl.conf"
 	nginxSSLConfPerms = 0o644
 )
 
-// SetCertsInNginxConfig sets the certificates in the nginx config file
-func SetCertsInNginxConfig(ssl *sslpaths.SSLPaths) error {
-	certLine := fmt.Sprintf("ssl_certificate %s;", ssl.GetCertPath())
-	keyLine := fmt.Sprintf("ssl_certificate_key %s;", ssl.GetKeyPath())
-
+// CheckNginxConfig checks the nginx config for the cert and key lines
+func CheckNginxConfig(ssl *sslpaths.SSLPaths) error {
 	b, err := os.ReadFile(nginxSSLConf)
 	if err != nil {
 		return fmt.Errorf("failed to read ssl.conf: %w", err)
 	}
 
 	lines := strings.Split(string(b), "\n")
 
-	if !slices.Contains(lines, certLine) || !slices.Contains(lines, keyLine) {
-		slog.Warn("cert or key line not found in nginx config, adding")
+	if !slices.Contains(lines, ssl.GetNginxConfigCertLine()) ||
+		!slices.Contains(lines, ssl.GetNginxConfigKeyLine()) {
+		slog.Warn("cert or key line not found in nginx config", "path", nginxSSLConf)
 
-		if err := writeNginxConfig(lines, certLine, keyLine); err != nil {
-			return fmt.Errorf("failed to write nginx config: %w", err)
-		}
+		return ErrNginxConfigMissingSSLDetails
 	}
 
 	return nil
 }
 
-// writeNginxConfig writes the cert and key lines to the nginx config
-func writeNginxConfig(lines []string, certLine string, keyLine string) error {
-	lines = setLine(lines, certlineRegex, certLine)
-	lines = setLine(lines, keylineRegex, keyLine)
+// WriteNginxConfig writes the cert and key lines to the nginx config
+func WriteNginxConfig(ssl *sslpaths.SSLPaths) error {
+	b, err := os.ReadFile(nginxSSLConf)
+	if err != nil {
+		return fmt.Errorf("failed to read ssl.conf: %w", err)
+	}
+
+	lines := strings.Split(string(b), "\n")
+
+	lines = setLine(lines, certlineRegex, ssl.GetNginxConfigCertLine())
+	lines = setLine(lines, keylineRegex, ssl.GetNginxConfigKeyLine())
 
 	if err := SetFSReadWrite(); err != nil {
 		return fmt.Errorf("failed filesystem mode change: %w", err)
@@ -60,9 +65,11 @@ func writeNginxConfig(lines []string, certLine string, keyLine string) error {
 	}()
 
 	if err := os.WriteFile(nginxSSLConf, []byte(strings.Join(lines, "\n")), nginxSSLConfPerms); err != nil {
-		return fmt.Errorf("failed to write ssl.conf: %w", err)
+		return fmt.Errorf("failed to write to nginx ssl config at %s: %w", nginxSSLConf, err)
 	}
 
+	slog.Info("wrote to nginx ssl config", "path", nginxSSLConf)
+
 	return nil
 }
 

internal/sslpaths/sslpaths.go

@@ -1,22 +1,31 @@
 package sslpaths
 
-import "path"
+import (
+	"fmt"
+	"path"
+)
 
 // SSLPaths is a struct that holds the paths to the SSL certificate and key
 type SSLPaths struct {
-	cert   string
-	dir    string
-	domain string
-	key    string
+	cert                string
+	dir                 string
+	domain              string
+	key                 string
+	nginxConfigCertLine string
+	nginxConfigKeyLine  string
 }
 
 func NewSSLPaths(dir, domain string) *SSLPaths {
-	return &SSLPaths{
+	sslP := &SSLPaths{
 		cert:   path.Join(dir, domain+".crt"),
 		dir:    dir,
 		domain: domain,
 		key:    path.Join(dir, domain+".key"),
 	}
+	sslP.nginxConfigCertLine = fmt.Sprintf("ssl_certificate %s;", sslP.GetCertPath())
+	sslP.nginxConfigKeyLine = fmt.Sprintf("ssl_certificate_key %s;", sslP.GetKeyPath())
+
+	return sslP
 }
 
 func (c *SSLPaths) GetCertPath() string {
@@ -34,3 +43,11 @@ func (c *SSLPaths) GetDir() string {
 func (c *SSLPaths) GetDomain() string {
 	return c.domain
 }
+
+func (c *SSLPaths) GetNginxConfigCertLine() string {
+	return c.nginxConfigCertLine
+}
+
+func (c *SSLPaths) GetNginxConfigKeyLine() string {
+	return c.nginxConfigKeyLine
+}

main.go

@@ -5,8 +5,10 @@ import (
 	"errors"
 	"fmt"
 	"log/slog"
+	"os"
 	"time"
 
+	systemd "github.com/coreos/go-systemd/v22/daemon"
 	"github.com/nateinaction/pikvm-tailscale-cert-renewer/internal/certmanager"
 	"github.com/nateinaction/pikvm-tailscale-cert-renewer/internal/pikvm"
 	"github.com/nateinaction/pikvm-tailscale-cert-renewer/internal/sslpaths"
@@ -20,12 +22,31 @@ const (
 func main() {
 	ctx := context.Background()
 
+	slog.Info("starting tailscale cert renewer")
+
+	systemdOk, err := systemd.SdNotify(false, systemd.SdNotifyReady)
+	if !systemdOk {
+		if err != nil {
+			slog.Error("failed to notify systemd, notify socket is unset")
+			os.Exit(1)
+		}
+
+		slog.Error("failed to notify systemd", "error", err)
+		os.Exit(1)
+	}
+
 	for {
+		systemdOk, err := systemd.SdNotify(false, systemd.SdNotifyWatchdog)
+		if !systemdOk {
+			if err != nil {
+				slog.Error("failed to notify systemd, notify socket is unset")
+			}
+
+			slog.Error("failed to notify systemd", "error", err)
+		}
+
 		if err := doCertCheckAndRenewal(ctx); err != nil {
 			slog.Error("failed to check or renew cert", "time_until_retry", timeToSleep, "error", err)
-			time.Sleep(timeToSleep)
-
-			continue
 		}
 
 		time.Sleep(timeToSleep)
@@ -42,24 +63,52 @@ func doCertCheckAndRenewal(ctx context.Context) error {
 
 	certManager := certmanager.NewCertManager(ssl)
 
-	if err := certManager.CheckCert(ctx); !errors.Is(err, certmanager.ErrCertDoesNotExist) &&
-		!errors.Is(err, certmanager.ErrKeyDoesNotExist) &&
-		!errors.Is(err, certmanager.ErrCertDoesNotMatch) &&
-		!errors.Is(err, certmanager.ErrKeyDoesNotMatch) {
-		return fmt.Errorf("failed to check cert: %w", err)
+	sslCertsChanged, err := changedSSLCerts(ctx, certManager)
+	if err != nil {
+		return fmt.Errorf("failed to check ssl certs: %w", err)
+	}
+
+	nginxConfigChanged, err := changedNginxConfig(ssl)
+	if err != nil {
+		return fmt.Errorf("failed to check nginx config: %w", err)
 	}
 
-	if err := certManager.GenerateCert(ctx); err != nil {
-		return fmt.Errorf("failed to generate cert: %w", err)
+	if sslCertsChanged || nginxConfigChanged {
+		if err := pikvm.RestartNginx(); err != nil {
+			return fmt.Errorf("failed to restart nginx: %w", err)
+		}
 	}
 
-	if err := pikvm.SetCertsInNginxConfig(ssl); err != nil {
-		return fmt.Errorf("failed to set certs in nginx config: %w", err)
+	return nil
+}
+
+func changedSSLCerts(ctx context.Context, certManager *certmanager.CertManager) (bool, error) {
+	if err := certManager.CheckCert(ctx); errors.Is(err, certmanager.ErrCertDoesNotExist) ||
+		errors.Is(err, certmanager.ErrKeyDoesNotExist) ||
+		errors.Is(err, certmanager.ErrCertDoesNotMatch) ||
+		errors.Is(err, certmanager.ErrKeyDoesNotMatch) {
+		if err := certManager.GenerateCert(ctx); err != nil {
+			return false, fmt.Errorf("failed to generate cert: %w", err)
+		}
+
+		return true, nil
+	} else if err != nil {
+		return false, fmt.Errorf("failed to check cert: %w", err)
 	}
 
-	if err := pikvm.RestartNginx(); err != nil {
-		return fmt.Errorf("failed to restart nginx: %w", err)
+	return false, nil
+}
+
+func changedNginxConfig(ssl *sslpaths.SSLPaths) (bool, error) {
+	if err := pikvm.CheckNginxConfig(ssl); errors.Is(err, pikvm.ErrNginxConfigMissingSSLDetails) {
+		if err := pikvm.WriteNginxConfig(ssl); err != nil {
+			return false, fmt.Errorf("failed to write nginx config: %w", err)
+		}
+
+		return true, nil
+	} else if err != nil {
+		return false, fmt.Errorf("failed to set certs in nginx config: %w", err)
 	}
 
-	return nil
+	return false, nil
 }

pikvm-tailscale-cert-renewer.service

@@ -3,9 +3,12 @@ Description=PiKVM Tailscale Certificate Renewer
 After=network.target
 
 [Service]
+Type=notify
 ExecStart=/usr/local/bin/pikvm-tailscale-cert-renewer
 Restart=always
 RestartSec=3
+TimeoutSec=5
+WatchdogSec=120
 User=root
 Group=root