From 080104819cc2f80f420d08971c46baf992691d06 Mon Sep 17 00:00:00 2001 From: Jon Hadfield Date: Sun, 3 Mar 2024 17:57:48 +0000 Subject: [PATCH] add ntfy notifications. --- go.mod | 2 +- go.sum | 4 +-- main.go | 25 ++++++-------- notify.go | 100 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 113 insertions(+), 18 deletions(-) create mode 100644 notify.go diff --git a/go.mod b/go.mod index 3f88bf9..c958c6e 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/hashicorp/go-retryablehttp v0.7.5 github.com/jonhadfield/githosts-utils v0.0.0-20240227215907-fdbfc9a27143 github.com/pkg/errors v0.9.1 - github.com/stretchr/testify v1.8.4 + github.com/stretchr/testify v1.9.0 gitlab.com/tozd/go/errors v0.8.1 gopkg.in/h2non/gock.v1 v1.1.2 ) diff --git a/go.sum b/go.sum index 5576ede..379a2fd 100644 --- a/go.sum +++ b/go.sum @@ -33,8 +33,8 @@ github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= gitlab.com/tozd/go/errors v0.8.1 h1:RfylffRAsl3PbDdHNUBEkTleTCiL/RIT+Ef8p0HRNCI= gitlab.com/tozd/go/errors v0.8.1/go.mod h1:PvIdUMLpPwxr+KEBxghQaCMydHXGYdJQn/PhdMqYREY= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/main.go b/main.go index 936754b..ecd1b1f 100644 --- a/main.go +++ b/main.go @@ -386,8 +386,6 @@ type BackupResults struct { } func execProviderBackups() { - var err error - startTime := time.Now() backupDir := os.Getenv(envGitBackupDir) @@ -434,7 +432,7 @@ func execProviderBackups() { f: time.RFC3339, } - _, failed := getBackupsStats(backupResults) + succeeded, failed := getBackupsStats(backupResults) if failed > 0 { logger.Println("backups completed with errors") @@ -442,19 +440,16 @@ func execProviderBackups() { logger.Println("backups complete") } - client := getHTTPClient(os.Getenv(envSobaLogLevel)) + notify(backupResults, succeeded, failed) - webHookURL := os.Getenv(envSobaWebHookURL) - if webHookURL != "" { - err = sendWebhook(client, sobaTime{ - Time: time.Now(), - f: time.RFC3339, - }, backupResults, os.Getenv(envSobaWebHookURL), os.Getenv(envSobaWebHookFormat)) - if err != nil { - logger.Printf("error sending webhook: %s", err) - } else { - logger.Println("webhook sent") - } + // help avoid thrashing provider apis if job auto-restarts + // after an early failure by adding delay if backup took less than 10 seconds + if time.Since(startTime) < time.Second*defaultEarlyErrorBackOffSeconds { + logger.Printf("backup took less than 10 seconds, "+ + "waiting %d seconds before next run to avoid thrashing"+ + " provider apis", defaultEarlyErrorBackOffSeconds) + + time.Sleep(time.Second * defaultEarlyErrorBackOffSeconds) } // help avoid thrashing provider apis if job auto-restartsrestarts diff --git a/notify.go b/notify.go new file mode 100644 index 0000000..d9d84bf --- /dev/null +++ b/notify.go @@ -0,0 +1,100 @@ +package main + +import ( + "fmt" + "net/http" + "net/url" + "os" + "strings" + "time" + + "github.com/hashicorp/go-retryablehttp" + "gitlab.com/tozd/go/errors" +) + +const ( + envSobaNtfyURL = "SOBA_NTFY_URL" +) + +func getResultsErrors(results BackupResults) []errors.E { + var errs []errors.E + + if results.Results == nil { + return nil + } + + for _, providerResults := range *results.Results { + errs = append(errs, providerResults.Results.Error) + } + + return errs +} + +func notify(backupResults BackupResults, succeeded int, failed int) { + // optimistic create retryable http client + httpClient := getHTTPClient(os.Getenv(envSobaLogLevel)) + + errs := getResultsErrors(backupResults) + + webHookURL := os.Getenv(envSobaWebHookURL) + if webHookURL != "" { + err := sendWebhook(httpClient, sobaTime{ + Time: time.Now(), + f: time.RFC3339, + }, backupResults, os.Getenv(envSobaWebHookURL), os.Getenv(envSobaWebHookFormat)) + if err != nil { + logger.Printf("error sending webhook: %s", err) + } else { + logger.Println("webhook sent") + } + } + + ntfyURL := os.Getenv(envSobaNtfyURL) + if ntfyURL != "" { + sendNtfy(httpClient, ntfyURL, succeeded, failed, errs) + } +} + +func sendNtfy(hc *retryablehttp.Client, nURL string, succeeded, failed int, errs []errors.E) { + nu, err := url.Parse(nURL) + if err != nil { + logger.Printf("ntfy failed to parse url: %v", err) + + return + } + + var req *retryablehttp.Request + + msg := fmt.Sprintf("completed: %d, failed: %d", + succeeded, failed) + + if len(errs) > 0 { + msg = fmt.Sprintf("%s\nerror: %s", msg, errs[0].Error()) + } + + req, err = retryablehttp.NewRequest(http.MethodPost, nu.String(), + strings.NewReader(msg)) + if err != nil { + logger.Printf("ntfy failed to create request: %v", err) + + return + } + + switch { + case succeeded > 0 && failed == 0: + req.Header.Set("Title", "🚀 soba backups succeeded") + case failed > 0 && succeeded > 0: + req.Header.Set("Title", "️⚠️ soba backups completed with errors") + default: + req.Header.Set("Title", "️🚨 soba backups failed") + } + + req.Header.Set("Tags", "soba,backup,git") + + _, err = hc.Do(req) + if err != nil { + logger.Printf("error: %s", err) + } + + logger.Println("ntfy publish sent") +}