From d63347b564ccf4763370086b8a688e24e93e6da6 Mon Sep 17 00:00:00 2001 From: frani Date: Mon, 14 Oct 2024 15:52:12 -0300 Subject: [PATCH 1/4] feat: add dockerize --- .dockerignore | 34 ++++++++++++++++++++++++++++++++++ .gitignore | 1 + Dockerfile | 40 ++++++++++++++++++++++++++++++++++++++++ compose.yaml | 23 +++++++++++++++++++++++ 4 files changed, 98 insertions(+) create mode 100644 .dockerignore create mode 100644 Dockerfile create mode 100644 compose.yaml diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..e1c7c69 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,34 @@ +# Include any files or directories that you don't want to be copied to your +# container here (e.g., local build artifacts, temporary files, etc.). +# +# For more help, visit the .dockerignore file reference guide at +# https://docs.docker.com/go/build-context-dockerignore/ + +**/.DS_Store +**/.classpath +**/.dockerignore +**/.env +**/.git +**/.gitignore +**/.project +**/.settings +**/.toolstarget +**/.vs +**/.vscode +**/*.*proj.user +**/*.dbmdl +**/*.jfm +**/bin +**/charts +**/docker-compose* +**/compose.y*ml +**/Dockerfile* +**/node_modules +**/npm-debug.log +**/obj +**/secrets.dev.yaml +**/values.dev.yaml +LICENSE +README.md + +/shove \ No newline at end of file diff --git a/.gitignore b/.gitignore index 61dc5e6..bcb403d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ *~ /cmd/shove/shove /shove +/redis diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..ecc596a --- /dev/null +++ b/Dockerfile @@ -0,0 +1,40 @@ +ARG GO_VERSION=1.21 +FROM --platform=$BUILDPLATFORM golang:${GO_VERSION} AS build +WORKDIR /src + +RUN --mount=type=cache,target=/go/pkg/mod/ \ + --mount=type=bind,source=go.sum,target=go.sum \ + --mount=type=bind,source=go.mod,target=go.mod \ + go mod download -x + +ARG TARGETARCH + +RUN --mount=type=cache,target=/go/pkg/mod/ \ + --mount=type=bind,target=. \ + CGO_ENABLED=0 GOARCH=$TARGETARCH go build -o /bin/server ./cmd/shove + +FROM alpine:latest AS final + +RUN --mount=type=cache,target=/var/cache/apk \ + apk --update add \ + ca-certificates \ + tzdata \ + && \ + update-ca-certificates + +ARG UID=10001 +RUN adduser \ + --disabled-password \ + --gecos "" \ + --home "/nonexistent" \ + --shell "/sbin/nologin" \ + --no-create-home \ + --uid "${UID}" \ + appuser +USER appuser + +COPY --from=build /bin/server /bin/ + +EXPOSE 8322 + +ENTRYPOINT [ "/bin/server" ] diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..f8f825c --- /dev/null +++ b/compose.yaml @@ -0,0 +1,23 @@ +services: + shove: + build: + context: . + target: final + container_name: shove + volumes: + - ./shove/webpush:/etc/shove/webpush + command: > + shove \ + -api-addr localhost:8322 \ + -queue-redis redis://redis:6379 \ + -webpush-vapid-keys-file /etc/shove/webpush/vapid-keys.json + depends_on: + - redis + ports: + - 8322:8322 + redis: + image: redis:7-alpine + volumes: + - ./shove/redis/data:/data + ports: + - 6379:6379 From a4f7cac2567856718506c89b50d784b532c59e65 Mon Sep 17 00:00:00 2001 From: frani Date: Mon, 14 Oct 2024 15:53:06 -0300 Subject: [PATCH 2/4] feat: add flag of read vapid json path --- README.md | 25 ++++++++----------------- cmd/shove/main.go | 35 +++++++++++++++++++++++++++++++++-- 2 files changed, 41 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 408d979..4afff12 100644 --- a/README.md +++ b/README.md @@ -9,11 +9,13 @@ This is the replacement for [Pulsus](https://github.com/pennersr/pulsus) which h ## Overview Design: + - Asynchronous: a push client can just fire & forget. - Multiple workers per push service. - Less moving parts: when using Redis, you can push directly to the queue, bypassing the need for the Shove server to be up and running. Supported push services: + - APNS - Email: supports automatic creation of email digests in case the rate limit is exceeded @@ -24,20 +26,19 @@ Supported push services: - Web Push Features: + - Feedback: asynchronously receive information on invalid device tokens. - Queueing: both in-memory and persistent via Redis. - Exponential back-off in case of failure. - Prometheus support. - Squashing of messages in case rate limits are exceeded. - ## Why? - https://github.com/appleboy/gorush/issues/386#issuecomment-479191179 - https://github.com/mercari/gaurun/issues/115 - ## Usage ### Running @@ -86,10 +87,11 @@ Usage: VAPID public key -webpush-vapid-public-key string VAPID public key + -webpush-vapid-keys-file string + VAPID keys file path -webpush-workers int The number of workers pushing Web messages (default 8) - Start the server: $ shove \ @@ -97,17 +99,15 @@ Start the server: -queue-redis redis://redis:6379 \ -fcm-credentials-file /etc/shove/fcm/credentials.json \ -apns-certificate-path /etc/shove/apns/production/bundle.pem -apns-sandbox-certificate-path /etc/shove/apns/sandbox/bundle.pem \ - -webpush-vapid-public-key=$VAPID_PUBLIC_KEY -webpush-vapid-private-key=$VAPID_PRIVATE_KEY \ + -webpush-vapid-key-file=/etc/shove/webpush/vapid-keys.json \ -telegram-bot-token $TELEGRAM_BOT_TOKEN - ### APNS Push an APNS notification: $ curl -i --data '{"service": "apns", "headers": {"apns-priority": 10, "apns-topic": "com.shove.app"}, "payload": {"aps": { "alert": "hi"}}, "token": "81b8ecff8cb6d22154404d43b9aeaaf6219dfbef2abb2fe313f3725f4505cb47"}' http://localhost:8322/api/push/apns - A successful push results in: HTTP/1.1 202 Accepted @@ -117,7 +117,6 @@ A successful push results in: OK - ### FCM Push an FCM notification: @@ -134,7 +133,6 @@ Or, post JSON: $ curl -i --data '{"url": "http://localhost:8000/api/webhook", "headers": {"foo": "bar"}, "data": {"hello": "world!"}}' http://localhost:8322/api/push/webhook - ### WebPush Push a WebPush notification: @@ -145,7 +143,6 @@ The subscription (serialized as a JSON string) is used for receiving feedback. Alternatively, you can specify an optional `token` parameter as done in the example above. - ### Telegram Push a Telegram notification: @@ -157,12 +154,10 @@ Shove requires strings to be passed. For users that disconnected from your bot the chat ID will be communicated back through the feedback mechanism. Here, the token will equal the unreachable chat ID. - ### Receive Feedback Outdated/invalid tokens are communicated back. To receive those, you can periodically query the feedback channel to receive token feedback, and remove those from your database: - $ curl -X POST 'http://localhost:8322/api/feedback' { @@ -173,7 +168,6 @@ Outdated/invalid tokens are communicated back. To receive those, you can periodi ] } - ### Email In order to keep your SMTP server safe from being blacklisted, the email service @@ -190,7 +184,7 @@ automatically digested. Push an email: - $ curl -i -X POST --data @./scripts/email.json http://localhost:8322/api/push/email + $ curl -i -X POST --data @./scripts/email.json http://localhost:8322/api/push/email If you send too many emails, you'll notice that they are digested, and at a later time, one digest mail is being sent: @@ -208,13 +202,12 @@ later time, one digest mail is being sent: 2021/03/23 21:16:12 email: Rate to john@doe.org exceeded, email digested 2021/03/23 21:16:18 email: Sending digest email - ### Redis Queues Shove is being used to push a high volume of notifications in a production environment, consisting of various microservices interacting together. In such a scenario, it is important that the various services are not too tightly coupled -to one another. For that purpose, Shove offers the ability to post +to one another. For that purpose, Shove offers the ability to post notifications directly to a Redis queue. Posting directly to the Redis queue, instead of using the HTTP service @@ -226,7 +219,6 @@ payloads being pushed, as they are mostly handed over as is to the upstream services. So, when using Shove this way, the client is responsible for handing over a raw payload. Here's an example: - package main import ( @@ -263,7 +255,6 @@ over a raw payload. Here's an example: } } - ## Status Used in production, over at: diff --git a/cmd/shove/main.go b/cmd/shove/main.go index a59899d..ff2b28f 100644 --- a/cmd/shove/main.go +++ b/cmd/shove/main.go @@ -2,6 +2,7 @@ package main import ( "context" + "encoding/json" "flag" "os" "os/signal" @@ -37,7 +38,8 @@ var redisURL = flag.String("queue-redis", "", "Use Redis queue (Redis URL)") var webhookWorkers = flag.Int("webhook-workers", 0, "The number of workers pushing Webhook messages") var webPushVAPIDPublicKey = flag.String("webpush-vapid-public-key", "", "VAPID public key") -var webPushVAPIDPrivateKey = flag.String("webpush-vapid-private-key", "", "VAPID public key") +var webPushVAPIDPrivateKey = flag.String("webpush-vapid-private-key", "", "VAPID private key") +var webPushVAPIDKeysFile = flag.String("webpush-vapid-keys-json", "", "JSON file containing VAPID keys") var webPushWorkers = flag.Int("webpush-workers", 8, "The number of workers pushing Web messages") var telegramBotToken = flag.String("telegram-bot-token", "", "Telegram bot token") @@ -140,7 +142,36 @@ func main() { } } - if *webPushVAPIDPrivateKey != "" { + // WebPush + // 1. If JSON file exists, read it and set both private and public key + // 2. If JSON file doesn't exist, check if both private and public keys are set + // 3. If one of option before happens, Create WebPush Service + // 4. Add to server + if *webPushVAPIDKeysFile != "" { + content, err := os.ReadFile(*webPushVAPIDKeysFile) + if err != nil { + slog.Error("Failed to read WebPush VAPID keys file", "error", err) + os.Exit(1) + } + var vapidKeys struct { + PublicKey string `json:"publicKey"` + PrivateKey string `json:"privateKey"` + } + err = json.Unmarshal(content, &vapidKeys) + if err != nil { + slog.Error("Failed to unmarshal WebPush VAPID keys file, please check your JSON file containt 'publicKey' and 'privatekey' keys", "error", err) + os.Exit(1) + } + web, err := webpush.NewWebPush(vapidKeys.PublicKey, vapidKeys.PrivateKey, newServiceLogger("webpush")) + if err != nil { + slog.Error("Failed to setup WebPush service", "error", err) + os.Exit(1) + } + if err := s.AddService(web, *webPushWorkers, services.SquashConfig{}); err != nil { + slog.Error("Failed to add WebPush service", "error", err) + os.Exit(1) + } + } else if *webPushVAPIDPrivateKey != "" { web, err := webpush.NewWebPush(*webPushVAPIDPublicKey, *webPushVAPIDPrivateKey, newServiceLogger("webpush")) if err != nil { slog.Error("Failed to setup WebPush service", "error", err) From da671dc8e3dd2f34f84df93241c44daed4bc681a Mon Sep 17 00:00:00 2001 From: frani Date: Mon, 14 Oct 2024 22:23:43 -0300 Subject: [PATCH 3/4] chore: - fix typo var name webPushVAPIDKeysJSON - priorise flag than file --- cmd/shove/main.go | 35 ++++++++++++++--------------------- 1 file changed, 14 insertions(+), 21 deletions(-) diff --git a/cmd/shove/main.go b/cmd/shove/main.go index ff2b28f..026b929 100644 --- a/cmd/shove/main.go +++ b/cmd/shove/main.go @@ -39,7 +39,7 @@ var webhookWorkers = flag.Int("webhook-workers", 0, "The number of workers pushi var webPushVAPIDPublicKey = flag.String("webpush-vapid-public-key", "", "VAPID public key") var webPushVAPIDPrivateKey = flag.String("webpush-vapid-private-key", "", "VAPID private key") -var webPushVAPIDKeysFile = flag.String("webpush-vapid-keys-json", "", "JSON file containing VAPID keys") +var webPushVAPIDKeysJSON = flag.String("webpush-vapid-keys-json", "", "JSON file containing VAPID keys") var webPushWorkers = flag.Int("webpush-workers", 8, "The number of workers pushing Web messages") var telegramBotToken = flag.String("telegram-bot-token", "", "Telegram bot token") @@ -142,13 +142,18 @@ func main() { } } - // WebPush - // 1. If JSON file exists, read it and set both private and public key - // 2. If JSON file doesn't exist, check if both private and public keys are set - // 3. If one of option before happens, Create WebPush Service - // 4. Add to server - if *webPushVAPIDKeysFile != "" { - content, err := os.ReadFile(*webPushVAPIDKeysFile) + if *webPushVAPIDPrivateKey != "" { + web, err := webpush.NewWebPush(*webPushVAPIDPublicKey, *webPushVAPIDPrivateKey, newServiceLogger("webpush")) + if err != nil { + slog.Error("Failed to setup WebPush service", "error", err) + os.Exit(1) + } + if err := s.AddService(web, *webPushWorkers, services.SquashConfig{}); err != nil { + slog.Error("Failed to add WebPush service", "error", err) + os.Exit(1) + } + } else if *webPushVAPIDKeysJSON != "" { + content, err := os.ReadFile(*webPushVAPIDKeysJSON) if err != nil { slog.Error("Failed to read WebPush VAPID keys file", "error", err) os.Exit(1) @@ -171,19 +176,7 @@ func main() { slog.Error("Failed to add WebPush service", "error", err) os.Exit(1) } - } else if *webPushVAPIDPrivateKey != "" { - web, err := webpush.NewWebPush(*webPushVAPIDPublicKey, *webPushVAPIDPrivateKey, newServiceLogger("webpush")) - if err != nil { - slog.Error("Failed to setup WebPush service", "error", err) - os.Exit(1) - } - if err := s.AddService(web, *webPushWorkers, services.SquashConfig{}); err != nil { - slog.Error("Failed to add WebPush service", "error", err) - os.Exit(1) - } - } - - if *telegramBotToken != "" { + } else if *telegramBotToken != "" { tg, err := telegram.NewTelegramService(*telegramBotToken, newServiceLogger("telegram")) if err != nil { slog.Error("Failed to setup Telegram service", "error", err) From 4d50db4577c94fd519f074a69e8a7f264631039d Mon Sep 17 00:00:00 2001 From: frani Date: Mon, 14 Oct 2024 22:24:25 -0300 Subject: [PATCH 4/4] chore: - update to go 1.23.1 - fix go build - use command to start container --- Dockerfile | 8 ++++---- compose.yaml | 17 ++++++++++++----- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/Dockerfile b/Dockerfile index ecc596a..bd69d0c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -ARG GO_VERSION=1.21 +ARG GO_VERSION=1.23.1 FROM --platform=$BUILDPLATFORM golang:${GO_VERSION} AS build WORKDIR /src @@ -11,7 +11,7 @@ ARG TARGETARCH RUN --mount=type=cache,target=/go/pkg/mod/ \ --mount=type=bind,target=. \ - CGO_ENABLED=0 GOARCH=$TARGETARCH go build -o /bin/server ./cmd/shove + CGO_ENABLED=0 GOARCH=$TARGETARCH go build -o /bin/shove ./cmd/shove/main.go FROM alpine:latest AS final @@ -33,8 +33,8 @@ RUN adduser \ appuser USER appuser -COPY --from=build /bin/server /bin/ +COPY --from=build /bin/shove /bin/shove EXPOSE 8322 -ENTRYPOINT [ "/bin/server" ] +CMD ["/bin/shove"] \ No newline at end of file diff --git a/compose.yaml b/compose.yaml index f8f825c..14dffcc 100644 --- a/compose.yaml +++ b/compose.yaml @@ -6,17 +6,24 @@ services: container_name: shove volumes: - ./shove/webpush:/etc/shove/webpush - command: > - shove \ - -api-addr localhost:8322 \ - -queue-redis redis://redis:6379 \ - -webpush-vapid-keys-file /etc/shove/webpush/vapid-keys.json + command: + [ + "shove", + "-api-addr", + "0.0.0.0:8322", + "-queue-redis", + "redis://redis:6379", + "-webpush-vapid-keys-json", + "/etc/shove/webpush/vapid-keys.json", + ] + depends_on: - redis ports: - 8322:8322 redis: image: redis:7-alpine + container_name: redis volumes: - ./shove/redis/data:/data ports: