Store and retrieve files (for now images) to an s3 like object storage

api/api.go

@@ -12,6 +12,7 @@ import (
 	"github.com/vocdoni/saas-backend/account"
 	"github.com/vocdoni/saas-backend/db"
 	"github.com/vocdoni/saas-backend/notifications"
+	"github.com/vocdoni/saas-backend/objectstorage"
 	"github.com/vocdoni/saas-backend/stripe"
 	"github.com/vocdoni/saas-backend/subscriptions"
 	"go.vocdoni.io/dvote/apiclient"
@@ -40,6 +41,8 @@ type APIConfig struct {
 	StripeClient *stripe.StripeClient
 	// Subscriptions permissions manager
 	Subscriptions *subscriptions.Subscriptions
+	// Object storage
+	ObjectStorage *objectstorage.ObjectStorageClient
 }
 
 // API type represents the API HTTP server with JWT authentication capabilities.
@@ -57,6 +60,7 @@ type API struct {
 	transparentMode bool
 	stripe          *stripe.StripeClient
 	subscriptions   *subscriptions.Subscriptions
+	objectStorage   *objectstorage.ObjectStorageClient
 }
 
 // New creates a new API HTTP server. It does not start the server. Use Start() for that.
@@ -77,6 +81,7 @@ func New(conf *APIConfig) *API {
 		transparentMode: conf.FullTransparentMode,
 		stripe:          conf.StripeClient,
 		subscriptions:   conf.Subscriptions,
+		objectStorage:   conf.ObjectStorage,
 	}
 }
 
@@ -153,6 +158,9 @@ func (a *API) initRouter() http.Handler {
 		// pending organization invitations
 		log.Infow("new route", "method", "GET", "path", organizationPendingMembersEndpoint)
 		r.Get(organizationPendingMembersEndpoint, a.pendingOrganizationMembersHandler)
+		// upload an image to the object storage
+		log.Infow("new route", "method", "POST", "path", objectStorageUploadTypedEndpoint)
+		r.Post(objectStorageUploadTypedEndpoint, a.uploadObjectStorageWithOriginHandler)
 	})
 
 	// Public routes

api/docs.md

@@ -35,6 +35,8 @@
 - [🏦 Plans](#-plans)
   - [🛒 Get Available Plans](#-get-plans)
   - [🛍️ Get Plan Info](#-get-plan-info)
+- [ Storage](#-storage)
+  - [  Upload image from origin](#-upload-image-with-origin)
 
 </details>
 
@@ -916,3 +918,37 @@ This request can be made only by organization admins.
 | `400` | `40010` | `malformed URL parameter` |
 | `400` | `40023` | `plan not found` |
 | `500` | `50002` | `internal server error` |
+
+
+## Storage
+
+### Upload image with origin 
+
+* **Path** `/storage/{origin}`
+* **Method** `POST`
+
+Accepting files uploaded by forms as such:
+```html
+<form action="http://localhost:8000" method="post" enctype="multipart/form-data">
+  <p><input type="text" name="text" value="text default">
+  <p><input type="file" name="file1">
+  <p><input type="file" name="file2">
+  <p><button type="submit">Submit</button>
+</form>
+```
+
+* **Response**
+
+This methods uploads the images/files to 3rd party object storages and returns the URI where they are publicy available in inline mode.
+```json
+{
+  "urls": ["https://file1.store.com","https://file1.store.com"]
+}
+```
+
+* **Errors**
+
+| HTTP Status | Error code | Message |
+|:---:|:---:|:---|
+| `401` | `40001` | `user not authorized` |
+| `500` | `50002` | `internal server error` |

api/object_storage.go

@@ -0,0 +1,86 @@
+package api
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+
+	"golang.org/x/crypto/sha3"
+)
+
+func (a *API) uploadObjectStorageWithOriginHandler(w http.ResponseWriter, r *http.Request) {
+	// check if the user is authenticated
+	// get the user from the request context
+	user, ok := userFromContext(r.Context())
+	if !ok {
+		ErrUnauthorized.Write(w)
+		return
+	}
+	meta := map[string]string{"address": user.Email}
+	// get the file origin ("organization" or "election") from the request
+	origin := r.URL.Query().Get("origin")
+	if origin == "" {
+		origin = "organization"
+	} else if origin != "organization" && origin != "election" {
+		ErrMalformedURLParam.With("origin must be 'organization' or 'election'").Write(w)
+		return
+	}
+	// 32 MB is the default used by FormFile() function
+	if err := r.ParseMultipartForm(32 << 20); err != nil {
+		ErrGenericInternalServerError.With("could not parse form").Write(w)
+		return
+	}
+
+	// Get a reference to the fileHeaders.
+	// They are accessible only after ParseMultipartForm is called
+	files := r.MultipartForm.File["file"]
+	var returnURLs []string
+	for _, fileHeader := range files {
+		// Open the file
+		file, err := fileHeader.Open()
+		if err != nil {
+			ErrGenericInternalServerError.Withf("cannot open file %s", err.Error()).Write(w)
+			break
+		}
+		defer func() {
+			if err := file.Close(); err != nil {
+				ErrGenericInternalServerError.Withf("cannot close file %s", err.Error()).Write(w)
+				return
+			}
+		}()
+		buff := make([]byte, 512)
+		_, err = file.Read(buff)
+		if err != nil {
+			ErrGenericInternalServerError.Withf("cannot read file %s", err.Error()).Write(w)
+			break
+		}
+		// checking the content type
+		// so we don't allow files other than images
+		filetype := http.DetectContentType(buff)
+		if filetype != "image/jpeg" && filetype != "image/png" && filetype != "image/jpg" {
+			ErrGenericInternalServerError.With("The provided file format is not allowed. Please upload a JPEG,JPG or PNG image").Write(w)
+			break
+		}
+		// get the file extension
+		fileExtension := strings.Split(filetype, "/")[1]
+		_, err = file.Seek(0, io.SeekStart)
+		if err != nil {
+			ErrGenericInternalServerError.Withf("%s", err.Error()).Write(w)
+			break
+		}
+		// Calculate filename using
+		// the origin, the SHA3-256 hash of the file and the file extension
+		// ${origin}/${sha3-256(file)}.${fileExtension}
+		hash := fmt.Sprintf("%x", sha3.Sum224(buff))
+		filename := fmt.Sprintf("%s/%s.%s", origin, hash, fileExtension)
+		// upload the file using the object storage client
+		// and get the URL of the uploaded file
+		if _, err = a.objectStorage.Put(filename, "inline", filetype, fileHeader.Size, file, meta); err != nil {
+			ErrGenericInternalServerError.Withf("cannot upload file %s", err.Error()).Write(w)
+			break
+		}
+		returnURLs = append(returnURLs, filename)
+	}
+	httpWriteJSON(w, map[string][]string{"urls": returnURLs})
+}

api/routes.go

@@ -66,4 +66,8 @@ const (
 	planInfoEndpoint = "/plans/{planID}"
 	// POST /subscriptions/webhook to receive the subscription webhook from stripe
 	subscriptionsWebhook = "/subscriptions/webhook"
+
+	// object storage routes
+	// POST /storage/{origin}
+	objectStorageUploadTypedEndpoint = "/storage/{origin}"
 )

cmd/service/main.go

@@ -12,6 +12,7 @@ import (
 	"github.com/vocdoni/saas-backend/db"
 	"github.com/vocdoni/saas-backend/notifications/mailtemplates"
 	"github.com/vocdoni/saas-backend/notifications/smtp"
+	"github.com/vocdoni/saas-backend/objectstorage"
 	"github.com/vocdoni/saas-backend/stripe"
 	"github.com/vocdoni/saas-backend/subscriptions"
 	"go.vocdoni.io/dvote/apiclient"
@@ -37,6 +38,11 @@ func main() {
 	flag.String("emailFromName", "Vocdoni", "Email service from name")
 	flag.String("stripeApiSecret", "", "Stripe API secret")
 	flag.String("stripeWebhookSecret", "", "Stripe Webhook secret")
+	flag.String("storageApiKey", "", "Object storage API key")
+	flag.String("storageApiSecret", "", "Object storage API secret")
+	flag.String("storageApiEndpoint", "", "Object storage API endpoint")
+	flag.String("storageApiRegion", "", "Object storage API region name")
+	flag.String("storageApiBucket", "", "Object storage API bucket name")
 	// parse flags
 	flag.Parse()
 	// initialize Viper
@@ -67,6 +73,12 @@ func main() {
 	// stripe vars
 	stripeApiSecret := viper.GetString("stripeApiSecret")
 	stripeWebhookSecret := viper.GetString("stripeWebhookSecret")
+	// object storage vars
+	storageApiKey := viper.GetString("storageApiKey")
+	storageApiSecret := viper.GetString("storageApiSecret")
+	storageApiEndpoint := viper.GetString("storageApiEndpoint")
+	storageApiRegion := viper.GetString("storageApiRegion")
+	storageApiBucket := viper.GetString("storageApiBucket")
 
 	log.Init("debug", "stdout", os.Stderr)
 	// create Stripe client and include it in the API configuration
@@ -144,6 +156,12 @@ func main() {
 		DB: database,
 	})
 	apiConf.Subscriptions = subscriptions
+	// initialize the s3 like  object storage
+	if apiConf.ObjectStorage, err = objectstorage.New(
+		storageApiKey, storageApiSecret, storageApiEndpoint, storageApiRegion, storageApiBucket,
+	); err != nil {
+		log.Fatalf("could not create the object storage: %v", err)
+	}
 	// create the local API server
 	api.New(apiConf).Start()
 	log.Infow("server started", "host", host, "port", port)

example.env

@@ -6,5 +6,11 @@ VOCDONI_SMTPUSERNAME=admin
 VOCDONI_SMTPPASSWORD=password
 [email protected]
 [email protected]
-STRIPE_API_SECRET=stripe_key
-STRIPE_WEBHOOK_SECRET=stripe_webhook_key
+VOCDONI_STRIPEAPISECRET=test
+VOCDONI_STRIPEWEBHOOKSEC=test
+VOCDONI_WEBURL=test
+VOCDONI_STORAGEAPIKEY=test
+VOCDONI_STORAGEAPISECRET=test
+VOCDONI_STORAGEAPIENDPOI=test
+VOCDONI_STORAGEAPIREGION=test
+VOCDONI_STORAGEAPIBUCKET=test

go.mod

@@ -10,6 +10,7 @@ require (
 	github.com/go-chi/cors v1.2.1
 	github.com/go-chi/jwtauth/v5 v5.3.1
 	github.com/lestrrat-go/jwx/v2 v2.0.20
+	github.com/minio/minio-go/v7 v7.0.81
 	github.com/spf13/pflag v1.0.5
 	github.com/spf13/viper v1.18.2
 	github.com/stripe/stripe-go/v81 v81.0.0
@@ -93,14 +94,15 @@ require (
 	github.com/getsentry/sentry-go v0.18.0 // indirect
 	github.com/glendc/go-external-ip v0.1.0 // indirect
 	github.com/go-chi/chi v4.1.2+incompatible // indirect
+	github.com/go-ini/ini v1.67.0 // indirect
 	github.com/go-kit/kit v0.13.0 // indirect
 	github.com/go-kit/log v0.2.1 // indirect
 	github.com/go-logfmt/logfmt v0.6.0 // indirect
 	github.com/go-logr/logr v1.4.1 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-ole/go-ole v1.3.0 // indirect
 	github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
-	github.com/goccy/go-json v0.10.2 // indirect
+	github.com/goccy/go-json v0.10.3 // indirect
 	github.com/godbus/dbus/v5 v5.1.0 // indirect
 	github.com/gofrs/flock v0.8.1 // indirect
 	github.com/gofrs/uuid v4.4.0+incompatible // indirect
@@ -177,8 +179,8 @@ require (
 	github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect
 	github.com/jbenet/goprocess v0.1.4 // indirect
 	github.com/jmhodges/levigo v1.0.0 // indirect
-	github.com/klauspost/compress v1.17.8 // indirect
-	github.com/klauspost/cpuid/v2 v2.2.7 // indirect
+	github.com/klauspost/compress v1.17.11 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.8 // indirect
 	github.com/koron/go-ssdp v0.0.4 // indirect
 	github.com/kr/pretty v0.3.1 // indirect
 	github.com/kr/text v0.2.0 // indirect
@@ -223,6 +225,7 @@ require (
 	github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect
 	github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect
 	github.com/minio/highwayhash v1.0.2 // indirect
+	github.com/minio/md5-simd v1.1.2 // indirect
 	github.com/minio/sha256-simd v1.0.1 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
@@ -291,6 +294,7 @@ require (
 	github.com/rivo/uniseg v0.2.0 // indirect
 	github.com/rogpeppe/go-internal v1.10.0 // indirect
 	github.com/rs/cors v1.10.1 // indirect
+	github.com/rs/xid v1.6.0 // indirect
 	github.com/rs/zerolog v1.33.0 // indirect
 	github.com/sagikazarmark/locafero v0.4.0 // indirect
 	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
@@ -349,15 +353,15 @@ require (
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.27.0 // indirect
 	go4.org v0.0.0-20230225012048-214862532bf5 // indirect
-	golang.org/x/crypto v0.22.0 // indirect
+	golang.org/x/crypto v0.28.0 // indirect
 	golang.org/x/exp v0.0.0-20240325151524-a685a6edb6d8 // indirect
 	golang.org/x/mod v0.17.0 // indirect
-	golang.org/x/net v0.24.0 // indirect
+	golang.org/x/net v0.30.0 // indirect
 	golang.org/x/oauth2 v0.17.0 // indirect
-	golang.org/x/sync v0.7.0 // indirect
-	golang.org/x/sys v0.19.0 // indirect
-	golang.org/x/text v0.14.0 // indirect
-	golang.org/x/tools v0.20.0 // indirect
+	golang.org/x/sync v0.8.0 // indirect
+	golang.org/x/sys v0.26.0 // indirect
+	golang.org/x/text v0.19.0 // indirect
+	golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect
 	golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
 	gonum.org/v1/gonum v0.14.0 // indirect
 	google.golang.org/appengine v1.6.8 // indirect