feat: s3-compatible-api - 1. add start option and configure; 2. optmi…

…ze providers interfaces and implements; 3. rewrite the server construct function

cmd/btfs/daemon.go

@@ -102,6 +102,7 @@ const (
 	chainID                   = "chain-id"
 	// apiAddrKwd    = "address-api"
 	// swarmAddrKwd  = "address-swarm"
+	enableS3CompatibleAPIKwd = "s3-compatible-api"
 )
 
 // BTFS daemon test exit error code
@@ -229,6 +230,7 @@ Headers.
 		// TODO: add way to override addresses. tricky part: updating the config if also --init.
 		// cmds.StringOption(apiAddrKwd, "Address for the daemon rpc API (overrides config)"),
 		// cmds.StringOption(swarmAddrKwd, "Address for the swarm socket (overrides config)"),
+		cmds.BoolOption(enableS3CompatibleAPIKwd, "Enable s3-compatible-api server"),
 	},
 	Subcommands: map[string]*cmds.Command{},
 	NoRemote:    true,
@@ -716,10 +718,15 @@ If the user need to start multiple nodes on the same machine, the configuration
 	}
 
 	// access-key init
-	accesskey.InitService(s3.GetProviders(statestore))
-	s3Server := s3.NewServer(statestore)
-	_ = s3Server.Start()
-	defer s3Server.Stop()
+	accesskey.InitService(s3.GetProviders())
+
+	// start s3-compatible-api server
+	s3OptEnable, s3Opt := req.Options[enableS3CompatibleAPIKwd].(bool)
+	if s3OptEnable || (!s3Opt && cfg.S3CompatibleAPI.Enable) {
+		s3Server := s3.NewServer(cfg.S3CompatibleAPI)
+		_ = s3Server.Start()
+		defer s3Server.Stop()
+	}
 
 	if SimpleMode == false {
 		// set Analytics flag if specified

go.mod

@@ -10,7 +10,7 @@ require (
 	github.com/bittorrent/go-btfs-chunker v0.4.0
 	github.com/bittorrent/go-btfs-cmds v0.3.0
 	github.com/bittorrent/go-btfs-common v0.9.0
-	github.com/bittorrent/go-btfs-config v0.12.3
+	github.com/bittorrent/go-btfs-config v0.13.0-pre2
 	github.com/bittorrent/go-btfs-files v0.3.1
 	github.com/bittorrent/go-btns v0.2.0
 	github.com/bittorrent/go-common/v2 v2.4.0

go.sum

@@ -207,8 +207,8 @@ github.com/bittorrent/go-btfs-cmds v0.3.0 h1:xpCBgk3zIm84Ne6EjeJgi8WLB5YJJUIFMjK
 github.com/bittorrent/go-btfs-cmds v0.3.0/go.mod h1:Fbac/Rou32G0jpoa6wLrNNDxcGOZbGfk+GiG0r3uEIU=
 github.com/bittorrent/go-btfs-common v0.9.0 h1:jHcFvYQmvmA4IdvVtkI5d/S/HW65Qz21C6oxeyK812w=
 github.com/bittorrent/go-btfs-common v0.9.0/go.mod h1:OG1n3DfcTxQYfLd5zco54LfL3IiDDaw3s7Igahu0Rj0=
-github.com/bittorrent/go-btfs-config v0.12.3 h1:Zi/GTwHo/PJV+90+w45P7axkWsUpOB/XFhgvNk+TwRs=
-github.com/bittorrent/go-btfs-config v0.12.3/go.mod h1:DNaHVC9wU84KLKoC4HkvdoFJKVZ7TF530qzfYu30fCI=
+github.com/bittorrent/go-btfs-config v0.13.0-pre2 h1:sneJ4a5bA15ST9WRUR4G+1FuGUVmszSEuihCeKzeyNk=
+github.com/bittorrent/go-btfs-config v0.13.0-pre2/go.mod h1:DNaHVC9wU84KLKoC4HkvdoFJKVZ7TF530qzfYu30fCI=
 github.com/bittorrent/go-btfs-files v0.3.0/go.mod h1:ylMf73m6oK94hL7VPblY1ZZpePsr6XbPV4BaNUwGZR0=
 github.com/bittorrent/go-btfs-files v0.3.1 h1:esq3j+6FtZ+SlaxKjVtiYgvXk/SWUiTcv0Q1MeJoPnQ=
 github.com/bittorrent/go-btfs-files v0.3.1/go.mod h1:ylMf73m6oK94hL7VPblY1ZZpePsr6XbPV4BaNUwGZR0=

s3/consts/consts.go

@@ -143,7 +143,18 @@ const (
 	ContentDisposition = "Content-Disposition"
 	Authorization      = "Authorization"
 	Action             = "Action"
+	XRequestWith       = "X-Requested-With"
 	Range              = "Range"
+	UserAgent          = "User-Agent"
+)
+
+// Standard HTTP cors headers
+const (
+	AccessControlAllowOrigin      = "Access-Control-Allow-Origin"
+	AccessControlAllowMethods     = "Access-Control-Allow-Methods"
+	AccessControlAllowHeaders     = "Access-Control-Allow-Headers"
+	AccessControlExposeHeaders    = "Access-Control-Expose-Headers"
+	AccessControlAllowCredentials = "Access-Control-Allow-Credentials"
 )
 
 // Standard BTFS HTTP response constants

s3/handlers/handlers.go

@@ -8,7 +8,6 @@ import (
 	"github.com/bittorrent/go-btfs/s3/responses"
 	"github.com/bittorrent/go-btfs/s3/services/accesskey"
 	"github.com/bittorrent/go-btfs/s3/services/bucket"
-	"github.com/bittorrent/go-btfs/s3/services/cors"
 	"github.com/bittorrent/go-btfs/s3/services/object"
 	"github.com/bittorrent/go-btfs/s3/services/sign"
 	"net/http"
@@ -22,29 +21,29 @@ const lockPrefix = "s3:lock/"
 var _ Handlerser = (*Handlers)(nil)
 
 type Handlers struct {
-	corsvc cors.Service
+	headers map[string][]string
+	nslock  ctxmu.MultiCtxRWLocker
+
 	acksvc accesskey.Service
 	sigsvc sign.Service
 	bucsvc bucket.Service
 	objsvc object.Service
-	nslock ctxmu.MultiCtxRWLocker
 }
 
 func NewHandlers(
-	corsvc cors.Service,
 	acksvc accesskey.Service,
 	sigsvc sign.Service,
 	bucsvc bucket.Service,
 	objsvc object.Service,
 	options ...Option,
 ) (handlers *Handlers) {
 	handlers = &Handlers{
-		corsvc: corsvc,
-		acksvc: acksvc,
-		sigsvc: sigsvc,
-		bucsvc: bucsvc,
-		objsvc: objsvc,
-		nslock: ctxmu.NewDefaultMultiCtxRWMutex(),
+		headers: defaultHeaders,
+		nslock:  ctxmu.NewDefaultMultiCtxRWMutex(),
+		acksvc:  acksvc,
+		sigsvc:  sigsvc,
+		bucsvc:  bucsvc,
+		objsvc:  objsvc,
 	}
 	for _, option := range options {
 		option(handlers)

s3/handlers/handlers_middlewares.go

@@ -4,20 +4,32 @@ import (
 	"errors"
 	"fmt"
 	"github.com/bittorrent/go-btfs/s3/cctx"
+	"github.com/bittorrent/go-btfs/s3/consts"
 	"github.com/bittorrent/go-btfs/s3/responses"
 	"github.com/bittorrent/go-btfs/s3/services/accesskey"
 	rscors "github.com/rs/cors"
 	"net/http"
 )
 
 func (h *Handlers) Cors(handler http.Handler) http.Handler {
-	return rscors.New(rscors.Options{
-		AllowedOrigins:   h.corsvc.GetAllowOrigins(),
-		AllowedMethods:   h.corsvc.GetAllowMethods(),
-		AllowedHeaders:   h.corsvc.GetAllowHeaders(),
-		ExposedHeaders:   h.corsvc.GetAllowHeaders(),
-		AllowCredentials: true,
-	}).Handler(handler)
+	headers := h.headers
+	cred := len(headers[consts.AccessControlAllowCredentials]) > 0 &&
+		headers[consts.AccessControlAllowCredentials][0] == "true"
+	ch := rscors.New(rscors.Options{
+		AllowedOrigins:   headers[consts.AccessControlAllowOrigin],
+		AllowedMethods:   headers[consts.AccessControlAllowMethods],
+		AllowedHeaders:   headers[consts.AccessControlExposeHeaders],
+		ExposedHeaders:   headers[consts.AccessControlAllowHeaders],
+		AllowCredentials: cred,
+	})
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		// add all user headers
+		for k, v := range h.headers {
+			w.Header()[k] = v
+		}
+		// next
+		ch.Handler(handler).ServeHTTP(w, r)
+	})
 }
 
 func (h *Handlers) Log(handler http.Handler) http.Handler {

s3/handlers/options.go

@@ -1,3 +1,63 @@
 package handlers
 
+import (
+	"github.com/bittorrent/go-btfs/s3/consts"
+	"net/http"
+)
+
+var defaultCorsMethods = []string{
+	http.MethodGet,
+	http.MethodPut,
+	http.MethodHead,
+	http.MethodPost,
+	http.MethodDelete,
+	http.MethodOptions,
+	http.MethodPatch,
+}
+
+var defaultCorsHeaders = []string{
+	consts.BTFSHash,
+	consts.Date,
+	consts.ETag,
+	consts.ServerInfo,
+	consts.Connection,
+	consts.AcceptRanges,
+	consts.ContentRange,
+	consts.ContentEncoding,
+	consts.ContentLength,
+	consts.ContentType,
+	consts.ContentMD5,
+	consts.ContentDisposition,
+	consts.LastModified,
+	consts.ContentLanguage,
+	consts.CacheControl,
+	consts.RetryAfter,
+	consts.AmzBucketRegion,
+	consts.Expires,
+	consts.Authorization,
+	consts.Action,
+	consts.XRequestWith,
+	consts.Range,
+	consts.UserAgent,
+	"X-Amz*",
+	"x-amz*",
+	"*",
+}
+
+var defaultHeaders = map[string][]string{
+	consts.AccessControlAllowOrigin:      []string{"*"},
+	consts.AccessControlAllowMethods:     defaultCorsMethods,
+	consts.AccessControlAllowHeaders:     defaultCorsHeaders,
+	consts.AccessControlExposeHeaders:    defaultCorsHeaders,
+	consts.AccessControlAllowCredentials: []string{"true"},
+}
+
 type Option func(handlers *Handlers)
+
+func WithHeaders(headers map[string][]string) Option {
+	return func(handlers *Handlers) {
+		if headers != nil {
+			handlers.headers = headers
+		}
+	}
+}

s3/providers/btfs_api.go

@@ -0,0 +1,41 @@
+package providers
+
+import (
+	"errors"
+	shell "github.com/bittorrent/go-btfs-api"
+	"io"
+)
+
+var _ FileStorer = (*BtfsAPI)(nil)
+
+type BtfsAPI struct {
+	shell *shell.Shell
+}
+
+func NewBtfsAPI(endpointUrl string) (api *BtfsAPI) {
+	api = &BtfsAPI{}
+	if endpointUrl == "" {
+		api.shell = shell.NewLocalShell()
+	} else {
+		api.shell = shell.NewShell(endpointUrl)
+	}
+	return
+}
+
+func (api *BtfsAPI) Store(r io.Reader) (id string, err error) {
+	id, err = api.shell.Add(r, shell.Pin(true))
+	return
+}
+
+func (api *BtfsAPI) Remove(id string) (err error) {
+	ok := api.shell.Remove(id)
+	if !ok {
+		err = errors.New("not removed")
+	}
+	return
+}
+
+func (api *BtfsAPI) Cat(id string) (rc io.ReadCloser, err error) {
+	rc, err = api.shell.Cat(id)
+	return
+}

s3/providers/proto.go

@@ -5,16 +5,20 @@ import (
 	"io"
 )
 
+var (
+	ErrStateStoreNotFound = errors.New("not found in state store")
+	ErrFileStoreNotFound  = errors.New("not found in file store")
+)
+
 type Providerser interface {
 	GetFileStore() FileStorer
 	GetStateStore() StateStorer
 }
 
 type FileStorer interface {
-	AddWithOpts(r io.Reader, pin bool, rawLeaves bool) (hash string, err error)
-	Remove(hash string) (removed bool)
-	Cat(path string) (readCloser io.ReadCloser, err error)
-	Unpin(path string) (err error)
+	Store(r io.Reader) (id string, err error)
+	Remove(id string) (err error)
+	Cat(id string) (readCloser io.ReadCloser, err error)
 }
 
 type StateStorer interface {
@@ -25,7 +29,3 @@ type StateStorer interface {
 }
 
 type StateStoreIterFunc func(key, value []byte) (stop bool, err error)
-
-var (
-	ErrStateStoreNotFound = errors.New("not found")
-)