Merge pull request #478 from bittorrent/feat/gateway-cid-filter

Feat/gateway cid filter

core/commands/cidstore.go

@@ -0,0 +1,207 @@
+package commands
+
+import (
+	"fmt"
+	"io"
+	"strings"
+
+	cmds "github.com/bittorrent/go-btfs-cmds"
+	"github.com/bittorrent/go-btfs/core/commands/cmdenv"
+	"github.com/ipfs/go-datastore"
+	"github.com/ipfs/go-datastore/query"
+)
+
+const (
+	SizeOptionName  = "size"
+	batchOptionName = "batch"
+)
+
+const (
+	FilterKeyPrefix = "/gateway/filter/cid"
+)
+
+const (
+	cidSeparator = ","
+)
+
+var CidStoreCmd = &cmds.Command{
+	Helptext: cmds.HelpText{
+		Tagline:          "Manage cid stored in this node but don't want to be get by gateway api.",
+		ShortDescription: "Commands for generate, update, get and list access-keys stored in this node.",
+	},
+	Subcommands: map[string]*cmds.Command{
+		"add":  addCidCmd,
+		"del":  delCidCmd,
+		"get":  getCidCmd,
+		"has":  hasCidCmd,
+		"list": listCidCmd,
+	},
+	NoLocal: true,
+}
+
+var addCidCmd = &cmds.Command{
+	Helptext: cmds.HelpText{
+		Tagline: "Add cid to store.",
+	},
+	Options: []cmds.Option{
+		cmds.BoolOption(batchOptionName, "b", "batch add cids, cids split by , and all exits will be deleted").WithDefault(false),
+	},
+	Arguments: []cmds.Argument{
+		cmds.StringArg("cid", true, false, "cid to add to store"),
+	},
+	Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error {
+		nd, err := cmdenv.GetNode(env)
+		if err != nil {
+			return err
+		}
+
+		batch, _ := req.Options[batchOptionName].(bool)
+		if batch {
+			cids := strings.Split(req.Arguments[0], cidSeparator)
+			batch, err := nd.Repo.Datastore().Batch(req.Context)
+			if err != nil {
+				return cmds.EmitOnce(res, err.Error())
+			}
+
+			// delete all exits
+			results, err := nd.Repo.Datastore().Query(req.Context, query.Query{
+				Prefix: FilterKeyPrefix,
+			})
+			if err != nil {
+				return cmds.EmitOnce(res, err.Error())
+			}
+			for v := range results.Next() {
+				err = batch.Delete(req.Context, datastore.NewKey(NewGatewayFilterKey(string(v.Value))))
+				if err != nil {
+					return cmds.EmitOnce(res, err.Error())
+				}
+			}
+
+			for _, v := range cids {
+				err = batch.Put(req.Context, datastore.NewKey(NewGatewayFilterKey(v)), []byte(v))
+				if err != nil {
+					return cmds.EmitOnce(res, err.Error())
+				}
+			}
+			err = batch.Commit(req.Context)
+			if err != nil {
+				return cmds.EmitOnce(res, err.Error())
+			}
+			return cmds.EmitOnce(res, "Add batch ok.")
+		}
+
+		err = nd.Repo.Datastore().Put(req.Context, datastore.NewKey(NewGatewayFilterKey(req.Arguments[0])),
+			[]byte(req.Arguments[0]))
+		if err != nil {
+			return cmds.EmitOnce(res, err.Error())
+		}
+		return cmds.EmitOnce(res, "Add ok.")
+	},
+}
+
+var getCidCmd = &cmds.Command{
+	Helptext: cmds.HelpText{
+		Tagline: "Get cid from store.",
+	},
+	Arguments: []cmds.Argument{
+		cmds.StringArg("cid", true, false, "cid to add to store"),
+	},
+	Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error {
+		nd, err := cmdenv.GetNode(env)
+		if err != nil {
+			return err
+		}
+		v, err := nd.Repo.Datastore().Get(req.Context, datastore.NewKey(NewGatewayFilterKey(req.Arguments[0])))
+		if err != nil {
+			return cmds.EmitOnce(res, err.Error())
+		}
+		return cmds.EmitOnce(res, string(v))
+	},
+}
+
+var delCidCmd = &cmds.Command{
+	Helptext: cmds.HelpText{
+		Tagline: "Delete cid from store.",
+	},
+	Arguments: []cmds.Argument{
+		cmds.StringArg("cid", true, false, "cid to add to store"),
+	},
+	Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error {
+		nd, err := cmdenv.GetNode(env)
+		if err != nil {
+			return err
+		}
+		err = nd.Repo.Datastore().Delete(req.Context, datastore.NewKey(NewGatewayFilterKey(req.Arguments[0])))
+		if err != nil {
+			return cmds.EmitOnce(res, err.Error())
+		}
+		return cmds.EmitOnce(res, "Del ok.")
+	},
+}
+
+var hasCidCmd = &cmds.Command{
+	Helptext: cmds.HelpText{
+		Tagline: "Check cid exits in store",
+	},
+	Arguments: []cmds.Argument{
+		cmds.StringArg("cid", true, false, "cid to add to store"),
+	},
+	Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error {
+		nd, err := cmdenv.GetNode(env)
+		if err != nil {
+			return err
+		}
+		exits, err := nd.Repo.Datastore().Has(req.Context, datastore.NewKey(NewGatewayFilterKey(req.Arguments[0])))
+		if err != nil {
+			return err
+		}
+		if !exits {
+			return cmds.EmitOnce(res, "Cid not exits")
+		}
+		return cmds.EmitOnce(res, "Cid exits")
+	},
+}
+
+var listCidCmd = &cmds.Command{
+	Helptext: cmds.HelpText{
+		Tagline: "List all cids in store",
+	},
+	Options: []cmds.Option{
+		cmds.IntOption(SizeOptionName, "s", "Number of cids to return."),
+	},
+	Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error {
+		nd, err := cmdenv.GetNode(env)
+		if err != nil {
+			return err
+		}
+		size, _ := req.Options[SizeOptionName].(int)
+		results, err := nd.Repo.Datastore().Query(req.Context, query.Query{
+			Prefix: FilterKeyPrefix,
+			Limit:  size,
+		})
+		if err != nil {
+			return err
+		}
+		var resStr []string
+		for v := range results.Next() {
+			resStr = append(resStr, string(v.Value))
+		}
+		return cmds.EmitOnce(res, resStr)
+	},
+	Encoders: cmds.EncoderMap{
+		cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, cids []string) error {
+			for _, v := range cids {
+				_, err := w.Write([]byte(v + "\n"))
+				if err != nil {
+					return err
+				}
+			}
+			return nil
+		}),
+	},
+	Type: []string{},
+}
+
+func NewGatewayFilterKey(key string) string {
+	return fmt.Sprintf("%s/%s", FilterKeyPrefix, key)
+}

core/commands/commands_test.go

@@ -365,6 +365,12 @@ func TestCommands(t *testing.T) {
 		"/dashboard/logout",
 		"/dashboard/change",
 		"/dashboard/validate",
+		"/cidstore",
+		"/cidstore/add",
+		"/cidstore/get",
+		"/cidstore/has",
+		"/cidstore/del",
+		"/cidstore/list",
 	}
 
 	cmdSet := make(map[string]struct{})

core/commands/root.go

@@ -183,6 +183,7 @@ var rootSubcommands = map[string]*cmds.Command{
 	"encrypt":        encryptCmd,
 	"decrypt":        decryptCmd,
 	"dashboard":      dashboardCmd,
+	"cidstore":       CidStoreCmd,
 }
 
 // RootRO is the readonly version of Root

core/corehttp/corehttp.go

@@ -6,6 +6,7 @@ package corehttp
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"net"
 	"net/http"
@@ -53,16 +54,17 @@ func makeHandler(n *core.IpfsNode, l net.Listener, options ...ServeOption) (http
 		}
 
 		err := interceptorBeforeReq(r, n)
-		if err != nil {
-			// set allow origin
-			w.Header().Set("Access-Control-Allow-Origin", "*")
-			if r.Method == http.MethodOptions {
-				w.Header().Set("Access-Control-Allow-Origin", "*")
-				w.Header().Set("Access-Control-Allow-Headers", "Content-Type, X-Stream-Output, X-Chunked-Output, X-Content-Length")
+
+		if errors.Is(err, ErrGatewayCidExits) {
+			http.Error(w, "", http.StatusNotFound)
+			return
+		}
+
+		if errors.Is(err, ErrNotLogin) || errors.Is(err, ErrInvalidToken) || errors.Is(err, ErrTwoStepCheckErr) {
+			if r.Method != http.MethodOptions {
+				http.Error(w, err.Error(), http.StatusUnauthorized)
 				return
 			}
-			http.Error(w, err.Error(), http.StatusUnauthorized)
-			return
 		}
 
 		topMux.ServeHTTP(w, r)

core/corehttp/corehttp_interceptor.go

@@ -17,22 +17,38 @@ const defaultTwoStepDuration = 30 * time.Minute
 
 const firstStepUrl = "dashboard/validate"
 
+var (
+	ErrNotLogin        = errors.New("please login")
+	ErrInvalidToken    = errors.New("invalid token")
+	ErrTwoStepCheckErr = errors.New("please validate your password first")
+	ErrGatewayCidExits = errors.New("cid exits")
+)
+
 func interceptorBeforeReq(r *http.Request, n *core.IpfsNode) error {
 	config, err := n.Repo.Config()
 	if err != nil {
 		return err
 	}
 
 	if config.API.EnableTokenAuth {
-		err := tokenCheckInterceptor(r, n)
+		err = tokenCheckInterceptor(r, n)
+		if err != nil {
+			return err
+		}
+
+		err = twoStepCheckInterceptor(r)
 		if err != nil {
 			return err
 		}
 	}
 
-	err = twoStepCheckInterceptor(r)
-	if err != nil {
-		return err
+	exits, err := gatewayCidInterceptor(r, n)
+	if err != nil || !exits {
+		return nil
+	}
+
+	if exits {
+		return ErrGatewayCidExits
 	}
 
 	return nil
@@ -46,7 +62,7 @@ func twoStepCheckInterceptor(r *http.Request) error {
 		return nil
 	}
 
-	return errors.New("please validate your password first")
+	return ErrTwoStepCheckErr
 }
 
 func interceptorAfterResp(r *http.Request, w http.ResponseWriter, n *core.IpfsNode) error {
@@ -67,7 +83,7 @@ func tokenCheckInterceptor(r *http.Request, n *core.IpfsNode) error {
 		return nil
 	}
 	if !commands.IsLogin {
-		return fmt.Errorf("please login")
+		return ErrNotLogin
 	}
 	args := r.URL.Query()
 	token := args.Get("token")
@@ -80,12 +96,25 @@ func tokenCheckInterceptor(r *http.Request, n *core.IpfsNode) error {
 		return err
 	}
 	if claims.PeerId != n.Identity.String() {
-		return fmt.Errorf("token is invalid")
+		return ErrInvalidToken
 	}
 
 	return nil
 }
 
+func gatewayCidInterceptor(r *http.Request, n *core.IpfsNode) (bool, error) {
+	if filterGatewayUrl(r) {
+		sPath := strings.Split(r.URL.Path, "/")
+		if len(sPath) < 3 {
+			return false, nil
+		}
+		key := strings.Split(r.URL.Path, "/")[2]
+		exits, err := n.Repo.Datastore().Has(r.Context(), ds.NewKey(commands.NewGatewayFilterKey(key)))
+		return exits, err
+	}
+	return false, nil
+}
+
 func filterNoNeedTokenCheckReq(r *http.Request, apiHost string, peerId string) bool {
 	if filterUrl(r) || filterP2pSchema(r, peerId) || filterLocalShellApi(r, apiHost) || filterGatewayUrl(r) {
 		return true