From fc48a0d46da42af8ef9f528baba3171bfde1ca82 Mon Sep 17 00:00:00 2001 From: cody Date: Wed, 20 Nov 2024 11:54:40 +0800 Subject: [PATCH 1/5] feat: gateway cid filter --- core/commands/cidstore.go | 173 ++++++++++++++++++++++++++ core/commands/commands_test.go | 6 + core/commands/root.go | 1 + core/corehttp/corehttp.go | 5 + core/corehttp/corehttp_interceptor.go | 28 ++++- 5 files changed, 212 insertions(+), 1 deletion(-) create mode 100644 core/commands/cidstore.go diff --git a/core/commands/cidstore.go b/core/commands/cidstore.go new file mode 100644 index 000000000..10eb34b27 --- /dev/null +++ b/core/commands/cidstore.go @@ -0,0 +1,173 @@ +package commands + +import ( + "fmt" + "io" + + cmds "github.com/bittorrent/go-btfs-cmds" + "github.com/bittorrent/go-btfs/core/commands/cmdenv" + "github.com/ipfs/go-datastore" + "github.com/ipfs/go-datastore/query" +) + +const ( + FilterKeyPrefix = "/gateway/filter/cid" +) + +type cidList struct { + Strings []string +} + +var CidStoreCmd = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "Manage cid stored in this node but don't want to be get by gateway api.", + ShortDescription: "Commands for generate, update, get and list access-keys stored in this node.", + }, + Subcommands: map[string]*cmds.Command{ + "add": addCidCmd, + "del": delCidCmd, + "get": getCidCmd, + "has": hasCidCmd, + "list": listCidCmd, + }, + NoLocal: true, +} + +var addCidCmd = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "Add cid to store.", + }, + Options: []cmds.Option{ + cmds.BoolOption(trickleOptionName, "t", "Use trickle-dag format for dag generation."), + cmds.BoolOption(pinOptionName, "Pin this object when adding.").WithDefault(true), + }, + Arguments: []cmds.Argument{ + cmds.StringArg("cid", true, false, "cid to add to store"), + }, + Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { + nd, err := cmdenv.GetNode(env) + if err != nil { + return err + } + err = nd.Repo.Datastore().Put(req.Context, datastore.NewKey(NewGatewayFilterKey(req.Arguments[0])), + []byte(req.Arguments[0])) + if err != nil { + return cmds.EmitOnce(res, err.Error()) + } + return cmds.EmitOnce(res, "Add ok.") + }, +} + +var getCidCmd = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "Get cid from store.", + }, + Options: []cmds.Option{ + cmds.BoolOption(trickleOptionName, "t", "Use trickle-dag format for dag generation."), + cmds.BoolOption(pinOptionName, "Pin this object when adding.").WithDefault(true), + }, + Arguments: []cmds.Argument{ + cmds.StringArg("cid", true, false, "cid to add to store"), + }, + Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { + nd, err := cmdenv.GetNode(env) + if err != nil { + return err + } + v, err := nd.Repo.Datastore().Get(req.Context, datastore.NewKey(NewGatewayFilterKey(req.Arguments[0]))) + if err != nil { + return cmds.EmitOnce(res, err.Error()) + } + return cmds.EmitOnce(res, string(v)) + }, +} + +var delCidCmd = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "Delete cid from store.", + }, + Options: []cmds.Option{ + cmds.BoolOption(trickleOptionName, "t", "Use trickle-dag format for dag generation."), + cmds.BoolOption(pinOptionName, "Pin this object when adding.").WithDefault(true), + }, + Arguments: []cmds.Argument{ + cmds.StringArg("cid", true, false, "cid to add to store"), + }, + Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { + nd, err := cmdenv.GetNode(env) + if err != nil { + return err + } + err = nd.Repo.Datastore().Delete(req.Context, datastore.NewKey(NewGatewayFilterKey(req.Arguments[0]))) + if err != nil { + return cmds.EmitOnce(res, err.Error()) + } + return cmds.EmitOnce(res, "Del ok.") + }, +} + +var hasCidCmd = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "Check cid exits in store", + }, + Options: []cmds.Option{ + cmds.BoolOption(trickleOptionName, "t", "Use trickle-dag format for dag generation."), + cmds.BoolOption(pinOptionName, "Pin this object when adding.").WithDefault(true), + }, + Arguments: []cmds.Argument{ + cmds.StringArg("cid", true, false, "cid to add to store"), + }, + Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { + nd, err := cmdenv.GetNode(env) + if err != nil { + return err + } + exits, err := nd.Repo.Datastore().Has(req.Context, datastore.NewKey(NewGatewayFilterKey(req.Arguments[0]))) + if err != nil { + return err + } + if !exits { + return cmds.EmitOnce(res, "Cid not exits") + } + return cmds.EmitOnce(res, "Cid exits") + }, +} + +var listCidCmd = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "List all cids in store", + }, + Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { + nd, err := cmdenv.GetNode(env) + if err != nil { + return err + } + results, err := nd.Repo.Datastore().Query(req.Context, query.Query{ + Prefix: FilterKeyPrefix, + }) + if err != nil { + return err + } + var resStr []string + for v := range results.Next() { + resStr = append(resStr, string(v.Value)) + } + return cmds.EmitOnce(res, resStr) + }, + Encoders: cmds.EncoderMap{ + cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, cids []string) error { + for _, v := range cids { + _, err := w.Write([]byte(v + "\n")) + if err != nil { + return err + } + } + return nil + }), + }, + Type: []string{}, +} + +func NewGatewayFilterKey(key string) string { + return fmt.Sprintf("%s/%s", FilterKeyPrefix, key) +} diff --git a/core/commands/commands_test.go b/core/commands/commands_test.go index e8a899036..dba7d1982 100644 --- a/core/commands/commands_test.go +++ b/core/commands/commands_test.go @@ -364,6 +364,12 @@ func TestCommands(t *testing.T) { "/dashboard/logout", "/dashboard/change", "/dashboard/validate", + "/cidstore", + "/cidstore/add", + "/cidstore/get", + "/cidstore/has", + "/cidstore/del", + "/cidstore/list", } cmdSet := make(map[string]struct{}) diff --git a/core/commands/root.go b/core/commands/root.go index 40192145c..7170b8ec0 100644 --- a/core/commands/root.go +++ b/core/commands/root.go @@ -183,6 +183,7 @@ var rootSubcommands = map[string]*cmds.Command{ "encrypt": encryptCmd, "decrypt": decryptCmd, "dashboard": dashboardCmd, + "cidstore": CidStoreCmd, } // RootRO is the readonly version of Root diff --git a/core/corehttp/corehttp.go b/core/corehttp/corehttp.go index 5b1dae616..eb69b6a6a 100644 --- a/core/corehttp/corehttp.go +++ b/core/corehttp/corehttp.go @@ -6,6 +6,7 @@ package corehttp import ( "context" + "errors" "fmt" "net" "net/http" @@ -53,6 +54,10 @@ func makeHandler(n *core.IpfsNode, l net.Listener, options ...ServeOption) (http } err := interceptorBeforeReq(r, n) + if errors.Is(err, ErrorGatewayCidExits) { + http.Error(w, "", http.StatusNotFound) + return + } if err != nil { // set allow origin w.Header().Set("Access-Control-Allow-Origin", "*") diff --git a/core/corehttp/corehttp_interceptor.go b/core/corehttp/corehttp_interceptor.go index 908f8ffe0..0b810ca0f 100644 --- a/core/corehttp/corehttp_interceptor.go +++ b/core/corehttp/corehttp_interceptor.go @@ -17,6 +17,10 @@ const defaultTwoStepDuration = 30 * time.Minute const firstStepUrl = "dashboard/validate" +var ( + ErrorGatewayCidExits = errors.New("cid exits") +) + func interceptorBeforeReq(r *http.Request, n *core.IpfsNode) error { config, err := n.Repo.Config() if err != nil { @@ -24,12 +28,21 @@ func interceptorBeforeReq(r *http.Request, n *core.IpfsNode) error { } if config.API.EnableTokenAuth { - err := tokenCheckInterceptor(r, n) + err = tokenCheckInterceptor(r, n) if err != nil { return err } } + exits, err := gatewayCidInterceptor(r, n) + if err != nil || !exits { + return nil + } + + if exits { + return ErrorGatewayCidExits + } + err = twoStepCheckInterceptor(r) if err != nil { return err @@ -86,6 +99,19 @@ func tokenCheckInterceptor(r *http.Request, n *core.IpfsNode) error { return nil } +func gatewayCidInterceptor(r *http.Request, n *core.IpfsNode) (bool, error) { + if filterGatewayUrl(r) { + sPath := strings.Split(r.URL.Path, "/") + if len(sPath) < 3 { + return false, nil + } + key := strings.Split(r.URL.Path, "/")[2] + exits, err := n.Repo.Datastore().Has(r.Context(), ds.NewKey(commands.NewGatewayFilterKey(key))) + return exits, err + } + return false, nil +} + func filterNoNeedTokenCheckReq(r *http.Request, apiHost string) bool { if filterUrl(r) || filterP2pSchema(r) || filterLocalShellApi(r, apiHost) || filterGatewayUrl(r) { return true From 4ee6a5523f4721472b880442bcdd6bedcbda3880 Mon Sep 17 00:00:00 2001 From: cody Date: Mon, 25 Nov 2024 16:19:22 +0800 Subject: [PATCH 2/5] feat: cidstore list size. --- core/commands/cidstore.go | 30 ++++++++++-------------------- 1 file changed, 10 insertions(+), 20 deletions(-) diff --git a/core/commands/cidstore.go b/core/commands/cidstore.go index 10eb34b27..6cf88bc51 100644 --- a/core/commands/cidstore.go +++ b/core/commands/cidstore.go @@ -11,12 +11,13 @@ import ( ) const ( - FilterKeyPrefix = "/gateway/filter/cid" + SizeOptionName = "size" ) -type cidList struct { - Strings []string -} +const ( + FilterKeyPrefix = "/gateway/filter/cid" + DefaultSize = 50 +) var CidStoreCmd = &cmds.Command{ Helptext: cmds.HelpText{ @@ -37,10 +38,6 @@ var addCidCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Add cid to store.", }, - Options: []cmds.Option{ - cmds.BoolOption(trickleOptionName, "t", "Use trickle-dag format for dag generation."), - cmds.BoolOption(pinOptionName, "Pin this object when adding.").WithDefault(true), - }, Arguments: []cmds.Argument{ cmds.StringArg("cid", true, false, "cid to add to store"), }, @@ -62,10 +59,6 @@ var getCidCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Get cid from store.", }, - Options: []cmds.Option{ - cmds.BoolOption(trickleOptionName, "t", "Use trickle-dag format for dag generation."), - cmds.BoolOption(pinOptionName, "Pin this object when adding.").WithDefault(true), - }, Arguments: []cmds.Argument{ cmds.StringArg("cid", true, false, "cid to add to store"), }, @@ -86,10 +79,6 @@ var delCidCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Delete cid from store.", }, - Options: []cmds.Option{ - cmds.BoolOption(trickleOptionName, "t", "Use trickle-dag format for dag generation."), - cmds.BoolOption(pinOptionName, "Pin this object when adding.").WithDefault(true), - }, Arguments: []cmds.Argument{ cmds.StringArg("cid", true, false, "cid to add to store"), }, @@ -110,10 +99,6 @@ var hasCidCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Check cid exits in store", }, - Options: []cmds.Option{ - cmds.BoolOption(trickleOptionName, "t", "Use trickle-dag format for dag generation."), - cmds.BoolOption(pinOptionName, "Pin this object when adding.").WithDefault(true), - }, Arguments: []cmds.Argument{ cmds.StringArg("cid", true, false, "cid to add to store"), }, @@ -137,13 +122,18 @@ var listCidCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "List all cids in store", }, + Options: []cmds.Option{ + cmds.IntOption(SizeOptionName, "s", "Number of cids to return.").WithDefault(DefaultSize), + }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { nd, err := cmdenv.GetNode(env) if err != nil { return err } + size := req.Options[SizeOptionName].(int) results, err := nd.Repo.Datastore().Query(req.Context, query.Query{ Prefix: FilterKeyPrefix, + Limit: size, }) if err != nil { return err From eef1e1fe323a61d449011d69a06d3c1438cfb533 Mon Sep 17 00:00:00 2001 From: cody Date: Mon, 2 Dec 2024 17:22:22 +0800 Subject: [PATCH 3/5] feat: cidstore add batch support --- core/commands/cidstore.go | 52 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 48 insertions(+), 4 deletions(-) diff --git a/core/commands/cidstore.go b/core/commands/cidstore.go index 6cf88bc51..21e56cbc0 100644 --- a/core/commands/cidstore.go +++ b/core/commands/cidstore.go @@ -3,6 +3,7 @@ package commands import ( "fmt" "io" + "strings" cmds "github.com/bittorrent/go-btfs-cmds" "github.com/bittorrent/go-btfs/core/commands/cmdenv" @@ -11,12 +12,16 @@ import ( ) const ( - SizeOptionName = "size" + SizeOptionName = "size" + batchOptionName = "batch" ) const ( FilterKeyPrefix = "/gateway/filter/cid" - DefaultSize = 50 +) + +const ( + cidSeparator = "," ) var CidStoreCmd = &cmds.Command{ @@ -38,6 +43,9 @@ var addCidCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Add cid to store.", }, + Options: []cmds.Option{ + cmds.BoolOption(batchOptionName, "b", "batch add cids, cids split by , and all exits will be deleted").WithDefault(false), + }, Arguments: []cmds.Argument{ cmds.StringArg("cid", true, false, "cid to add to store"), }, @@ -46,6 +54,42 @@ var addCidCmd = &cmds.Command{ if err != nil { return err } + + batch, _ := req.Options[batchOptionName].(bool) + if batch { + cids := strings.Split(req.Arguments[0], cidSeparator) + batch, err := nd.Repo.Datastore().Batch(req.Context) + if err != nil { + return cmds.EmitOnce(res, err.Error()) + } + + // delete all exits + results, err := nd.Repo.Datastore().Query(req.Context, query.Query{ + Prefix: FilterKeyPrefix, + }) + if err != nil { + return cmds.EmitOnce(res, err.Error()) + } + for v := range results.Next() { + err = batch.Delete(req.Context, datastore.NewKey(NewGatewayFilterKey(string(v.Value)))) + if err != nil { + return cmds.EmitOnce(res, err.Error()) + } + } + + for _, v := range cids { + err = batch.Put(req.Context, datastore.NewKey(NewGatewayFilterKey(v)), []byte(v)) + if err != nil { + return cmds.EmitOnce(res, err.Error()) + } + } + err = batch.Commit(req.Context) + if err != nil { + return cmds.EmitOnce(res, err.Error()) + } + return cmds.EmitOnce(res, "Add batch ok.") + } + err = nd.Repo.Datastore().Put(req.Context, datastore.NewKey(NewGatewayFilterKey(req.Arguments[0])), []byte(req.Arguments[0])) if err != nil { @@ -123,14 +167,14 @@ var listCidCmd = &cmds.Command{ Tagline: "List all cids in store", }, Options: []cmds.Option{ - cmds.IntOption(SizeOptionName, "s", "Number of cids to return.").WithDefault(DefaultSize), + cmds.IntOption(SizeOptionName, "s", "Number of cids to return."), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { nd, err := cmdenv.GetNode(env) if err != nil { return err } - size := req.Options[SizeOptionName].(int) + size, _ := req.Options[SizeOptionName].(int) results, err := nd.Repo.Datastore().Query(req.Context, query.Query{ Prefix: FilterKeyPrefix, Limit: size, From d04d2b48f9425fd649ffe9e2d9fb70396d79d124 Mon Sep 17 00:00:00 2001 From: cody Date: Thu, 5 Dec 2024 16:27:37 +0800 Subject: [PATCH 4/5] feat: two step check need token auth enable. --- core/corehttp/corehttp_interceptor.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/core/corehttp/corehttp_interceptor.go b/core/corehttp/corehttp_interceptor.go index 0b810ca0f..7fcd33a47 100644 --- a/core/corehttp/corehttp_interceptor.go +++ b/core/corehttp/corehttp_interceptor.go @@ -32,6 +32,11 @@ func interceptorBeforeReq(r *http.Request, n *core.IpfsNode) error { if err != nil { return err } + + err = twoStepCheckInterceptor(r) + if err != nil { + return err + } } exits, err := gatewayCidInterceptor(r, n) @@ -43,11 +48,6 @@ func interceptorBeforeReq(r *http.Request, n *core.IpfsNode) error { return ErrorGatewayCidExits } - err = twoStepCheckInterceptor(r) - if err != nil { - return err - } - return nil } From 8ad01293d42a012d7adf8ae5ef4470d43cf34a12 Mon Sep 17 00:00:00 2001 From: cody Date: Thu, 5 Dec 2024 17:58:20 +0800 Subject: [PATCH 5/5] refactor: http interceptor. --- core/corehttp/corehttp.go | 15 ++++++--------- core/corehttp/corehttp_interceptor.go | 13 ++++++++----- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/core/corehttp/corehttp.go b/core/corehttp/corehttp.go index eb69b6a6a..9af5c7f82 100644 --- a/core/corehttp/corehttp.go +++ b/core/corehttp/corehttp.go @@ -54,20 +54,17 @@ func makeHandler(n *core.IpfsNode, l net.Listener, options ...ServeOption) (http } err := interceptorBeforeReq(r, n) - if errors.Is(err, ErrorGatewayCidExits) { + + if errors.Is(err, ErrGatewayCidExits) { http.Error(w, "", http.StatusNotFound) return } - if err != nil { - // set allow origin - w.Header().Set("Access-Control-Allow-Origin", "*") - if r.Method == http.MethodOptions { - w.Header().Set("Access-Control-Allow-Origin", "*") - w.Header().Set("Access-Control-Allow-Headers", "Content-Type, X-Stream-Output, X-Chunked-Output, X-Content-Length") + + if errors.Is(err, ErrNotLogin) || errors.Is(err, ErrInvalidToken) || errors.Is(err, ErrTwoStepCheckErr) { + if r.Method != http.MethodOptions { + http.Error(w, err.Error(), http.StatusUnauthorized) return } - http.Error(w, err.Error(), http.StatusUnauthorized) - return } topMux.ServeHTTP(w, r) diff --git a/core/corehttp/corehttp_interceptor.go b/core/corehttp/corehttp_interceptor.go index dea1d9e5f..d2e5e1f99 100644 --- a/core/corehttp/corehttp_interceptor.go +++ b/core/corehttp/corehttp_interceptor.go @@ -18,7 +18,10 @@ const defaultTwoStepDuration = 30 * time.Minute const firstStepUrl = "dashboard/validate" var ( - ErrorGatewayCidExits = errors.New("cid exits") + ErrNotLogin = errors.New("please login") + ErrInvalidToken = errors.New("invalid token") + ErrTwoStepCheckErr = errors.New("please validate your password first") + ErrGatewayCidExits = errors.New("cid exits") ) func interceptorBeforeReq(r *http.Request, n *core.IpfsNode) error { @@ -45,7 +48,7 @@ func interceptorBeforeReq(r *http.Request, n *core.IpfsNode) error { } if exits { - return ErrorGatewayCidExits + return ErrGatewayCidExits } return nil @@ -59,7 +62,7 @@ func twoStepCheckInterceptor(r *http.Request) error { return nil } - return errors.New("please validate your password first") + return ErrTwoStepCheckErr } func interceptorAfterResp(r *http.Request, w http.ResponseWriter, n *core.IpfsNode) error { @@ -80,7 +83,7 @@ func tokenCheckInterceptor(r *http.Request, n *core.IpfsNode) error { return nil } if !commands.IsLogin { - return fmt.Errorf("please login") + return ErrNotLogin } args := r.URL.Query() token := args.Get("token") @@ -93,7 +96,7 @@ func tokenCheckInterceptor(r *http.Request, n *core.IpfsNode) error { return err } if claims.PeerId != n.Identity.String() { - return fmt.Errorf("token is invalid") + return ErrInvalidToken } return nil