From c25d7f29b5ba065154ddd97f81caf27c53455e9f Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Fri, 4 Oct 2024 00:07:19 +0200 Subject: [PATCH 1/2] feat(routing/http): support IPIP-484 and streaming - make requests with `filter-protocols` from IPIP-484 - define defaults and allow override via env for testing - prefer streaming responses --- config/routing.go | 25 +++++++++++++++++++++++++ core/node/libp2p/routingopt.go | 17 +---------------- docs/environment-variables.md | 12 ++++++++++++ routing/delegated.go | 3 +++ 4 files changed, 41 insertions(+), 16 deletions(-) diff --git a/config/routing.go b/config/routing.go index 231cbca732c..427593522e9 100644 --- a/config/routing.go +++ b/config/routing.go @@ -3,12 +3,27 @@ package config import ( "encoding/json" "fmt" + "os" "runtime" + "strings" ) var ( DefaultAcceleratedDHTClient = false DefaultLoopbackAddressesOnLanDHT = false + + // Default HTTP routers used in parallel to DHT when Routing.Type = "auto" + DefaultHTTPRouters = getEnvOrDefault("IPFS_HTTP_ROUTERS", []string{ + "https://cid.contact", // https://github.com/ipfs/kubo/issues/9422#issuecomment-1338142084 + }) + + // Default filter-protocols to pass along with delegated routing requests (as defined in IPIP-484) + // and also filter out locally + DefaultHTTPRoutersFilterProtocols = getEnvOrDefault("IPFS_HTTP_ROUTERS_FILTER_PROTOCOLS", []string{ + "unknown", // allow results without protocol list, we can do libp2p identify to test them + "transport-bitswap", + // TODO: add 'transport-ipfs-gateway-http' once https://github.com/ipfs/rainbow/issues/125 is addressed + }) ) // Routing defines configuration options for libp2p routing. @@ -180,3 +195,13 @@ type ConfigRouter struct { type Method struct { RouterName string } + +// getEnvOrDefault reads space or comma separated strings from env if present, +// and uses provided defaultValue as a fallback +func getEnvOrDefault(key string, defaultValue []string) []string { + if value, exists := os.LookupEnv(key); exists { + splitFunc := func(r rune) bool { return r == ',' || r == ' ' } + return strings.FieldsFunc(value, splitFunc) + } + return defaultValue +} diff --git a/core/node/libp2p/routingopt.go b/core/node/libp2p/routingopt.go index 869b7ef0652..292f49fe46e 100644 --- a/core/node/libp2p/routingopt.go +++ b/core/node/libp2p/routingopt.go @@ -2,8 +2,6 @@ package libp2p import ( "context" - "os" - "strings" "time" "github.com/ipfs/go-datastore" @@ -31,23 +29,10 @@ type RoutingOptionArgs struct { type RoutingOption func(args RoutingOptionArgs) (routing.Routing, error) -// Default HTTP routers used in parallel to DHT when Routing.Type = "auto" -var defaultHTTPRouters = []string{ - "https://cid.contact", // https://github.com/ipfs/kubo/issues/9422#issuecomment-1338142084 - // TODO: add an independent router from Cloudflare -} - -func init() { - // Override HTTP routers if custom ones were passed via env - if routers := os.Getenv("IPFS_HTTP_ROUTERS"); routers != "" { - defaultHTTPRouters = strings.Split(routers, " ") - } -} - func constructDefaultHTTPRouters(cfg *config.Config) ([]*routinghelpers.ParallelRouter, error) { var routers []*routinghelpers.ParallelRouter // Append HTTP routers for additional speed - for _, endpoint := range defaultHTTPRouters { + for _, endpoint := range config.DefaultHTTPRouters { httpRouter, err := irouting.ConstructHTTPRouter(endpoint, cfg.Identity.PeerID, httpAddrsFromConfig(cfg.Addresses), cfg.Identity.PrivKey) if err != nil { return nil, err diff --git a/docs/environment-variables.md b/docs/environment-variables.md index f0f6b3f183a..b384e51addb 100644 --- a/docs/environment-variables.md +++ b/docs/environment-variables.md @@ -131,6 +131,18 @@ The above will replace implicit HTTP routers with single one, allowing for inspection/debug of HTTP requests sent by Kubo via `while true ; do nc -l 7423; done` or more advanced tools like [mitmproxy](https://docs.mitmproxy.org/stable/#mitmproxy). +Default: `config.DefaultHTTPRouters` + +## `IPFS_HTTP_ROUTERS_FILTER_PROTOCOLS` + +Overrides values passed with `filter-protocols` parameter defined in IPIP-484. +Value is space-separated. + +```console +$ IPFS_HTTP_ROUTERS_FILTER_PROTOCOLS="unknown transport-bitswap transport-foo" ipfs daemon +``` + +Default: `config.DefaultHTTPRoutersFilterProtocols` ## `IPFS_CONTENT_BLOCKING_DISABLE` diff --git a/routing/delegated.go b/routing/delegated.go index e830c1aa197..f5f98a03f25 100644 --- a/routing/delegated.go +++ b/routing/delegated.go @@ -206,6 +206,9 @@ func httpRoutingFromConfig(conf config.Router, extraHTTP *ExtraHTTPParams) (rout drclient.WithIdentity(key), drclient.WithProviderInfo(addrInfo.ID, addrInfo.Addrs), drclient.WithUserAgent(version.GetUserAgentVersion()), + drclient.WithProtocolFilter(config.DefaultHTTPRoutersFilterProtocols), + drclient.WithStreamResultsRequired(), // https://specs.ipfs.tech/routing/http-routing-v1/#streaming + drclient.WithDisabledLocalFiltering(false), // force local filtering in case remote server does not support IPIP-484 ) if err != nil { return nil, err From 02139f6fd07c919ed65d806bfc1c1cd3a85da05a Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Fri, 4 Oct 2024 00:50:14 +0200 Subject: [PATCH 2/2] chore: config consts --- config/routing.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/config/routing.go b/config/routing.go index 427593522e9..52d8b7296b4 100644 --- a/config/routing.go +++ b/config/routing.go @@ -8,10 +8,12 @@ import ( "strings" ) -var ( +const ( DefaultAcceleratedDHTClient = false DefaultLoopbackAddressesOnLanDHT = false +) +var ( // Default HTTP routers used in parallel to DHT when Routing.Type = "auto" DefaultHTTPRouters = getEnvOrDefault("IPFS_HTTP_ROUTERS", []string{ "https://cid.contact", // https://github.com/ipfs/kubo/issues/9422#issuecomment-1338142084