diff --git a/bitswap/benchmarks_test.go b/bitswap/benchmarks_test.go index 99316ca81d..4a28d0849c 100644 --- a/bitswap/benchmarks_test.go +++ b/bitswap/benchmarks_test.go @@ -18,11 +18,11 @@ import ( cid "github.com/ipfs/go-cid" delay "github.com/ipfs/go-ipfs-delay" - mockrouting "github.com/ipfs/go-ipfs-routing/mock" "github.com/ipfs/go-libipfs/bitswap" bsnet "github.com/ipfs/go-libipfs/bitswap/network" testinstance "github.com/ipfs/go-libipfs/bitswap/testinstance" tn "github.com/ipfs/go-libipfs/bitswap/testnet" + mockrouting "github.com/ipfs/go-libipfs/routing/mock" ) type fetchFunc func(b *testing.B, bs *bitswap.Bitswap, ks []cid.Cid) diff --git a/bitswap/bitswap.go b/bitswap/bitswap.go index 73000f3424..ac9ac323c1 100644 --- a/bitswap/bitswap.go +++ b/bitswap/bitswap.go @@ -13,9 +13,9 @@ import ( "github.com/ipfs/go-metrics-interface" "github.com/ipfs/go-cid" - blockstore "github.com/ipfs/go-ipfs-blockstore" - exchange "github.com/ipfs/go-ipfs-exchange-interface" blocks "github.com/ipfs/go-libipfs/blocks" + blockstore "github.com/ipfs/go-libipfs/blockstore" + exchange "github.com/ipfs/go-libipfs/exchange" logging "github.com/ipfs/go-log" "github.com/libp2p/go-libp2p/core/peer" diff --git a/bitswap/bitswap_test.go b/bitswap/bitswap_test.go index 16c5f41627..72933be003 100644 --- a/bitswap/bitswap_test.go +++ b/bitswap/bitswap_test.go @@ -13,7 +13,6 @@ import ( detectrace "github.com/ipfs/go-detect-race" blocksutil "github.com/ipfs/go-ipfs-blocksutil" delay "github.com/ipfs/go-ipfs-delay" - mockrouting "github.com/ipfs/go-ipfs-routing/mock" ipld "github.com/ipfs/go-ipld-format" "github.com/ipfs/go-libipfs/bitswap" bsmsg "github.com/ipfs/go-libipfs/bitswap/message" @@ -22,6 +21,7 @@ import ( tn "github.com/ipfs/go-libipfs/bitswap/testnet" blocks "github.com/ipfs/go-libipfs/blocks" "github.com/ipfs/go-libipfs/internal/test" + mockrouting "github.com/ipfs/go-libipfs/routing/mock" tu "github.com/libp2p/go-libp2p-testing/etc" p2ptestutil "github.com/libp2p/go-libp2p-testing/netutil" peer "github.com/libp2p/go-libp2p/core/peer" diff --git a/bitswap/client/bitswap_with_sessions_test.go b/bitswap/client/bitswap_with_sessions_test.go index 37a5786f0c..fa15a704ee 100644 --- a/bitswap/client/bitswap_with_sessions_test.go +++ b/bitswap/client/bitswap_with_sessions_test.go @@ -9,13 +9,13 @@ import ( cid "github.com/ipfs/go-cid" blocksutil "github.com/ipfs/go-ipfs-blocksutil" delay "github.com/ipfs/go-ipfs-delay" - mockrouting "github.com/ipfs/go-ipfs-routing/mock" "github.com/ipfs/go-libipfs/bitswap" "github.com/ipfs/go-libipfs/bitswap/client/internal/session" testinstance "github.com/ipfs/go-libipfs/bitswap/testinstance" tn "github.com/ipfs/go-libipfs/bitswap/testnet" blocks "github.com/ipfs/go-libipfs/blocks" "github.com/ipfs/go-libipfs/internal/test" + mockrouting "github.com/ipfs/go-libipfs/routing/mock" tu "github.com/libp2p/go-libp2p-testing/etc" ) diff --git a/bitswap/client/client.go b/bitswap/client/client.go index 31942ae60c..122b44ab5b 100644 --- a/bitswap/client/client.go +++ b/bitswap/client/client.go @@ -14,8 +14,6 @@ import ( "go.opentelemetry.io/otel/trace" "github.com/ipfs/go-cid" - blockstore "github.com/ipfs/go-ipfs-blockstore" - exchange "github.com/ipfs/go-ipfs-exchange-interface" bsbpm "github.com/ipfs/go-libipfs/bitswap/client/internal/blockpresencemanager" bsgetter "github.com/ipfs/go-libipfs/bitswap/client/internal/getter" bsmq "github.com/ipfs/go-libipfs/bitswap/client/internal/messagequeue" @@ -33,6 +31,8 @@ import ( bsnet "github.com/ipfs/go-libipfs/bitswap/network" "github.com/ipfs/go-libipfs/bitswap/tracer" blocks "github.com/ipfs/go-libipfs/blocks" + blockstore "github.com/ipfs/go-libipfs/blockstore" + exchange "github.com/ipfs/go-libipfs/exchange" logging "github.com/ipfs/go-log" "github.com/ipfs/go-metrics-interface" process "github.com/jbenet/goprocess" diff --git a/bitswap/client/internal/sessionmanager/sessionmanager.go b/bitswap/client/internal/sessionmanager/sessionmanager.go index 690077e70a..1e7971751a 100644 --- a/bitswap/client/internal/sessionmanager/sessionmanager.go +++ b/bitswap/client/internal/sessionmanager/sessionmanager.go @@ -11,12 +11,12 @@ import ( "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" - exchange "github.com/ipfs/go-ipfs-exchange-interface" "github.com/ipfs/go-libipfs/bitswap/client/internal" bsbpm "github.com/ipfs/go-libipfs/bitswap/client/internal/blockpresencemanager" notifications "github.com/ipfs/go-libipfs/bitswap/client/internal/notifications" bssession "github.com/ipfs/go-libipfs/bitswap/client/internal/session" bssim "github.com/ipfs/go-libipfs/bitswap/client/internal/sessioninterestmanager" + exchange "github.com/ipfs/go-libipfs/exchange" peer "github.com/libp2p/go-libp2p/core/peer" ) diff --git a/bitswap/internal/defaults/defaults.go b/bitswap/internal/defaults/defaults.go index f9494a0da6..fc193529d8 100644 --- a/bitswap/internal/defaults/defaults.go +++ b/bitswap/internal/defaults/defaults.go @@ -29,7 +29,7 @@ const ( // Maximum size of the wantlist we are willing to keep in memory. MaxQueuedWantlistEntiresPerPeer = 1024 - // Copied from github.com/ipfs/go-verifcid#maximumHashLength + // Copied from github.com/ipfs/go-libipfs/verifcid#maximumHashLength // FIXME: expose this in go-verifcid. MaximumHashLength = 128 MaximumAllowedCid = binary.MaxVarintLen64*4 + MaximumHashLength diff --git a/bitswap/message/message.go b/bitswap/message/message.go index 42c0a46a29..2af79e2ac9 100644 --- a/bitswap/message/message.go +++ b/bitswap/message/message.go @@ -13,7 +13,7 @@ import ( pool "github.com/libp2p/go-buffer-pool" msgio "github.com/libp2p/go-msgio" - u "github.com/ipfs/go-ipfs-util" + u "github.com/ipfs/go-libipfs/util" "github.com/libp2p/go-libp2p/core/network" ) diff --git a/bitswap/message/message_test.go b/bitswap/message/message_test.go index e4559f8b48..0ed64c7417 100644 --- a/bitswap/message/message_test.go +++ b/bitswap/message/message_test.go @@ -9,8 +9,8 @@ import ( pb "github.com/ipfs/go-libipfs/bitswap/message/pb" cid "github.com/ipfs/go-cid" - u "github.com/ipfs/go-ipfs-util" blocks "github.com/ipfs/go-libipfs/blocks" + u "github.com/ipfs/go-libipfs/util" ) func mkFakeCid(s string) cid.Cid { diff --git a/bitswap/message/pb/cid_test.go b/bitswap/message/pb/cid_test.go index d0d5d2b23c..ab76305022 100644 --- a/bitswap/message/pb/cid_test.go +++ b/bitswap/message/pb/cid_test.go @@ -5,7 +5,7 @@ import ( "testing" "github.com/ipfs/go-cid" - u "github.com/ipfs/go-ipfs-util" + u "github.com/ipfs/go-libipfs/util" pb "github.com/ipfs/go-libipfs/bitswap/message/pb" ) diff --git a/bitswap/network/ipfs_impl_test.go b/bitswap/network/ipfs_impl_test.go index 62b3ac98e9..7f3ffdc51a 100644 --- a/bitswap/network/ipfs_impl_test.go +++ b/bitswap/network/ipfs_impl_test.go @@ -9,13 +9,13 @@ import ( ds "github.com/ipfs/go-datastore" blocksutil "github.com/ipfs/go-ipfs-blocksutil" - mockrouting "github.com/ipfs/go-ipfs-routing/mock" bsmsg "github.com/ipfs/go-libipfs/bitswap/message" pb "github.com/ipfs/go-libipfs/bitswap/message/pb" bsnet "github.com/ipfs/go-libipfs/bitswap/network" "github.com/ipfs/go-libipfs/bitswap/network/internal" tn "github.com/ipfs/go-libipfs/bitswap/testnet" "github.com/ipfs/go-libipfs/internal/test" + mockrouting "github.com/ipfs/go-libipfs/routing/mock" tnet "github.com/libp2p/go-libp2p-testing/net" "github.com/libp2p/go-libp2p/core/host" "github.com/libp2p/go-libp2p/core/network" diff --git a/bitswap/server/internal/decision/blockstoremanager.go b/bitswap/server/internal/decision/blockstoremanager.go index 4171c038db..d03c6788aa 100644 --- a/bitswap/server/internal/decision/blockstoremanager.go +++ b/bitswap/server/internal/decision/blockstoremanager.go @@ -6,9 +6,9 @@ import ( "sync" cid "github.com/ipfs/go-cid" - bstore "github.com/ipfs/go-ipfs-blockstore" ipld "github.com/ipfs/go-ipld-format" blocks "github.com/ipfs/go-libipfs/blocks" + bstore "github.com/ipfs/go-libipfs/blockstore" "github.com/ipfs/go-metrics-interface" ) diff --git a/bitswap/server/internal/decision/blockstoremanager_test.go b/bitswap/server/internal/decision/blockstoremanager_test.go index 06c5ec56d1..508fbdc381 100644 --- a/bitswap/server/internal/decision/blockstoremanager_test.go +++ b/bitswap/server/internal/decision/blockstoremanager_test.go @@ -11,10 +11,10 @@ import ( ds "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/delayed" ds_sync "github.com/ipfs/go-datastore/sync" - blockstore "github.com/ipfs/go-ipfs-blockstore" delay "github.com/ipfs/go-ipfs-delay" "github.com/ipfs/go-libipfs/bitswap/internal/testutil" blocks "github.com/ipfs/go-libipfs/blocks" + blockstore "github.com/ipfs/go-libipfs/blockstore" "github.com/ipfs/go-libipfs/internal/test" "github.com/ipfs/go-metrics-interface" ) diff --git a/bitswap/server/internal/decision/engine.go b/bitswap/server/internal/decision/engine.go index 208fef7403..dde3f61aea 100644 --- a/bitswap/server/internal/decision/engine.go +++ b/bitswap/server/internal/decision/engine.go @@ -11,13 +11,13 @@ import ( "github.com/google/uuid" "github.com/ipfs/go-cid" - bstore "github.com/ipfs/go-ipfs-blockstore" wl "github.com/ipfs/go-libipfs/bitswap/client/wantlist" "github.com/ipfs/go-libipfs/bitswap/internal/defaults" bsmsg "github.com/ipfs/go-libipfs/bitswap/message" pb "github.com/ipfs/go-libipfs/bitswap/message/pb" bmetrics "github.com/ipfs/go-libipfs/bitswap/metrics" blocks "github.com/ipfs/go-libipfs/blocks" + bstore "github.com/ipfs/go-libipfs/blockstore" logging "github.com/ipfs/go-log" "github.com/ipfs/go-metrics-interface" "github.com/ipfs/go-peertaskqueue" diff --git a/bitswap/server/internal/decision/engine_test.go b/bitswap/server/internal/decision/engine_test.go index 5d93ad83f5..76e9674761 100644 --- a/bitswap/server/internal/decision/engine_test.go +++ b/bitswap/server/internal/decision/engine_test.go @@ -16,11 +16,11 @@ import ( "github.com/ipfs/go-cid" ds "github.com/ipfs/go-datastore" dssync "github.com/ipfs/go-datastore/sync" - blockstore "github.com/ipfs/go-ipfs-blockstore" "github.com/ipfs/go-libipfs/bitswap/internal/testutil" message "github.com/ipfs/go-libipfs/bitswap/message" pb "github.com/ipfs/go-libipfs/bitswap/message/pb" blocks "github.com/ipfs/go-libipfs/blocks" + blockstore "github.com/ipfs/go-libipfs/blockstore" "github.com/ipfs/go-libipfs/internal/test" process "github.com/jbenet/goprocess" peer "github.com/libp2p/go-libp2p/core/peer" diff --git a/bitswap/server/server.go b/bitswap/server/server.go index 7918b73d78..e1fa3d899e 100644 --- a/bitswap/server/server.go +++ b/bitswap/server/server.go @@ -9,7 +9,6 @@ import ( "time" "github.com/ipfs/go-cid" - blockstore "github.com/ipfs/go-ipfs-blockstore" "github.com/ipfs/go-libipfs/bitswap/internal/defaults" "github.com/ipfs/go-libipfs/bitswap/message" pb "github.com/ipfs/go-libipfs/bitswap/message/pb" @@ -18,6 +17,7 @@ import ( "github.com/ipfs/go-libipfs/bitswap/server/internal/decision" "github.com/ipfs/go-libipfs/bitswap/tracer" blocks "github.com/ipfs/go-libipfs/blocks" + blockstore "github.com/ipfs/go-libipfs/blockstore" logging "github.com/ipfs/go-log" "github.com/ipfs/go-metrics-interface" process "github.com/jbenet/goprocess" diff --git a/bitswap/testinstance/testinstance.go b/bitswap/testinstance/testinstance.go index 47e22b7a2e..b32260e037 100644 --- a/bitswap/testinstance/testinstance.go +++ b/bitswap/testinstance/testinstance.go @@ -7,11 +7,11 @@ import ( ds "github.com/ipfs/go-datastore" delayed "github.com/ipfs/go-datastore/delayed" ds_sync "github.com/ipfs/go-datastore/sync" - blockstore "github.com/ipfs/go-ipfs-blockstore" delay "github.com/ipfs/go-ipfs-delay" "github.com/ipfs/go-libipfs/bitswap" bsnet "github.com/ipfs/go-libipfs/bitswap/network" tn "github.com/ipfs/go-libipfs/bitswap/testnet" + blockstore "github.com/ipfs/go-libipfs/blockstore" tnet "github.com/libp2p/go-libp2p-testing/net" p2ptestutil "github.com/libp2p/go-libp2p-testing/netutil" peer "github.com/libp2p/go-libp2p/core/peer" diff --git a/bitswap/testnet/network_test.go b/bitswap/testnet/network_test.go index 388b9c9c39..fd49ded733 100644 --- a/bitswap/testnet/network_test.go +++ b/bitswap/testnet/network_test.go @@ -9,8 +9,8 @@ import ( bsnet "github.com/ipfs/go-libipfs/bitswap/network" delay "github.com/ipfs/go-ipfs-delay" - mockrouting "github.com/ipfs/go-ipfs-routing/mock" blocks "github.com/ipfs/go-libipfs/blocks" + mockrouting "github.com/ipfs/go-libipfs/routing/mock" tnet "github.com/libp2p/go-libp2p-testing/net" "github.com/libp2p/go-libp2p/core/peer" diff --git a/bitswap/testnet/peernet.go b/bitswap/testnet/peernet.go index ea98b98f45..ea0826651a 100644 --- a/bitswap/testnet/peernet.go +++ b/bitswap/testnet/peernet.go @@ -6,7 +6,7 @@ import ( bsnet "github.com/ipfs/go-libipfs/bitswap/network" ds "github.com/ipfs/go-datastore" - mockrouting "github.com/ipfs/go-ipfs-routing/mock" + mockrouting "github.com/ipfs/go-libipfs/routing/mock" tnet "github.com/libp2p/go-libp2p-testing/net" "github.com/libp2p/go-libp2p/core/peer" diff --git a/bitswap/testnet/virtual.go b/bitswap/testnet/virtual.go index b743489662..bf43dc31d2 100644 --- a/bitswap/testnet/virtual.go +++ b/bitswap/testnet/virtual.go @@ -13,7 +13,7 @@ import ( cid "github.com/ipfs/go-cid" delay "github.com/ipfs/go-ipfs-delay" - mockrouting "github.com/ipfs/go-ipfs-routing/mock" + mockrouting "github.com/ipfs/go-libipfs/routing/mock" tnet "github.com/libp2p/go-libp2p-testing/net" "github.com/libp2p/go-libp2p/core/connmgr" diff --git a/blocks/blocks.go b/blocks/blocks.go index 3d3894b3f3..3681031f47 100644 --- a/blocks/blocks.go +++ b/blocks/blocks.go @@ -8,7 +8,7 @@ import ( "fmt" cid "github.com/ipfs/go-cid" - u "github.com/ipfs/go-ipfs-util" + u "github.com/ipfs/go-libipfs/util" mh "github.com/multiformats/go-multihash" ) diff --git a/blocks/blocks_test.go b/blocks/blocks_test.go index 18f0c1f38c..3e0c0eec24 100644 --- a/blocks/blocks_test.go +++ b/blocks/blocks_test.go @@ -5,7 +5,7 @@ import ( "testing" cid "github.com/ipfs/go-cid" - u "github.com/ipfs/go-ipfs-util" + u "github.com/ipfs/go-libipfs/util" mh "github.com/multiformats/go-multihash" ) diff --git a/blockservice/blockservice.go b/blockservice/blockservice.go new file mode 100644 index 0000000000..94f44ea8f7 --- /dev/null +++ b/blockservice/blockservice.go @@ -0,0 +1,470 @@ +// package blockservice implements a BlockService interface that provides +// a single GetBlock/AddBlock interface that seamlessly retrieves data either +// locally or from a remote peer through the exchange. +package blockservice + +import ( + "context" + "io" + "sync" + + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" + + cid "github.com/ipfs/go-cid" + ipld "github.com/ipfs/go-ipld-format" + blocks "github.com/ipfs/go-libipfs/blocks" + blockstore "github.com/ipfs/go-libipfs/blockstore" + exchange "github.com/ipfs/go-libipfs/exchange" + "github.com/ipfs/go-libipfs/verifcid" + logging "github.com/ipfs/go-log/v2" + + "github.com/ipfs/go-libipfs/blockservice/internal" +) + +var logger = logging.Logger("blockservice") + +// BlockGetter is the common interface shared between blockservice sessions and +// the blockservice. +type BlockGetter interface { + // GetBlock gets the requested block. + GetBlock(ctx context.Context, c cid.Cid) (blocks.Block, error) + + // GetBlocks does a batch request for the given cids, returning blocks as + // they are found, in no particular order. + // + // It may not be able to find all requested blocks (or the context may + // be canceled). In that case, it will close the channel early. It is up + // to the consumer to detect this situation and keep track which blocks + // it has received and which it hasn't. + GetBlocks(ctx context.Context, ks []cid.Cid) <-chan blocks.Block +} + +// BlockService is a hybrid block datastore. It stores data in a local +// datastore and may retrieve data from a remote Exchange. +// It uses an internal `datastore.Datastore` instance to store values. +type BlockService interface { + io.Closer + BlockGetter + + // Blockstore returns a reference to the underlying blockstore + Blockstore() blockstore.Blockstore + + // Exchange returns a reference to the underlying exchange (usually bitswap) + Exchange() exchange.Interface + + // AddBlock puts a given block to the underlying datastore + AddBlock(ctx context.Context, o blocks.Block) error + + // AddBlocks adds a slice of blocks at the same time using batching + // capabilities of the underlying datastore whenever possible. + AddBlocks(ctx context.Context, bs []blocks.Block) error + + // DeleteBlock deletes the given block from the blockservice. + DeleteBlock(ctx context.Context, o cid.Cid) error +} + +type blockService struct { + blockstore blockstore.Blockstore + exchange exchange.Interface + // If checkFirst is true then first check that a block doesn't + // already exist to avoid republishing the block on the exchange. + checkFirst bool +} + +// NewBlockService creates a BlockService with given datastore instance. +func New(bs blockstore.Blockstore, rem exchange.Interface) BlockService { + if rem == nil { + logger.Debug("blockservice running in local (offline) mode.") + } + + return &blockService{ + blockstore: bs, + exchange: rem, + checkFirst: true, + } +} + +// NewWriteThrough creates a BlockService that guarantees writes will go +// through to the blockstore and are not skipped by cache checks. +func NewWriteThrough(bs blockstore.Blockstore, rem exchange.Interface) BlockService { + if rem == nil { + logger.Debug("blockservice running in local (offline) mode.") + } + + return &blockService{ + blockstore: bs, + exchange: rem, + checkFirst: false, + } +} + +// Blockstore returns the blockstore behind this blockservice. +func (s *blockService) Blockstore() blockstore.Blockstore { + return s.blockstore +} + +// Exchange returns the exchange behind this blockservice. +func (s *blockService) Exchange() exchange.Interface { + return s.exchange +} + +// NewSession creates a new session that allows for +// controlled exchange of wantlists to decrease the bandwidth overhead. +// If the current exchange is a SessionExchange, a new exchange +// session will be created. Otherwise, the current exchange will be used +// directly. +func NewSession(ctx context.Context, bs BlockService) *Session { + exch := bs.Exchange() + if sessEx, ok := exch.(exchange.SessionExchange); ok { + return &Session{ + sessCtx: ctx, + ses: nil, + sessEx: sessEx, + bs: bs.Blockstore(), + notifier: exch, + } + } + return &Session{ + ses: exch, + sessCtx: ctx, + bs: bs.Blockstore(), + notifier: exch, + } +} + +// AddBlock adds a particular block to the service, Putting it into the datastore. +func (s *blockService) AddBlock(ctx context.Context, o blocks.Block) error { + ctx, span := internal.StartSpan(ctx, "blockService.AddBlock") + defer span.End() + + c := o.Cid() + // hash security + err := verifcid.ValidateCid(c) + if err != nil { + return err + } + if s.checkFirst { + if has, err := s.blockstore.Has(ctx, c); has || err != nil { + return err + } + } + + if err := s.blockstore.Put(ctx, o); err != nil { + return err + } + + logger.Debugf("BlockService.BlockAdded %s", c) + + if s.exchange != nil { + if err := s.exchange.NotifyNewBlocks(ctx, o); err != nil { + logger.Errorf("NotifyNewBlocks: %s", err.Error()) + } + } + + return nil +} + +func (s *blockService) AddBlocks(ctx context.Context, bs []blocks.Block) error { + ctx, span := internal.StartSpan(ctx, "blockService.AddBlocks") + defer span.End() + + // hash security + for _, b := range bs { + err := verifcid.ValidateCid(b.Cid()) + if err != nil { + return err + } + } + var toput []blocks.Block + if s.checkFirst { + toput = make([]blocks.Block, 0, len(bs)) + for _, b := range bs { + has, err := s.blockstore.Has(ctx, b.Cid()) + if err != nil { + return err + } + if !has { + toput = append(toput, b) + } + } + } else { + toput = bs + } + + if len(toput) == 0 { + return nil + } + + err := s.blockstore.PutMany(ctx, toput) + if err != nil { + return err + } + + if s.exchange != nil { + logger.Debugf("BlockService.BlockAdded %d blocks", len(toput)) + if err := s.exchange.NotifyNewBlocks(ctx, toput...); err != nil { + logger.Errorf("NotifyNewBlocks: %s", err.Error()) + } + } + return nil +} + +// GetBlock retrieves a particular block from the service, +// Getting it from the datastore using the key (hash). +func (s *blockService) GetBlock(ctx context.Context, c cid.Cid) (blocks.Block, error) { + ctx, span := internal.StartSpan(ctx, "blockService.GetBlock", trace.WithAttributes(attribute.Stringer("CID", c))) + defer span.End() + + var f func() notifiableFetcher + if s.exchange != nil { + f = s.getExchange + } + + return getBlock(ctx, c, s.blockstore, f) // hash security +} + +func (s *blockService) getExchange() notifiableFetcher { + return s.exchange +} + +func getBlock(ctx context.Context, c cid.Cid, bs blockstore.Blockstore, fget func() notifiableFetcher) (blocks.Block, error) { + err := verifcid.ValidateCid(c) // hash security + if err != nil { + return nil, err + } + + block, err := bs.Get(ctx, c) + if err == nil { + return block, nil + } + + if ipld.IsNotFound(err) && fget != nil { + f := fget() // Don't load the exchange until we have to + + // TODO be careful checking ErrNotFound. If the underlying + // implementation changes, this will break. + logger.Debug("Blockservice: Searching bitswap") + blk, err := f.GetBlock(ctx, c) + if err != nil { + return nil, err + } + // also write in the blockstore for caching, inform the exchange that the block is available + err = bs.Put(ctx, blk) + if err != nil { + return nil, err + } + err = f.NotifyNewBlocks(ctx, blk) + if err != nil { + return nil, err + } + logger.Debugf("BlockService.BlockFetched %s", c) + return blk, nil + } + + logger.Debug("Blockservice GetBlock: Not found") + return nil, err +} + +// GetBlocks gets a list of blocks asynchronously and returns through +// the returned channel. +// NB: No guarantees are made about order. +func (s *blockService) GetBlocks(ctx context.Context, ks []cid.Cid) <-chan blocks.Block { + ctx, span := internal.StartSpan(ctx, "blockService.GetBlocks") + defer span.End() + + var f func() notifiableFetcher + if s.exchange != nil { + f = s.getExchange + } + + return getBlocks(ctx, ks, s.blockstore, f) // hash security +} + +func getBlocks(ctx context.Context, ks []cid.Cid, bs blockstore.Blockstore, fget func() notifiableFetcher) <-chan blocks.Block { + out := make(chan blocks.Block) + + go func() { + defer close(out) + + allValid := true + for _, c := range ks { + if err := verifcid.ValidateCid(c); err != nil { + allValid = false + break + } + } + + if !allValid { + ks2 := make([]cid.Cid, 0, len(ks)) + for _, c := range ks { + // hash security + if err := verifcid.ValidateCid(c); err == nil { + ks2 = append(ks2, c) + } else { + logger.Errorf("unsafe CID (%s) passed to blockService.GetBlocks: %s", c, err) + } + } + ks = ks2 + } + + var misses []cid.Cid + for _, c := range ks { + hit, err := bs.Get(ctx, c) + if err != nil { + misses = append(misses, c) + continue + } + select { + case out <- hit: + case <-ctx.Done(): + return + } + } + + if len(misses) == 0 || fget == nil { + return + } + + f := fget() // don't load exchange unless we have to + rblocks, err := f.GetBlocks(ctx, misses) + if err != nil { + logger.Debugf("Error with GetBlocks: %s", err) + return + } + + // batch available blocks together + const batchSize = 32 + batch := make([]blocks.Block, 0, batchSize) + for { + var noMoreBlocks bool + batchLoop: + for len(batch) < batchSize { + select { + case b, ok := <-rblocks: + if !ok { + noMoreBlocks = true + break batchLoop + } + + logger.Debugf("BlockService.BlockFetched %s", b.Cid()) + batch = append(batch, b) + case <-ctx.Done(): + return + default: + break batchLoop + } + } + + // also write in the blockstore for caching, inform the exchange that the blocks are available + err = bs.PutMany(ctx, batch) + if err != nil { + logger.Errorf("could not write blocks from the network to the blockstore: %s", err) + return + } + + err = f.NotifyNewBlocks(ctx, batch...) + if err != nil { + logger.Errorf("could not tell the exchange about new blocks: %s", err) + return + } + + for _, b := range batch { + select { + case out <- b: + case <-ctx.Done(): + return + } + } + batch = batch[:0] + if noMoreBlocks { + break + } + } + }() + return out +} + +// DeleteBlock deletes a block in the blockservice from the datastore +func (s *blockService) DeleteBlock(ctx context.Context, c cid.Cid) error { + ctx, span := internal.StartSpan(ctx, "blockService.DeleteBlock", trace.WithAttributes(attribute.Stringer("CID", c))) + defer span.End() + + err := s.blockstore.DeleteBlock(ctx, c) + if err == nil { + logger.Debugf("BlockService.BlockDeleted %s", c) + } + return err +} + +func (s *blockService) Close() error { + logger.Debug("blockservice is shutting down...") + return s.exchange.Close() +} + +type notifier interface { + NotifyNewBlocks(context.Context, ...blocks.Block) error +} + +// Session is a helper type to provide higher level access to bitswap sessions +type Session struct { + bs blockstore.Blockstore + ses exchange.Fetcher + sessEx exchange.SessionExchange + sessCtx context.Context + notifier notifier + lk sync.Mutex +} + +type notifiableFetcher interface { + exchange.Fetcher + notifier +} + +type notifiableFetcherWrapper struct { + exchange.Fetcher + notifier +} + +func (s *Session) getSession() notifiableFetcher { + s.lk.Lock() + defer s.lk.Unlock() + if s.ses == nil { + s.ses = s.sessEx.NewSession(s.sessCtx) + } + + return notifiableFetcherWrapper{s.ses, s.notifier} +} + +func (s *Session) getExchange() notifiableFetcher { + return notifiableFetcherWrapper{s.ses, s.notifier} +} + +func (s *Session) getFetcherFactory() func() notifiableFetcher { + if s.sessEx != nil { + return s.getSession + } + if s.ses != nil { + // Our exchange isn't session compatible, let's fallback to non sessions fetches + return s.getExchange + } + return nil +} + +// GetBlock gets a block in the context of a request session +func (s *Session) GetBlock(ctx context.Context, c cid.Cid) (blocks.Block, error) { + ctx, span := internal.StartSpan(ctx, "Session.GetBlock", trace.WithAttributes(attribute.Stringer("CID", c))) + defer span.End() + + return getBlock(ctx, c, s.bs, s.getFetcherFactory()) // hash security +} + +// GetBlocks gets blocks in the context of a request session +func (s *Session) GetBlocks(ctx context.Context, ks []cid.Cid) <-chan blocks.Block { + ctx, span := internal.StartSpan(ctx, "Session.GetBlocks") + defer span.End() + + return getBlocks(ctx, ks, s.bs, s.getFetcherFactory()) // hash security +} + +var _ BlockGetter = (*Session)(nil) diff --git a/blockservice/blockservice_test.go b/blockservice/blockservice_test.go new file mode 100644 index 0000000000..f3315fcc38 --- /dev/null +++ b/blockservice/blockservice_test.go @@ -0,0 +1,243 @@ +package blockservice + +import ( + "context" + "testing" + + cid "github.com/ipfs/go-cid" + ds "github.com/ipfs/go-datastore" + dssync "github.com/ipfs/go-datastore/sync" + butil "github.com/ipfs/go-ipfs-blocksutil" + ipld "github.com/ipfs/go-ipld-format" + blocks "github.com/ipfs/go-libipfs/blocks" + blockstore "github.com/ipfs/go-libipfs/blockstore" + exchange "github.com/ipfs/go-libipfs/exchange" + offline "github.com/ipfs/go-libipfs/exchange/offline" +) + +func TestWriteThroughWorks(t *testing.T) { + bstore := &PutCountingBlockstore{ + blockstore.NewBlockstore(dssync.MutexWrap(ds.NewMapDatastore())), + 0, + } + exchbstore := blockstore.NewBlockstore(dssync.MutexWrap(ds.NewMapDatastore())) + exch := offline.Exchange(exchbstore) + bserv := NewWriteThrough(bstore, exch) + bgen := butil.NewBlockGenerator() + + block := bgen.Next() + + t.Logf("PutCounter: %d", bstore.PutCounter) + err := bserv.AddBlock(context.Background(), block) + if err != nil { + t.Fatal(err) + } + if bstore.PutCounter != 1 { + t.Fatalf("expected just one Put call, have: %d", bstore.PutCounter) + } + + err = bserv.AddBlock(context.Background(), block) + if err != nil { + t.Fatal(err) + } + if bstore.PutCounter != 2 { + t.Fatalf("Put should have called again, should be 2 is: %d", bstore.PutCounter) + } +} + +func TestExchangeWrite(t *testing.T) { + bstore := &PutCountingBlockstore{ + blockstore.NewBlockstore(dssync.MutexWrap(ds.NewMapDatastore())), + 0, + } + exchbstore := blockstore.NewBlockstore(dssync.MutexWrap(ds.NewMapDatastore())) + exch := ¬ifyCountingExchange{ + offline.Exchange(exchbstore), + 0, + } + bserv := NewWriteThrough(bstore, exch) + bgen := butil.NewBlockGenerator() + + for name, fetcher := range map[string]BlockGetter{ + "blockservice": bserv, + "session": NewSession(context.Background(), bserv), + } { + t.Run(name, func(t *testing.T) { + // GetBlock + block := bgen.Next() + err := exchbstore.Put(context.Background(), block) + if err != nil { + t.Fatal(err) + } + got, err := fetcher.GetBlock(context.Background(), block.Cid()) + if err != nil { + t.Fatal(err) + } + if got.Cid() != block.Cid() { + t.Fatalf("GetBlock returned unexpected block") + } + if bstore.PutCounter != 1 { + t.Fatalf("expected one Put call, have: %d", bstore.PutCounter) + } + if exch.notifyCount != 1 { + t.Fatalf("expected one NotifyNewBlocks call, have: %d", exch.notifyCount) + } + + // GetBlocks + b1 := bgen.Next() + err = exchbstore.Put(context.Background(), b1) + if err != nil { + t.Fatal(err) + } + b2 := bgen.Next() + err = exchbstore.Put(context.Background(), b2) + if err != nil { + t.Fatal(err) + } + bchan := fetcher.GetBlocks(context.Background(), []cid.Cid{b1.Cid(), b2.Cid()}) + var gotBlocks []blocks.Block + for b := range bchan { + gotBlocks = append(gotBlocks, b) + } + if len(gotBlocks) != 2 { + t.Fatalf("expected to retrieve 2 blocks, got %d", len(gotBlocks)) + } + if bstore.PutCounter != 3 { + t.Fatalf("expected 3 Put call, have: %d", bstore.PutCounter) + } + if exch.notifyCount != 3 { + t.Fatalf("expected one NotifyNewBlocks call, have: %d", exch.notifyCount) + } + + // reset counts + bstore.PutCounter = 0 + exch.notifyCount = 0 + }) + } +} + +func TestLazySessionInitialization(t *testing.T) { + ctx := context.Background() + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + bstore := blockstore.NewBlockstore(dssync.MutexWrap(ds.NewMapDatastore())) + bstore2 := blockstore.NewBlockstore(dssync.MutexWrap(ds.NewMapDatastore())) + bstore3 := blockstore.NewBlockstore(dssync.MutexWrap(ds.NewMapDatastore())) + session := offline.Exchange(bstore2) + exch := offline.Exchange(bstore3) + sessionExch := &fakeSessionExchange{Interface: exch, session: session} + bservSessEx := NewWriteThrough(bstore, sessionExch) + bgen := butil.NewBlockGenerator() + + block := bgen.Next() + err := bstore.Put(ctx, block) + if err != nil { + t.Fatal(err) + } + block2 := bgen.Next() + err = bstore2.Put(ctx, block2) + if err != nil { + t.Fatal(err) + } + err = session.NotifyNewBlocks(ctx, block2) + if err != nil { + t.Fatal(err) + } + + bsession := NewSession(ctx, bservSessEx) + if bsession.ses != nil { + t.Fatal("Session exchange should not instantiated session immediately") + } + returnedBlock, err := bsession.GetBlock(ctx, block.Cid()) + if err != nil { + t.Fatal("Should have fetched block locally") + } + if returnedBlock.Cid() != block.Cid() { + t.Fatal("Got incorrect block") + } + if bsession.ses != nil { + t.Fatal("Session exchange should not instantiated session if local store had block") + } + returnedBlock, err = bsession.GetBlock(ctx, block2.Cid()) + if err != nil { + t.Fatal("Should have fetched block remotely") + } + if returnedBlock.Cid() != block2.Cid() { + t.Fatal("Got incorrect block") + } + if bsession.ses != session { + t.Fatal("Should have initialized session to fetch block") + } +} + +var _ blockstore.Blockstore = (*PutCountingBlockstore)(nil) + +type PutCountingBlockstore struct { + blockstore.Blockstore + PutCounter int +} + +func (bs *PutCountingBlockstore) Put(ctx context.Context, block blocks.Block) error { + bs.PutCounter++ + return bs.Blockstore.Put(ctx, block) +} + +func (bs *PutCountingBlockstore) PutMany(ctx context.Context, blocks []blocks.Block) error { + bs.PutCounter += len(blocks) + return bs.Blockstore.PutMany(ctx, blocks) +} + +var _ exchange.Interface = (*notifyCountingExchange)(nil) + +type notifyCountingExchange struct { + exchange.Interface + notifyCount int +} + +func (n *notifyCountingExchange) NotifyNewBlocks(ctx context.Context, blocks ...blocks.Block) error { + n.notifyCount += len(blocks) + return n.Interface.NotifyNewBlocks(ctx, blocks...) +} + +var _ exchange.SessionExchange = (*fakeSessionExchange)(nil) + +type fakeSessionExchange struct { + exchange.Interface + session exchange.Fetcher +} + +func (fe *fakeSessionExchange) NewSession(ctx context.Context) exchange.Fetcher { + if ctx == nil { + panic("nil context") + } + return fe.session +} + +func TestNilExchange(t *testing.T) { + ctx := context.Background() + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + bgen := butil.NewBlockGenerator() + block := bgen.Next() + + bs := blockstore.NewBlockstore(dssync.MutexWrap(ds.NewMapDatastore())) + bserv := NewWriteThrough(bs, nil) + sess := NewSession(ctx, bserv) + _, err := sess.GetBlock(ctx, block.Cid()) + if !ipld.IsNotFound(err) { + t.Fatal("expected block to not be found") + } + err = bs.Put(ctx, block) + if err != nil { + t.Fatal(err) + } + b, err := sess.GetBlock(ctx, block.Cid()) + if err != nil { + t.Fatal(err) + } + if b.Cid() != block.Cid() { + t.Fatal("got the wrong block") + } +} diff --git a/blockservice/internal/tracing.go b/blockservice/internal/tracing.go new file mode 100644 index 0000000000..96a61ff423 --- /dev/null +++ b/blockservice/internal/tracing.go @@ -0,0 +1,13 @@ +package internal + +import ( + "context" + "fmt" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/trace" +) + +func StartSpan(ctx context.Context, name string, opts ...trace.SpanStartOption) (context.Context, trace.Span) { + return otel.Tracer("go-blockservice").Start(ctx, fmt.Sprintf("Blockservice.%s", name), opts...) +} diff --git a/blockservice/test/blocks_test.go b/blockservice/test/blocks_test.go new file mode 100644 index 0000000000..76b3e37722 --- /dev/null +++ b/blockservice/test/blocks_test.go @@ -0,0 +1,100 @@ +package bstest + +import ( + "bytes" + "context" + "fmt" + "testing" + "time" + + . "github.com/ipfs/go-libipfs/blockservice" + + cid "github.com/ipfs/go-cid" + ds "github.com/ipfs/go-datastore" + dssync "github.com/ipfs/go-datastore/sync" + blocks "github.com/ipfs/go-libipfs/blocks" + blockstore "github.com/ipfs/go-libipfs/blockstore" + offline "github.com/ipfs/go-libipfs/exchange/offline" + u "github.com/ipfs/go-libipfs/util" +) + +func newObject(data []byte) blocks.Block { + return blocks.NewBlock(data) +} + +func TestBlocks(t *testing.T) { + bstore := blockstore.NewBlockstore(dssync.MutexWrap(ds.NewMapDatastore())) + bs := New(bstore, offline.Exchange(bstore)) + defer bs.Close() + + o := newObject([]byte("beep boop")) + h := cid.NewCidV0(u.Hash([]byte("beep boop"))) + if !o.Cid().Equals(h) { + t.Error("Block key and data multihash key not equal") + } + + err := bs.AddBlock(context.Background(), o) + if err != nil { + t.Error("failed to add block to BlockService", err) + return + } + + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + b2, err := bs.GetBlock(ctx, o.Cid()) + if err != nil { + t.Error("failed to retrieve block from BlockService", err) + return + } + + if !o.Cid().Equals(b2.Cid()) { + t.Error("Block keys not equal.") + } + + if !bytes.Equal(o.RawData(), b2.RawData()) { + t.Error("Block data is not equal.") + } +} + +func makeObjects(n int) []blocks.Block { + var out []blocks.Block + for i := 0; i < n; i++ { + out = append(out, newObject([]byte(fmt.Sprintf("object %d", i)))) + } + return out +} + +func TestGetBlocksSequential(t *testing.T) { + var servs = Mocks(4) + for _, s := range servs { + defer s.Close() + } + objs := makeObjects(50) + + var cids []cid.Cid + for _, o := range objs { + cids = append(cids, o.Cid()) + err := servs[0].AddBlock(context.Background(), o) + if err != nil { + t.Fatal(err) + } + } + + t.Log("one instance at a time, get blocks concurrently") + + for i := 1; i < len(servs); i++ { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*50) + defer cancel() + out := servs[i].GetBlocks(ctx, cids) + gotten := make(map[string]blocks.Block) + for blk := range out { + if _, ok := gotten[blk.Cid().KeyString()]; ok { + t.Fatal("Got duplicate block!") + } + gotten[blk.Cid().KeyString()] = blk + } + if len(gotten) != len(objs) { + t.Fatalf("Didnt get enough blocks back: %d/%d", len(gotten), len(objs)) + } + } +} diff --git a/blockservice/test/mock.go b/blockservice/test/mock.go new file mode 100644 index 0000000000..34a3e42ac3 --- /dev/null +++ b/blockservice/test/mock.go @@ -0,0 +1,23 @@ +package bstest + +import ( + delay "github.com/ipfs/go-ipfs-delay" + testinstance "github.com/ipfs/go-libipfs/bitswap/testinstance" + tn "github.com/ipfs/go-libipfs/bitswap/testnet" + "github.com/ipfs/go-libipfs/blockservice" + mockrouting "github.com/ipfs/go-libipfs/routing/mock" +) + +// Mocks returns |n| connected mock Blockservices +func Mocks(n int) []blockservice.BlockService { + net := tn.VirtualNetwork(mockrouting.NewServer(), delay.Fixed(0)) + sg := testinstance.NewTestInstanceGenerator(net, nil, nil) + + instances := sg.Instances(n) + + var servs []blockservice.BlockService + for _, i := range instances { + servs = append(servs, blockservice.New(i.Blockstore(), i.Exchange)) + } + return servs +} diff --git a/blockstore/arc_cache.go b/blockstore/arc_cache.go new file mode 100644 index 0000000000..c1919e922f --- /dev/null +++ b/blockstore/arc_cache.go @@ -0,0 +1,408 @@ +package blockstore + +import ( + "context" + "sort" + "sync" + + lru "github.com/hashicorp/golang-lru" + cid "github.com/ipfs/go-cid" + ipld "github.com/ipfs/go-ipld-format" + blocks "github.com/ipfs/go-libipfs/blocks" + metrics "github.com/ipfs/go-metrics-interface" +) + +type cacheHave bool +type cacheSize int + +type lock struct { + mu sync.RWMutex + refcnt int +} + +// arccache wraps a BlockStore with an Adaptive Replacement Cache (ARC) that +// does not store the actual blocks, just metadata about them: existence and +// size. This provides block access-time improvements, allowing +// to short-cut many searches without querying the underlying datastore. +type arccache struct { + lklk sync.Mutex + lks map[string]*lock + + cache *lru.TwoQueueCache + + blockstore Blockstore + viewer Viewer + + hits metrics.Counter + total metrics.Counter +} + +var _ Blockstore = (*arccache)(nil) +var _ Viewer = (*arccache)(nil) + +func newARCCachedBS(ctx context.Context, bs Blockstore, lruSize int) (*arccache, error) { + cache, err := lru.New2Q(lruSize) + if err != nil { + return nil, err + } + c := &arccache{cache: cache, blockstore: bs, lks: make(map[string]*lock)} + c.hits = metrics.NewCtx(ctx, "arc.hits_total", "Number of ARC cache hits").Counter() + c.total = metrics.NewCtx(ctx, "arc_total", "Total number of ARC cache requests").Counter() + if v, ok := bs.(Viewer); ok { + c.viewer = v + } + return c, nil +} + +func (b *arccache) lock(k string, write bool) { + b.lklk.Lock() + lk, ok := b.lks[k] + if !ok { + lk = new(lock) + b.lks[k] = lk + } + lk.refcnt++ + b.lklk.Unlock() + if write { + lk.mu.Lock() + } else { + lk.mu.RLock() + } +} + +func (b *arccache) unlock(key string, write bool) { + b.lklk.Lock() + lk := b.lks[key] + lk.refcnt-- + if lk.refcnt == 0 { + delete(b.lks, key) + } + b.lklk.Unlock() + if write { + lk.mu.Unlock() + } else { + lk.mu.RUnlock() + } +} + +func cacheKey(k cid.Cid) string { + return string(k.Hash()) +} + +func (b *arccache) DeleteBlock(ctx context.Context, k cid.Cid) error { + if !k.Defined() { + return nil + } + + key := cacheKey(k) + + if has, _, ok := b.queryCache(key); ok && !has { + return nil + } + + b.lock(key, true) + defer b.unlock(key, true) + + err := b.blockstore.DeleteBlock(ctx, k) + if err == nil { + b.cacheHave(key, false) + } else { + b.cacheInvalidate(key) + } + return err +} + +func (b *arccache) Has(ctx context.Context, k cid.Cid) (bool, error) { + if !k.Defined() { + logger.Error("undefined cid in arccache") + // Return cache invalid so the call to blockstore happens + // in case of invalid key and correct error is created. + return false, nil + } + + key := cacheKey(k) + + if has, _, ok := b.queryCache(key); ok { + return has, nil + } + + b.lock(key, false) + defer b.unlock(key, false) + + has, err := b.blockstore.Has(ctx, k) + if err != nil { + return false, err + } + b.cacheHave(key, has) + return has, nil +} + +func (b *arccache) GetSize(ctx context.Context, k cid.Cid) (int, error) { + if !k.Defined() { + return -1, ipld.ErrNotFound{Cid: k} + } + + key := cacheKey(k) + + if has, blockSize, ok := b.queryCache(key); ok { + if !has { + // don't have it, return + return -1, ipld.ErrNotFound{Cid: k} + } + if blockSize >= 0 { + // have it and we know the size + return blockSize, nil + } + // we have it but don't know the size, ask the datastore. + } + + b.lock(key, false) + defer b.unlock(key, false) + + blockSize, err := b.blockstore.GetSize(ctx, k) + if ipld.IsNotFound(err) { + b.cacheHave(key, false) + } else if err == nil { + b.cacheSize(key, blockSize) + } + return blockSize, err +} + +func (b *arccache) View(ctx context.Context, k cid.Cid, callback func([]byte) error) error { + // shortcircuit and fall back to Get if the underlying store + // doesn't support Viewer. + if b.viewer == nil { + blk, err := b.Get(ctx, k) + if err != nil { + return err + } + return callback(blk.RawData()) + } + + if !k.Defined() { + return ipld.ErrNotFound{Cid: k} + } + + key := cacheKey(k) + + if has, _, ok := b.queryCache(key); ok && !has { + // short circuit if the cache deterministically tells us the item + // doesn't exist. + return ipld.ErrNotFound{Cid: k} + } + + b.lock(key, false) + defer b.unlock(key, false) + + var cberr error + var size int + if err := b.viewer.View(ctx, k, func(buf []byte) error { + size = len(buf) + cberr = callback(buf) + return nil + }); err != nil { + if ipld.IsNotFound(err) { + b.cacheHave(key, false) + } + return err + } + + b.cacheSize(key, size) + + return cberr +} + +func (b *arccache) Get(ctx context.Context, k cid.Cid) (blocks.Block, error) { + if !k.Defined() { + return nil, ipld.ErrNotFound{Cid: k} + } + + key := cacheKey(k) + + if has, _, ok := b.queryCache(key); ok && !has { + return nil, ipld.ErrNotFound{Cid: k} + } + + b.lock(key, false) + defer b.unlock(key, false) + + bl, err := b.blockstore.Get(ctx, k) + if bl == nil && ipld.IsNotFound(err) { + b.cacheHave(key, false) + } else if bl != nil { + b.cacheSize(key, len(bl.RawData())) + } + return bl, err +} + +func (b *arccache) Put(ctx context.Context, bl blocks.Block) error { + key := cacheKey(bl.Cid()) + + if has, _, ok := b.queryCache(key); ok && has { + return nil + } + + b.lock(key, true) + defer b.unlock(key, true) + + err := b.blockstore.Put(ctx, bl) + if err == nil { + b.cacheSize(key, len(bl.RawData())) + } else { + b.cacheInvalidate(key) + } + return err +} + +type keyedBlocks struct { + keys []string + blocks []blocks.Block +} + +func (b *keyedBlocks) Len() int { + return len(b.keys) +} + +func (b *keyedBlocks) Less(i, j int) bool { + return b.keys[i] < b.keys[j] +} + +func (b *keyedBlocks) Swap(i, j int) { + b.keys[i], b.keys[j] = b.keys[j], b.keys[i] + b.blocks[i], b.blocks[j] = b.blocks[j], b.blocks[i] +} + +func (b *keyedBlocks) append(key string, blk blocks.Block) { + b.keys = append(b.keys, key) + b.blocks = append(b.blocks, blk) +} + +func (b *keyedBlocks) isEmpty() bool { + return len(b.keys) == 0 +} + +func (b *keyedBlocks) sortAndDedup() { + if b.isEmpty() { + return + } + + sort.Sort(b) + + // https://github.com/golang/go/wiki/SliceTricks#in-place-deduplicate-comparable + j := 0 + for i := 1; i < len(b.keys); i++ { + if b.keys[j] == b.keys[i] { + continue + } + j++ + b.keys[j] = b.keys[i] + b.blocks[j] = b.blocks[i] + } + + b.keys = b.keys[:j+1] + b.blocks = b.blocks[:j+1] +} + +func newKeyedBlocks(cap int) *keyedBlocks { + return &keyedBlocks{ + keys: make([]string, 0, cap), + blocks: make([]blocks.Block, 0, cap), + } +} + +func (b *arccache) PutMany(ctx context.Context, bs []blocks.Block) error { + good := newKeyedBlocks(len(bs)) + for _, blk := range bs { + // call put on block if result is inconclusive or we are sure that + // the block isn't in storage + key := cacheKey(blk.Cid()) + if has, _, ok := b.queryCache(key); !ok || (ok && !has) { + good.append(key, blk) + } + } + + if good.isEmpty() { + return nil + } + + good.sortAndDedup() + + for _, key := range good.keys { + b.lock(key, true) + } + + defer func() { + for _, key := range good.keys { + b.unlock(key, true) + } + }() + + err := b.blockstore.PutMany(ctx, good.blocks) + if err != nil { + return err + } + for i, key := range good.keys { + b.cacheSize(key, len(good.blocks[i].RawData())) + } + + return nil +} + +func (b *arccache) HashOnRead(enabled bool) { + b.blockstore.HashOnRead(enabled) +} + +func (b *arccache) cacheHave(key string, have bool) { + b.cache.Add(key, cacheHave(have)) +} + +func (b *arccache) cacheSize(key string, blockSize int) { + b.cache.Add(key, cacheSize(blockSize)) +} + +func (b *arccache) cacheInvalidate(key string) { + b.cache.Remove(key) +} + +// queryCache checks if the CID is in the cache. If so, it returns: +// +// - exists (bool): whether the CID is known to exist or not. +// - size (int): the size if cached, or -1 if not cached. +// - ok (bool): whether present in the cache. +// +// When ok is false, the answer in inconclusive and the caller must ignore the +// other two return values. Querying the underying store is necessary. +// +// When ok is true, exists carries the correct answer, and size carries the +// size, if known, or -1 if not. +func (b *arccache) queryCache(k string) (exists bool, size int, ok bool) { + b.total.Inc() + + h, ok := b.cache.Get(k) + if ok { + b.hits.Inc() + switch h := h.(type) { + case cacheHave: + return bool(h), -1, true + case cacheSize: + return true, int(h), true + } + } + return false, -1, false +} + +func (b *arccache) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) { + return b.blockstore.AllKeysChan(ctx) +} + +func (b *arccache) GCLock(ctx context.Context) Unlocker { + return b.blockstore.(GCBlockstore).GCLock(ctx) +} + +func (b *arccache) PinLock(ctx context.Context) Unlocker { + return b.blockstore.(GCBlockstore).PinLock(ctx) +} + +func (b *arccache) GCRequested(ctx context.Context) bool { + return b.blockstore.(GCBlockstore).GCRequested(ctx) +} diff --git a/blockstore/arc_cache_test.go b/blockstore/arc_cache_test.go new file mode 100644 index 0000000000..ee2ed6e76f --- /dev/null +++ b/blockstore/arc_cache_test.go @@ -0,0 +1,399 @@ +package blockstore + +import ( + "context" + "io" + "math/rand" + "sync/atomic" + "testing" + "time" + + cid "github.com/ipfs/go-cid" + ds "github.com/ipfs/go-datastore" + syncds "github.com/ipfs/go-datastore/sync" + ipld "github.com/ipfs/go-ipld-format" + blocks "github.com/ipfs/go-libipfs/blocks" +) + +var exampleBlock = blocks.NewBlock([]byte("foo")) + +func testArcCached(ctx context.Context, bs Blockstore) (*arccache, error) { + if ctx == nil { + ctx = context.TODO() + } + opts := DefaultCacheOpts() + opts.HasBloomFilterSize = 0 + opts.HasBloomFilterHashes = 0 + bbs, err := CachedBlockstore(ctx, bs, opts) + if err == nil { + return bbs.(*arccache), nil + } + return nil, err +} + +func createStores(t testing.TB) (*arccache, Blockstore, *callbackDatastore) { + cd := &callbackDatastore{f: func() {}, ds: ds.NewMapDatastore()} + bs := NewBlockstore(syncds.MutexWrap(cd)) + arc, err := testArcCached(context.TODO(), bs) + if err != nil { + t.Fatal(err) + } + return arc, bs, cd +} + +func trap(message string, cd *callbackDatastore, t *testing.T) { + cd.SetFunc(func() { + t.Fatal(message) + }) +} +func untrap(cd *callbackDatastore) { + cd.SetFunc(func() {}) +} + +func TestRemoveCacheEntryOnDelete(t *testing.T) { + arc, _, cd := createStores(t) + + arc.Put(bg, exampleBlock) + + cd.Lock() + writeHitTheDatastore := false + cd.Unlock() + + cd.SetFunc(func() { + writeHitTheDatastore = true + }) + + arc.DeleteBlock(bg, exampleBlock.Cid()) + arc.Put(bg, exampleBlock) + if !writeHitTheDatastore { + t.Fail() + } +} + +func TestElideDuplicateWrite(t *testing.T) { + arc, _, cd := createStores(t) + + arc.Put(bg, exampleBlock) + trap("write hit datastore", cd, t) + arc.Put(bg, exampleBlock) +} + +func TestHasRequestTriggersCache(t *testing.T) { + arc, _, cd := createStores(t) + + arc.Has(bg, exampleBlock.Cid()) + trap("has hit datastore", cd, t) + if has, err := arc.Has(bg, exampleBlock.Cid()); has || err != nil { + t.Fatal("has was true but there is no such block") + } + + untrap(cd) + err := arc.Put(bg, exampleBlock) + if err != nil { + t.Fatal(err) + } + + trap("has hit datastore", cd, t) + + if has, err := arc.Has(bg, exampleBlock.Cid()); !has || err != nil { + t.Fatal("has returned invalid result") + } +} + +func TestGetFillsCache(t *testing.T) { + arc, _, cd := createStores(t) + + if bl, err := arc.Get(bg, exampleBlock.Cid()); bl != nil || err == nil { + t.Fatal("block was found or there was no error") + } + + trap("has hit datastore", cd, t) + + if has, err := arc.Has(bg, exampleBlock.Cid()); has || err != nil { + t.Fatal("has was true but there is no such block") + } + if _, err := arc.GetSize(bg, exampleBlock.Cid()); !ipld.IsNotFound(err) { + t.Fatal("getsize was true but there is no such block") + } + + untrap(cd) + + if err := arc.Put(bg, exampleBlock); err != nil { + t.Fatal(err) + } + + trap("has hit datastore", cd, t) + + if has, err := arc.Has(bg, exampleBlock.Cid()); !has || err != nil { + t.Fatal("has returned invalid result") + } + if blockSize, err := arc.GetSize(bg, exampleBlock.Cid()); blockSize == -1 || err != nil { + t.Fatal("getsize returned invalid result", blockSize, err) + } +} + +func TestGetAndDeleteFalseShortCircuit(t *testing.T) { + arc, _, cd := createStores(t) + + arc.Has(bg, exampleBlock.Cid()) + arc.GetSize(bg, exampleBlock.Cid()) + + trap("get hit datastore", cd, t) + + if bl, err := arc.Get(bg, exampleBlock.Cid()); bl != nil || !ipld.IsNotFound(err) { + t.Fatal("get returned invalid result") + } + + if arc.DeleteBlock(bg, exampleBlock.Cid()) != nil { + t.Fatal("expected deletes to be idempotent") + } +} + +func TestArcCreationFailure(t *testing.T) { + if arc, err := newARCCachedBS(context.TODO(), nil, -1); arc != nil || err == nil { + t.Fatal("expected error and no cache") + } +} + +func TestInvalidKey(t *testing.T) { + arc, _, _ := createStores(t) + + bl, err := arc.Get(bg, cid.Cid{}) + + if bl != nil { + t.Fatal("blocks should be nil") + } + if err == nil { + t.Fatal("expected error") + } +} + +func TestHasAfterSucessfulGetIsCached(t *testing.T) { + arc, bs, cd := createStores(t) + + bs.Put(bg, exampleBlock) + + arc.Get(bg, exampleBlock.Cid()) + + trap("has hit datastore", cd, t) + arc.Has(bg, exampleBlock.Cid()) +} + +func TestGetSizeAfterSucessfulGetIsCached(t *testing.T) { + arc, bs, cd := createStores(t) + + bs.Put(bg, exampleBlock) + + arc.Get(bg, exampleBlock.Cid()) + + trap("has hit datastore", cd, t) + arc.GetSize(bg, exampleBlock.Cid()) +} + +func TestGetSizeAfterSucessfulHas(t *testing.T) { + arc, bs, _ := createStores(t) + + bs.Put(bg, exampleBlock) + has, err := arc.Has(bg, exampleBlock.Cid()) + if err != nil { + t.Fatal(err) + } + if !has { + t.Fatal("expected to have block") + } + + if size, err := arc.GetSize(bg, exampleBlock.Cid()); err != nil { + t.Fatal(err) + } else if size != len(exampleBlock.RawData()) { + t.Fatalf("expected size %d, got %d", len(exampleBlock.RawData()), size) + } +} + +func TestGetSizeMissingZeroSizeBlock(t *testing.T) { + arc, bs, cd := createStores(t) + emptyBlock := blocks.NewBlock([]byte{}) + missingBlock := blocks.NewBlock([]byte("missingBlock")) + + bs.Put(bg, emptyBlock) + + arc.Get(bg, emptyBlock.Cid()) + + trap("has hit datastore", cd, t) + if blockSize, err := arc.GetSize(bg, emptyBlock.Cid()); blockSize != 0 || err != nil { + t.Fatal("getsize returned invalid result") + } + untrap(cd) + + arc.Get(bg, missingBlock.Cid()) + + trap("has hit datastore", cd, t) + if _, err := arc.GetSize(bg, missingBlock.Cid()); !ipld.IsNotFound(err) { + t.Fatal("getsize returned invalid result") + } +} + +func TestDifferentKeyObjectsWork(t *testing.T) { + arc, bs, cd := createStores(t) + + bs.Put(bg, exampleBlock) + + arc.Get(bg, exampleBlock.Cid()) + + trap("has hit datastore", cd, t) + cidstr := exampleBlock.Cid().String() + + ncid, err := cid.Decode(cidstr) + if err != nil { + t.Fatal(err) + } + + arc.Has(bg, ncid) +} + +func TestPutManyCaches(t *testing.T) { + t.Run("happy path PutMany", func(t *testing.T) { + arc, _, cd := createStores(t) + arc.PutMany(bg, []blocks.Block{exampleBlock}) + + trap("has hit datastore", cd, t) + arc.Has(bg, exampleBlock.Cid()) + arc.GetSize(bg, exampleBlock.Cid()) + untrap(cd) + arc.DeleteBlock(bg, exampleBlock.Cid()) + + arc.Put(bg, exampleBlock) + trap("PunMany has hit datastore", cd, t) + arc.PutMany(bg, []blocks.Block{exampleBlock}) + }) + + t.Run("PutMany with duplicates", func(t *testing.T) { + arc, _, cd := createStores(t) + arc.PutMany(bg, []blocks.Block{exampleBlock, exampleBlock}) + + trap("has hit datastore", cd, t) + arc.Has(bg, exampleBlock.Cid()) + arc.GetSize(bg, exampleBlock.Cid()) + untrap(cd) + arc.DeleteBlock(bg, exampleBlock.Cid()) + + arc.Put(bg, exampleBlock) + trap("PunMany has hit datastore", cd, t) + arc.PutMany(bg, []blocks.Block{exampleBlock}) + }) +} + +func BenchmarkARCCacheConcurrentOps(b *testing.B) { + // ~4k blocks seems high enough to be realistic, + // but low enough to cause collisions. + // Keep it as a power of 2, to simplify code below. + const numBlocks = 4 << 10 + + dummyBlocks := make([]blocks.Block, numBlocks) + + { + // scope dummyRand to prevent its unsafe concurrent use below + dummyRand := rand.New(rand.NewSource(time.Now().UnixNano())) + for i := range dummyBlocks { + dummy := make([]byte, 32) + if _, err := io.ReadFull(dummyRand, dummy); err != nil { + b.Fatal(err) + } + dummyBlocks[i] = blocks.NewBlock(dummy) + } + } + + // Each test begins with half the blocks present in the cache. + // This allows test cases to have both hits and misses, + // regardless of whether or not they do Puts. + putHalfBlocks := func(arc *arccache) { + for i, block := range dummyBlocks { + if i%2 == 0 { + if err := arc.Put(bg, block); err != nil { + b.Fatal(err) + } + } + } + } + + // We always mix just two operations at a time. + const numOps = 2 + var testOps = []struct { + name string + ops [numOps]func(*arccache, blocks.Block) + }{ + {"PutDelete", [...]func(*arccache, blocks.Block){ + func(arc *arccache, block blocks.Block) { + arc.Put(bg, block) + }, + func(arc *arccache, block blocks.Block) { + arc.DeleteBlock(bg, block.Cid()) + }, + }}, + {"GetDelete", [...]func(*arccache, blocks.Block){ + func(arc *arccache, block blocks.Block) { + arc.Get(bg, block.Cid()) + }, + func(arc *arccache, block blocks.Block) { + arc.DeleteBlock(bg, block.Cid()) + }, + }}, + {"GetPut", [...]func(*arccache, blocks.Block){ + func(arc *arccache, block blocks.Block) { + arc.Get(bg, block.Cid()) + }, + func(arc *arccache, block blocks.Block) { + arc.Put(bg, block) + }, + }}, + } + + for _, test := range testOps { + test := test // prevent reuse of the range var + b.Run(test.name, func(b *testing.B) { + arc, _, _ := createStores(b) + putHalfBlocks(arc) + var opCounts [numOps]uint64 + + b.ResetTimer() + b.ReportAllocs() + + b.RunParallel(func(pb *testing.PB) { + rnd := rand.New(rand.NewSource(time.Now().UnixNano())) + for pb.Next() { + n := rnd.Int63() + blockIdx := n % numBlocks // lower bits decide the block + opIdx := (n / numBlocks) % numOps // higher bits decide what operation + + block := dummyBlocks[blockIdx] + op := test.ops[opIdx] + op(arc, block) + + atomic.AddUint64(&opCounts[opIdx], 1) + } + }) + + // We expect each op to fire roughly an equal amount of times. + // Error otherwise, as that likely means the logic is wrong. + var minIdx, maxIdx int + var minCount, maxCount uint64 + for opIdx, count := range opCounts { + if minCount == 0 || count < minCount { + minIdx = opIdx + minCount = count + } + if maxCount == 0 || count > maxCount { + maxIdx = opIdx + maxCount = count + } + } + // Skip this check if we ran few times, to avoid false positives. + if maxCount > 100 { + ratio := float64(maxCount) / float64(minCount) + if maxRatio := 2.0; ratio > maxRatio { + b.Fatalf("op %d ran %fx as many times as %d", maxIdx, ratio, minIdx) + } + } + + }) + } +} diff --git a/blockstore/blockstore.go b/blockstore/blockstore.go new file mode 100644 index 0000000000..89bad9b8b5 --- /dev/null +++ b/blockstore/blockstore.go @@ -0,0 +1,346 @@ +// Package blockstore implements a thin wrapper over a datastore, giving a +// clean interface for Getting and Putting block objects. +package blockstore + +import ( + "context" + "errors" + "sync" + "sync/atomic" + + cid "github.com/ipfs/go-cid" + ds "github.com/ipfs/go-datastore" + dsns "github.com/ipfs/go-datastore/namespace" + dsq "github.com/ipfs/go-datastore/query" + ipld "github.com/ipfs/go-ipld-format" + blocks "github.com/ipfs/go-libipfs/blocks" + dshelp "github.com/ipfs/go-libipfs/datastore/dshelp" + logging "github.com/ipfs/go-log" + uatomic "go.uber.org/atomic" +) + +var logger = logging.Logger("blockstore") + +// BlockPrefix namespaces blockstore datastores +var BlockPrefix = ds.NewKey("blocks") + +// ErrHashMismatch is an error returned when the hash of a block +// is different than expected. +var ErrHashMismatch = errors.New("block in storage has different hash than requested") + +// Blockstore wraps a Datastore block-centered methods and provides a layer +// of abstraction which allows to add different caching strategies. +type Blockstore interface { + DeleteBlock(context.Context, cid.Cid) error + Has(context.Context, cid.Cid) (bool, error) + Get(context.Context, cid.Cid) (blocks.Block, error) + + // GetSize returns the CIDs mapped BlockSize + GetSize(context.Context, cid.Cid) (int, error) + + // Put puts a given block to the underlying datastore + Put(context.Context, blocks.Block) error + + // PutMany puts a slice of blocks at the same time using batching + // capabilities of the underlying datastore whenever possible. + PutMany(context.Context, []blocks.Block) error + + // AllKeysChan returns a channel from which + // the CIDs in the Blockstore can be read. It should respect + // the given context, closing the channel if it becomes Done. + AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) + + // HashOnRead specifies if every read block should be + // rehashed to make sure it matches its CID. + HashOnRead(enabled bool) +} + +// Viewer can be implemented by blockstores that offer zero-copy access to +// values. +// +// Callers of View must not mutate or retain the byte slice, as it could be +// an mmapped memory region, or a pooled byte buffer. +// +// View is especially suitable for deserialising in place. +// +// The callback will only be called iff the query operation is successful (and +// the block is found); otherwise, the error will be propagated. Errors returned +// by the callback will be propagated as well. +type Viewer interface { + View(ctx context.Context, cid cid.Cid, callback func([]byte) error) error +} + +// GCLocker abstract functionality to lock a blockstore when performing +// garbage-collection operations. +type GCLocker interface { + // GCLock locks the blockstore for garbage collection. No operations + // that expect to finish with a pin should ocurr simultaneously. + // Reading during GC is safe, and requires no lock. + GCLock(context.Context) Unlocker + + // PinLock locks the blockstore for sequences of puts expected to finish + // with a pin (before GC). Multiple put->pin sequences can write through + // at the same time, but no GC should happen simulatenously. + // Reading during Pinning is safe, and requires no lock. + PinLock(context.Context) Unlocker + + // GcRequested returns true if GCLock has been called and is waiting to + // take the lock + GCRequested(context.Context) bool +} + +// GCBlockstore is a blockstore that can safely run garbage-collection +// operations. +type GCBlockstore interface { + Blockstore + GCLocker +} + +// NewGCBlockstore returns a default implementation of GCBlockstore +// using the given Blockstore and GCLocker. +func NewGCBlockstore(bs Blockstore, gcl GCLocker) GCBlockstore { + return gcBlockstore{bs, gcl} +} + +type gcBlockstore struct { + Blockstore + GCLocker +} + +// Option is a default implementation Blockstore option +type Option struct { + f func(bs *blockstore) +} + +// WriteThrough skips checking if the blockstore already has a block before +// writing it. +func WriteThrough() Option { + return Option{ + func(bs *blockstore) { + bs.writeThrough = true + }, + } +} + +// NoPrefix avoids wrapping the blockstore into the BlockPrefix namespace +// ("/blocks"), so keys will not be modified in any way. +func NoPrefix() Option { + return Option{ + func(bs *blockstore) { + bs.noPrefix = true + }, + } +} + +// NewBlockstore returns a default Blockstore implementation +// using the provided datastore.Batching backend. +func NewBlockstore(d ds.Batching, opts ...Option) Blockstore { + bs := &blockstore{ + datastore: d, + rehash: uatomic.NewBool(false), + } + + for _, o := range opts { + o.f(bs) + } + + if !bs.noPrefix { + bs.datastore = dsns.Wrap(bs.datastore, BlockPrefix) + } + return bs +} + +// NewBlockstoreNoPrefix returns a default Blockstore implementation +// using the provided datastore.Batching backend. +// This constructor does not modify input keys in any way +// +// Deprecated: Use NewBlockstore with the NoPrefix option instead. +func NewBlockstoreNoPrefix(d ds.Batching) Blockstore { + return NewBlockstore(d, NoPrefix()) +} + +type blockstore struct { + datastore ds.Batching + + rehash *uatomic.Bool + writeThrough bool + noPrefix bool +} + +func (bs *blockstore) HashOnRead(enabled bool) { + bs.rehash.Store(enabled) +} + +func (bs *blockstore) Get(ctx context.Context, k cid.Cid) (blocks.Block, error) { + if !k.Defined() { + logger.Error("undefined cid in blockstore") + return nil, ipld.ErrNotFound{Cid: k} + } + bdata, err := bs.datastore.Get(ctx, dshelp.MultihashToDsKey(k.Hash())) + if err == ds.ErrNotFound { + return nil, ipld.ErrNotFound{Cid: k} + } + if err != nil { + return nil, err + } + if bs.rehash.Load() { + rbcid, err := k.Prefix().Sum(bdata) + if err != nil { + return nil, err + } + + if !rbcid.Equals(k) { + return nil, ErrHashMismatch + } + + return blocks.NewBlockWithCid(bdata, rbcid) + } + return blocks.NewBlockWithCid(bdata, k) +} + +func (bs *blockstore) Put(ctx context.Context, block blocks.Block) error { + k := dshelp.MultihashToDsKey(block.Cid().Hash()) + + // Has is cheaper than Put, so see if we already have it + if !bs.writeThrough { + exists, err := bs.datastore.Has(ctx, k) + if err == nil && exists { + return nil // already stored. + } + } + return bs.datastore.Put(ctx, k, block.RawData()) +} + +func (bs *blockstore) PutMany(ctx context.Context, blocks []blocks.Block) error { + if len(blocks) == 1 { + // performance fast-path + return bs.Put(ctx, blocks[0]) + } + + t, err := bs.datastore.Batch(ctx) + if err != nil { + return err + } + for _, b := range blocks { + k := dshelp.MultihashToDsKey(b.Cid().Hash()) + + if !bs.writeThrough { + exists, err := bs.datastore.Has(ctx, k) + if err == nil && exists { + continue + } + } + + err = t.Put(ctx, k, b.RawData()) + if err != nil { + return err + } + } + return t.Commit(ctx) +} + +func (bs *blockstore) Has(ctx context.Context, k cid.Cid) (bool, error) { + return bs.datastore.Has(ctx, dshelp.MultihashToDsKey(k.Hash())) +} + +func (bs *blockstore) GetSize(ctx context.Context, k cid.Cid) (int, error) { + size, err := bs.datastore.GetSize(ctx, dshelp.MultihashToDsKey(k.Hash())) + if err == ds.ErrNotFound { + return -1, ipld.ErrNotFound{Cid: k} + } + return size, err +} + +func (bs *blockstore) DeleteBlock(ctx context.Context, k cid.Cid) error { + return bs.datastore.Delete(ctx, dshelp.MultihashToDsKey(k.Hash())) +} + +// AllKeysChan runs a query for keys from the blockstore. +// this is very simplistic, in the future, take dsq.Query as a param? +// +// AllKeysChan respects context. +func (bs *blockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) { + + // KeysOnly, because that would be _a lot_ of data. + q := dsq.Query{KeysOnly: true} + res, err := bs.datastore.Query(ctx, q) + if err != nil { + return nil, err + } + + output := make(chan cid.Cid, dsq.KeysOnlyBufSize) + go func() { + defer func() { + res.Close() // ensure exit (signals early exit, too) + close(output) + }() + + for { + e, ok := res.NextSync() + if !ok { + return + } + if e.Error != nil { + logger.Errorf("blockstore.AllKeysChan got err: %s", e.Error) + return + } + + // need to convert to key.Key using key.KeyFromDsKey. + bk, err := dshelp.BinaryFromDsKey(ds.RawKey(e.Key)) + if err != nil { + logger.Warningf("error parsing key from binary: %s", err) + continue + } + k := cid.NewCidV1(cid.Raw, bk) + select { + case <-ctx.Done(): + return + case output <- k: + } + } + }() + + return output, nil +} + +// NewGCLocker returns a default implementation of +// GCLocker using standard [RW] mutexes. +func NewGCLocker() GCLocker { + return &gclocker{} +} + +type gclocker struct { + lk sync.RWMutex + gcreq int32 +} + +// Unlocker represents an object which can Unlock +// something. +type Unlocker interface { + Unlock(context.Context) +} + +type unlocker struct { + unlock func() +} + +func (u *unlocker) Unlock(_ context.Context) { + u.unlock() + u.unlock = nil // ensure its not called twice +} + +func (bs *gclocker) GCLock(_ context.Context) Unlocker { + atomic.AddInt32(&bs.gcreq, 1) + bs.lk.Lock() + atomic.AddInt32(&bs.gcreq, -1) + return &unlocker{bs.lk.Unlock} +} + +func (bs *gclocker) PinLock(_ context.Context) Unlocker { + bs.lk.RLock() + return &unlocker{bs.lk.RUnlock} +} + +func (bs *gclocker) GCRequested(_ context.Context) bool { + return atomic.LoadInt32(&bs.gcreq) > 0 +} diff --git a/blockstore/blockstore_test.go b/blockstore/blockstore_test.go new file mode 100644 index 0000000000..5a9a70e59f --- /dev/null +++ b/blockstore/blockstore_test.go @@ -0,0 +1,333 @@ +package blockstore + +import ( + "bytes" + "context" + "fmt" + "testing" + + cid "github.com/ipfs/go-cid" + ds "github.com/ipfs/go-datastore" + dsq "github.com/ipfs/go-datastore/query" + ds_sync "github.com/ipfs/go-datastore/sync" + ipld "github.com/ipfs/go-ipld-format" + blocks "github.com/ipfs/go-libipfs/blocks" + u "github.com/ipfs/go-libipfs/util" +) + +func TestGetWhenKeyNotPresent(t *testing.T) { + bs := NewBlockstore(ds_sync.MutexWrap(ds.NewMapDatastore())) + c := cid.NewCidV0(u.Hash([]byte("stuff"))) + bl, err := bs.Get(bg, c) + + if bl != nil { + t.Error("nil block expected") + } + if err == nil { + t.Error("error expected, got nil") + } +} + +func TestGetWhenKeyIsNil(t *testing.T) { + bs := NewBlockstore(ds_sync.MutexWrap(ds.NewMapDatastore())) + _, err := bs.Get(bg, cid.Cid{}) + if !ipld.IsNotFound(err) { + t.Fail() + } +} + +func TestPutThenGetBlock(t *testing.T) { + bs := NewBlockstore(ds_sync.MutexWrap(ds.NewMapDatastore())) + block := blocks.NewBlock([]byte("some data")) + + err := bs.Put(bg, block) + if err != nil { + t.Fatal(err) + } + + blockFromBlockstore, err := bs.Get(bg, block.Cid()) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(block.RawData(), blockFromBlockstore.RawData()) { + t.Fail() + } +} + +func TestCidv0v1(t *testing.T) { + bs := NewBlockstore(ds_sync.MutexWrap(ds.NewMapDatastore())) + block := blocks.NewBlock([]byte("some data")) + + err := bs.Put(bg, block) + if err != nil { + t.Fatal(err) + } + + blockFromBlockstore, err := bs.Get(bg, cid.NewCidV1(cid.DagProtobuf, block.Cid().Hash())) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(block.RawData(), blockFromBlockstore.RawData()) { + t.Fail() + } +} + +func TestPutThenGetSizeBlock(t *testing.T) { + bs := NewBlockstore(ds_sync.MutexWrap(ds.NewMapDatastore())) + block := blocks.NewBlock([]byte("some data")) + missingBlock := blocks.NewBlock([]byte("missingBlock")) + emptyBlock := blocks.NewBlock([]byte{}) + + err := bs.Put(bg, block) + if err != nil { + t.Fatal(err) + } + + blockSize, err := bs.GetSize(bg, block.Cid()) + if err != nil { + t.Fatal(err) + } + if len(block.RawData()) != blockSize { + t.Fail() + } + + err = bs.Put(bg, emptyBlock) + if err != nil { + t.Fatal(err) + } + + if blockSize, err := bs.GetSize(bg, emptyBlock.Cid()); blockSize != 0 || err != nil { + t.Fatal(err) + } + + if blockSize, err := bs.GetSize(bg, missingBlock.Cid()); blockSize != -1 || err == nil { + t.Fatal("getsize returned invalid result") + } +} + +type countHasDS struct { + ds.Datastore + hasCount int +} + +func (ds *countHasDS) Has(ctx context.Context, key ds.Key) (exists bool, err error) { + ds.hasCount += 1 + return ds.Datastore.Has(ctx, key) +} + +func TestPutUsesHas(t *testing.T) { + // Some datastores rely on the implementation detail that Put checks Has + // first, to avoid overriding existing objects' metadata. This test ensures + // that Blockstore continues to behave this way. + // Please ping https://github.com/ipfs/go-libipfs/blockstore/pull/47 if this + // behavior is being removed. + ds := &countHasDS{ + Datastore: ds.NewMapDatastore(), + } + bs := NewBlockstore(ds_sync.MutexWrap(ds)) + bl := blocks.NewBlock([]byte("some data")) + if err := bs.Put(bg, bl); err != nil { + t.Fatal(err) + } + if err := bs.Put(bg, bl); err != nil { + t.Fatal(err) + } + if ds.hasCount != 2 { + t.Errorf("Blockstore did not call Has before attempting Put, this breaks compatibility") + } +} + +func TestHashOnRead(t *testing.T) { + orginalDebug := u.Debug + defer (func() { + u.Debug = orginalDebug + })() + u.Debug = false + + bs := NewBlockstore(ds_sync.MutexWrap(ds.NewMapDatastore())) + bl := blocks.NewBlock([]byte("some data")) + blBad, err := blocks.NewBlockWithCid([]byte("some other data"), bl.Cid()) + if err != nil { + t.Fatal("debug is off, still got an error") + } + bl2 := blocks.NewBlock([]byte("some other data")) + bs.Put(bg, blBad) + bs.Put(bg, bl2) + bs.HashOnRead(true) + + if _, err := bs.Get(bg, bl.Cid()); err != ErrHashMismatch { + t.Fatalf("expected '%v' got '%v'\n", ErrHashMismatch, err) + } + + if b, err := bs.Get(bg, bl2.Cid()); err != nil || b.String() != bl2.String() { + t.Fatal("got wrong blocks") + } +} + +func newBlockStoreWithKeys(t *testing.T, d ds.Datastore, N int) (Blockstore, []cid.Cid) { + if d == nil { + d = ds.NewMapDatastore() + } + bs := NewBlockstore(ds_sync.MutexWrap(d)) + + keys := make([]cid.Cid, N) + for i := 0; i < N; i++ { + block := blocks.NewBlock([]byte(fmt.Sprintf("some data %d", i))) + err := bs.Put(bg, block) + if err != nil { + t.Fatal(err) + } + keys[i] = block.Cid() + } + return bs, keys +} + +func collect(ch <-chan cid.Cid) []cid.Cid { + var keys []cid.Cid + for k := range ch { + keys = append(keys, k) + } + return keys +} + +func TestAllKeysSimple(t *testing.T) { + bs, keys := newBlockStoreWithKeys(t, nil, 100) + + ctx := context.Background() + ch, err := bs.AllKeysChan(ctx) + if err != nil { + t.Fatal(err) + } + keys2 := collect(ch) + + // for _, k2 := range keys2 { + // t.Log("found ", k2.B58String()) + // } + + expectMatches(t, keys, keys2) +} + +func TestAllKeysRespectsContext(t *testing.T) { + N := 100 + + d := &queryTestDS{ds: ds.NewMapDatastore()} + bs, _ := newBlockStoreWithKeys(t, d, N) + + started := make(chan struct{}, 1) + done := make(chan struct{}, 1) + errors := make(chan error, 100) + + getKeys := func(ctx context.Context) { + started <- struct{}{} + ch, err := bs.AllKeysChan(ctx) // once without cancelling + if err != nil { + errors <- err + } + _ = collect(ch) + done <- struct{}{} + errors <- nil // a nil one to signal break + } + + var results dsq.Results + var resultsmu = make(chan struct{}) + resultChan := make(chan dsq.Result) + d.SetFunc(func(q dsq.Query) (dsq.Results, error) { + results = dsq.ResultsWithChan(q, resultChan) + resultsmu <- struct{}{} + return results, nil + }) + + go getKeys(context.Background()) + + // make sure it's waiting. + <-started + <-resultsmu + select { + case <-done: + t.Fatal("sync is wrong") + case <-results.Process().Closing(): + t.Fatal("should not be closing") + case <-results.Process().Closed(): + t.Fatal("should not be closed") + default: + } + + e := dsq.Entry{Key: BlockPrefix.ChildString("foo").String()} + resultChan <- dsq.Result{Entry: e} // let it go. + close(resultChan) + <-done // should be done now. + <-results.Process().Closed() // should be closed now + + // print any errors + for err := range errors { + if err == nil { + break + } + t.Error(err) + } + +} + +func expectMatches(t *testing.T, expect, actual []cid.Cid) { + t.Helper() + + if len(expect) != len(actual) { + t.Errorf("expect and actual differ: %d != %d", len(expect), len(actual)) + } + + actualSet := make(map[string]bool, len(actual)) + for _, k := range actual { + actualSet[string(k.Hash())] = true + } + + for _, ek := range expect { + if !actualSet[string(ek.Hash())] { + t.Error("expected key not found: ", ek) + } + } +} + +type queryTestDS struct { + cb func(q dsq.Query) (dsq.Results, error) + ds ds.Datastore +} + +func (c *queryTestDS) SetFunc(f func(dsq.Query) (dsq.Results, error)) { c.cb = f } + +func (c *queryTestDS) Put(ctx context.Context, key ds.Key, value []byte) (err error) { + return c.ds.Put(ctx, key, value) +} + +func (c *queryTestDS) Get(ctx context.Context, key ds.Key) (value []byte, err error) { + return c.ds.Get(ctx, key) +} + +func (c *queryTestDS) Has(ctx context.Context, key ds.Key) (exists bool, err error) { + return c.ds.Has(ctx, key) +} + +func (c *queryTestDS) GetSize(ctx context.Context, key ds.Key) (size int, err error) { + return c.ds.GetSize(ctx, key) +} + +func (c *queryTestDS) Delete(ctx context.Context, key ds.Key) (err error) { + return c.ds.Delete(ctx, key) +} + +func (c *queryTestDS) Query(ctx context.Context, q dsq.Query) (dsq.Results, error) { + if c.cb != nil { + return c.cb(q) + } + return c.ds.Query(ctx, q) +} + +func (c *queryTestDS) Sync(ctx context.Context, key ds.Key) error { + return c.ds.Sync(ctx, key) +} + +func (c *queryTestDS) Batch(_ context.Context) (ds.Batch, error) { + return ds.NewBasicBatch(c), nil +} +func (c *queryTestDS) Close() error { + return nil +} diff --git a/blockstore/bloom_cache.go b/blockstore/bloom_cache.go new file mode 100644 index 0000000000..44b7590486 --- /dev/null +++ b/blockstore/bloom_cache.go @@ -0,0 +1,231 @@ +package blockstore + +import ( + "context" + "fmt" + "sync/atomic" + "time" + + bloom "github.com/ipfs/bbloom" + cid "github.com/ipfs/go-cid" + ipld "github.com/ipfs/go-ipld-format" + blocks "github.com/ipfs/go-libipfs/blocks" + metrics "github.com/ipfs/go-metrics-interface" +) + +// bloomCached returns a Blockstore that caches Has requests using a Bloom +// filter. bloomSize is size of bloom filter in bytes. hashCount specifies the +// number of hashing functions in the bloom filter (usually known as k). +func bloomCached(ctx context.Context, bs Blockstore, bloomSize, hashCount int) (*bloomcache, error) { + bl, err := bloom.New(float64(bloomSize), float64(hashCount)) + if err != nil { + return nil, err + } + bc := &bloomcache{ + blockstore: bs, + bloom: bl, + hits: metrics.NewCtx(ctx, "bloom.hits_total", + "Number of cache hits in bloom cache").Counter(), + total: metrics.NewCtx(ctx, "bloom_total", + "Total number of requests to bloom cache").Counter(), + buildChan: make(chan struct{}), + } + if v, ok := bs.(Viewer); ok { + bc.viewer = v + } + go func() { + err := bc.build(ctx) + if err != nil { + select { + case <-ctx.Done(): + logger.Warning("Cache rebuild closed by context finishing: ", err) + default: + logger.Error(err) + } + return + } + if metrics.Active() { + fill := metrics.NewCtx(ctx, "bloom_fill_ratio", + "Ratio of bloom filter fullnes, (updated once a minute)").Gauge() + + t := time.NewTicker(1 * time.Minute) + defer t.Stop() + for { + select { + case <-ctx.Done(): + return + case <-t.C: + fill.Set(bc.bloom.FillRatioTS()) + } + } + } + }() + return bc, nil +} + +type bloomcache struct { + active int32 + + bloom *bloom.Bloom + buildErr error + + buildChan chan struct{} + blockstore Blockstore + viewer Viewer + + // Statistics + hits metrics.Counter + total metrics.Counter +} + +var _ Blockstore = (*bloomcache)(nil) +var _ Viewer = (*bloomcache)(nil) + +func (b *bloomcache) BloomActive() bool { + return atomic.LoadInt32(&b.active) != 0 +} + +func (b *bloomcache) Wait(ctx context.Context) error { + select { + case <-ctx.Done(): + return ctx.Err() + case <-b.buildChan: + return b.buildErr + } +} + +func (b *bloomcache) build(ctx context.Context) error { + evt := logger.EventBegin(ctx, "bloomcache.build") + defer evt.Done() + defer close(b.buildChan) + + ch, err := b.blockstore.AllKeysChan(ctx) + if err != nil { + b.buildErr = fmt.Errorf("AllKeysChan failed in bloomcache rebuild with: %v", err) + return b.buildErr + } + for { + select { + case key, ok := <-ch: + if !ok { + atomic.StoreInt32(&b.active, 1) + return nil + } + b.bloom.AddTS(key.Hash()) // Use binary key, the more compact the better + case <-ctx.Done(): + b.buildErr = ctx.Err() + return b.buildErr + } + } +} + +func (b *bloomcache) DeleteBlock(ctx context.Context, k cid.Cid) error { + if has, ok := b.hasCached(k); ok && !has { + return nil + } + + return b.blockstore.DeleteBlock(ctx, k) +} + +// if ok == false has is inconclusive +// if ok == true then has respons to question: is it contained +func (b *bloomcache) hasCached(k cid.Cid) (has bool, ok bool) { + b.total.Inc() + if !k.Defined() { + logger.Error("undefined in bloom cache") + // Return cache invalid so call to blockstore + // in case of invalid key is forwarded deeper + return false, false + } + if b.BloomActive() { + blr := b.bloom.HasTS(k.Hash()) + if !blr { // not contained in bloom is only conclusive answer bloom gives + b.hits.Inc() + return false, true + } + } + return false, false +} + +func (b *bloomcache) Has(ctx context.Context, k cid.Cid) (bool, error) { + if has, ok := b.hasCached(k); ok { + return has, nil + } + + return b.blockstore.Has(ctx, k) +} + +func (b *bloomcache) GetSize(ctx context.Context, k cid.Cid) (int, error) { + if has, ok := b.hasCached(k); ok && !has { + return -1, ipld.ErrNotFound{Cid: k} + } + + return b.blockstore.GetSize(ctx, k) +} + +func (b *bloomcache) View(ctx context.Context, k cid.Cid, callback func([]byte) error) error { + if b.viewer == nil { + blk, err := b.Get(ctx, k) + if err != nil { + return err + } + return callback(blk.RawData()) + } + + if has, ok := b.hasCached(k); ok && !has { + return ipld.ErrNotFound{Cid: k} + } + return b.viewer.View(ctx, k, callback) +} + +func (b *bloomcache) Get(ctx context.Context, k cid.Cid) (blocks.Block, error) { + if has, ok := b.hasCached(k); ok && !has { + return nil, ipld.ErrNotFound{Cid: k} + } + + return b.blockstore.Get(ctx, k) +} + +func (b *bloomcache) Put(ctx context.Context, bl blocks.Block) error { + // See comment in PutMany + err := b.blockstore.Put(ctx, bl) + if err == nil { + b.bloom.AddTS(bl.Cid().Hash()) + } + return err +} + +func (b *bloomcache) PutMany(ctx context.Context, bs []blocks.Block) error { + // bloom cache gives only conclusive resulty if key is not contained + // to reduce number of puts we need conclusive information if block is contained + // this means that PutMany can't be improved with bloom cache so we just + // just do a passthrough. + err := b.blockstore.PutMany(ctx, bs) + if err != nil { + return err + } + for _, bl := range bs { + b.bloom.AddTS(bl.Cid().Hash()) + } + return nil +} + +func (b *bloomcache) HashOnRead(enabled bool) { + b.blockstore.HashOnRead(enabled) +} + +func (b *bloomcache) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) { + return b.blockstore.AllKeysChan(ctx) +} + +func (b *bloomcache) GCLock(ctx context.Context) Unlocker { + return b.blockstore.(GCBlockstore).GCLock(ctx) +} + +func (b *bloomcache) PinLock(ctx context.Context) Unlocker { + return b.blockstore.(GCBlockstore).PinLock(ctx) +} + +func (b *bloomcache) GCRequested(ctx context.Context) bool { + return b.blockstore.(GCBlockstore).GCRequested(ctx) +} diff --git a/blockstore/bloom_cache_test.go b/blockstore/bloom_cache_test.go new file mode 100644 index 0000000000..ddde3f5370 --- /dev/null +++ b/blockstore/bloom_cache_test.go @@ -0,0 +1,215 @@ +package blockstore + +import ( + "context" + "fmt" + "sync" + "testing" + "time" + + ds "github.com/ipfs/go-datastore" + dsq "github.com/ipfs/go-datastore/query" + syncds "github.com/ipfs/go-datastore/sync" + ipld "github.com/ipfs/go-ipld-format" + blocks "github.com/ipfs/go-libipfs/blocks" +) + +var bg = context.Background() + +func testBloomCached(ctx context.Context, bs Blockstore) (*bloomcache, error) { + if ctx == nil { + ctx = context.Background() + } + opts := DefaultCacheOpts() + opts.HasARCCacheSize = 0 + bbs, err := CachedBlockstore(ctx, bs, opts) + if err == nil { + return bbs.(*bloomcache), nil + } + return nil, err +} + +func TestPutManyAddsToBloom(t *testing.T) { + bs := NewBlockstore(syncds.MutexWrap(ds.NewMapDatastore())) + + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) + defer cancel() + + cachedbs, err := testBloomCached(ctx, bs) + if err != nil { + t.Fatal(err) + } + + if err := cachedbs.Wait(ctx); err != nil { + t.Fatalf("Failed while waiting for the filter to build: %d", cachedbs.bloom.ElementsAdded()) + } + + block1 := blocks.NewBlock([]byte("foo")) + block2 := blocks.NewBlock([]byte("bar")) + emptyBlock := blocks.NewBlock([]byte{}) + + cachedbs.PutMany(bg, []blocks.Block{block1, emptyBlock}) + has, err := cachedbs.Has(bg, block1.Cid()) + if err != nil { + t.Fatal(err) + } + blockSize, err := cachedbs.GetSize(bg, block1.Cid()) + if err != nil { + t.Fatal(err) + } + if blockSize == -1 || !has { + t.Fatal("added block is reported missing") + } + + has, err = cachedbs.Has(bg, block2.Cid()) + if err != nil { + t.Fatal(err) + } + blockSize, err = cachedbs.GetSize(bg, block2.Cid()) + if err != nil && !ipld.IsNotFound(err) { + t.Fatal(err) + } + if blockSize > -1 || has { + t.Fatal("not added block is reported to be in blockstore") + } + + has, err = cachedbs.Has(bg, emptyBlock.Cid()) + if err != nil { + t.Fatal(err) + } + blockSize, err = cachedbs.GetSize(bg, emptyBlock.Cid()) + if err != nil { + t.Fatal(err) + } + if blockSize != 0 || !has { + t.Fatal("added block is reported missing") + } +} + +func TestReturnsErrorWhenSizeNegative(t *testing.T) { + bs := NewBlockstore(syncds.MutexWrap(ds.NewMapDatastore())) + _, err := bloomCached(context.Background(), bs, -1, 1) + if err == nil { + t.Fail() + } +} +func TestHasIsBloomCached(t *testing.T) { + cd := &callbackDatastore{f: func() {}, ds: ds.NewMapDatastore()} + bs := NewBlockstore(syncds.MutexWrap(cd)) + + for i := 0; i < 1000; i++ { + bs.Put(bg, blocks.NewBlock([]byte(fmt.Sprintf("data: %d", i)))) + } + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) + defer cancel() + + cachedbs, err := testBloomCached(ctx, bs) + if err != nil { + t.Fatal(err) + } + + if err := cachedbs.Wait(ctx); err != nil { + t.Fatalf("Failed while waiting for the filter to build: %d", cachedbs.bloom.ElementsAdded()) + } + + cacheFails := 0 + cd.SetFunc(func() { + cacheFails++ + }) + + for i := 0; i < 1000; i++ { + cachedbs.Has(bg, blocks.NewBlock([]byte(fmt.Sprintf("data: %d", i+2000))).Cid()) + } + + if float64(cacheFails)/float64(1000) > float64(0.05) { + t.Fatalf("Bloom filter has cache miss rate of more than 5%%") + } + + cacheFails = 0 + block := blocks.NewBlock([]byte("newBlock")) + + cachedbs.PutMany(bg, []blocks.Block{block}) + if cacheFails != 2 { + t.Fatalf("expected two datastore hits: %d", cacheFails) + } + cachedbs.Put(bg, block) + if cacheFails != 3 { + t.Fatalf("expected datastore hit: %d", cacheFails) + } + + if has, err := cachedbs.Has(bg, block.Cid()); !has || err != nil { + t.Fatal("has gave wrong response") + } + + bl, err := cachedbs.Get(bg, block.Cid()) + if bl.String() != block.String() { + t.Fatal("block data doesn't match") + } + + if err != nil { + t.Fatal("there should't be an error") + } +} + +var _ ds.Batching = (*callbackDatastore)(nil) + +type callbackDatastore struct { + sync.Mutex + f func() + ds ds.Datastore +} + +func (c *callbackDatastore) SetFunc(f func()) { + c.Lock() + defer c.Unlock() + c.f = f +} + +func (c *callbackDatastore) CallF() { + c.Lock() + defer c.Unlock() + c.f() +} + +func (c *callbackDatastore) Put(ctx context.Context, key ds.Key, value []byte) (err error) { + c.CallF() + return c.ds.Put(ctx, key, value) +} + +func (c *callbackDatastore) Get(ctx context.Context, key ds.Key) (value []byte, err error) { + c.CallF() + return c.ds.Get(ctx, key) +} + +func (c *callbackDatastore) Has(ctx context.Context, key ds.Key) (exists bool, err error) { + c.CallF() + return c.ds.Has(ctx, key) +} + +func (c *callbackDatastore) GetSize(ctx context.Context, key ds.Key) (size int, err error) { + c.CallF() + return c.ds.GetSize(ctx, key) +} + +func (c *callbackDatastore) Close() error { + return nil +} + +func (c *callbackDatastore) Delete(ctx context.Context, key ds.Key) (err error) { + c.CallF() + return c.ds.Delete(ctx, key) +} + +func (c *callbackDatastore) Query(ctx context.Context, q dsq.Query) (dsq.Results, error) { + c.CallF() + return c.ds.Query(ctx, q) +} + +func (c *callbackDatastore) Sync(ctx context.Context, key ds.Key) error { + c.CallF() + return c.ds.Sync(ctx, key) +} + +func (c *callbackDatastore) Batch(_ context.Context) (ds.Batch, error) { + return ds.NewBasicBatch(c), nil +} diff --git a/blockstore/caching.go b/blockstore/caching.go new file mode 100644 index 0000000000..798b84ce2b --- /dev/null +++ b/blockstore/caching.go @@ -0,0 +1,55 @@ +package blockstore + +import ( + "context" + "errors" + + metrics "github.com/ipfs/go-metrics-interface" +) + +// CacheOpts wraps options for CachedBlockStore(). +// Next to each option is it aproximate memory usage per unit +type CacheOpts struct { + HasBloomFilterSize int // 1 byte + HasBloomFilterHashes int // No size, 7 is usually best, consult bloom papers + HasARCCacheSize int // 32 bytes +} + +// DefaultCacheOpts returns a CacheOpts initialized with default values. +func DefaultCacheOpts() CacheOpts { + return CacheOpts{ + HasBloomFilterSize: 512 << 10, + HasBloomFilterHashes: 7, + HasARCCacheSize: 64 << 10, + } +} + +// CachedBlockstore returns a blockstore wrapped in an ARCCache and +// then in a bloom filter cache, if the options indicate it. +func CachedBlockstore( + ctx context.Context, + bs Blockstore, + opts CacheOpts) (cbs Blockstore, err error) { + cbs = bs + + if opts.HasBloomFilterSize < 0 || opts.HasBloomFilterHashes < 0 || + opts.HasARCCacheSize < 0 { + return nil, errors.New("all options for cache need to be greater than zero") + } + + if opts.HasBloomFilterSize != 0 && opts.HasBloomFilterHashes == 0 { + return nil, errors.New("bloom filter hash count can't be 0 when there is size set") + } + + ctx = metrics.CtxSubScope(ctx, "bs.cache") + + if opts.HasARCCacheSize > 0 { + cbs, err = newARCCachedBS(ctx, cbs, opts.HasARCCacheSize) + } + if opts.HasBloomFilterSize != 0 { + // *8 because of bytes to bits conversion + cbs, err = bloomCached(ctx, cbs, opts.HasBloomFilterSize*8, opts.HasBloomFilterHashes) + } + + return cbs, err +} diff --git a/blockstore/caching_test.go b/blockstore/caching_test.go new file mode 100644 index 0000000000..16066ad18c --- /dev/null +++ b/blockstore/caching_test.go @@ -0,0 +1,38 @@ +package blockstore + +import ( + "context" + "testing" +) + +func TestCachingOptsLessThanZero(t *testing.T) { + opts := DefaultCacheOpts() + opts.HasARCCacheSize = -1 + + if _, err := CachedBlockstore(context.TODO(), nil, opts); err == nil { + t.Error("wrong ARC setting was not detected") + } + + opts = DefaultCacheOpts() + opts.HasBloomFilterSize = -1 + + if _, err := CachedBlockstore(context.TODO(), nil, opts); err == nil { + t.Error("negative bloom size was not detected") + } + + opts = DefaultCacheOpts() + opts.HasBloomFilterHashes = -1 + + if _, err := CachedBlockstore(context.TODO(), nil, opts); err == nil { + t.Error("negative hashes setting was not detected") + } +} + +func TestBloomHashesAtZero(t *testing.T) { + opts := DefaultCacheOpts() + opts.HasBloomFilterHashes = 0 + + if _, err := CachedBlockstore(context.TODO(), nil, opts); err == nil { + t.Error("zero hashes setting with positive size was not detected") + } +} diff --git a/blockstore/idstore.go b/blockstore/idstore.go new file mode 100644 index 0000000000..c25a7cf0d1 --- /dev/null +++ b/blockstore/idstore.go @@ -0,0 +1,123 @@ +package blockstore + +import ( + "context" + "io" + + cid "github.com/ipfs/go-cid" + blocks "github.com/ipfs/go-libipfs/blocks" + mh "github.com/multiformats/go-multihash" +) + +// idstore wraps a BlockStore to add support for identity hashes +type idstore struct { + bs Blockstore + viewer Viewer +} + +var _ Blockstore = (*idstore)(nil) +var _ Viewer = (*idstore)(nil) +var _ io.Closer = (*idstore)(nil) + +func NewIdStore(bs Blockstore) Blockstore { + ids := &idstore{bs: bs} + if v, ok := bs.(Viewer); ok { + ids.viewer = v + } + return ids +} + +func extractContents(k cid.Cid) (bool, []byte) { + // Pre-check by calling Prefix(), this much faster than extracting the hash. + if k.Prefix().MhType != mh.IDENTITY { + return false, nil + } + + dmh, err := mh.Decode(k.Hash()) + if err != nil || dmh.Code != mh.IDENTITY { + return false, nil + } + return true, dmh.Digest +} + +func (b *idstore) DeleteBlock(ctx context.Context, k cid.Cid) error { + isId, _ := extractContents(k) + if isId { + return nil + } + return b.bs.DeleteBlock(ctx, k) +} + +func (b *idstore) Has(ctx context.Context, k cid.Cid) (bool, error) { + isId, _ := extractContents(k) + if isId { + return true, nil + } + return b.bs.Has(ctx, k) +} + +func (b *idstore) View(ctx context.Context, k cid.Cid, callback func([]byte) error) error { + if b.viewer == nil { + blk, err := b.Get(ctx, k) + if err != nil { + return err + } + return callback(blk.RawData()) + } + isId, bdata := extractContents(k) + if isId { + return callback(bdata) + } + return b.viewer.View(ctx, k, callback) +} + +func (b *idstore) GetSize(ctx context.Context, k cid.Cid) (int, error) { + isId, bdata := extractContents(k) + if isId { + return len(bdata), nil + } + return b.bs.GetSize(ctx, k) +} + +func (b *idstore) Get(ctx context.Context, k cid.Cid) (blocks.Block, error) { + isId, bdata := extractContents(k) + if isId { + return blocks.NewBlockWithCid(bdata, k) + } + return b.bs.Get(ctx, k) +} + +func (b *idstore) Put(ctx context.Context, bl blocks.Block) error { + isId, _ := extractContents(bl.Cid()) + if isId { + return nil + } + return b.bs.Put(ctx, bl) +} + +func (b *idstore) PutMany(ctx context.Context, bs []blocks.Block) error { + toPut := make([]blocks.Block, 0, len(bs)) + for _, bl := range bs { + isId, _ := extractContents(bl.Cid()) + if isId { + continue + } + toPut = append(toPut, bl) + } + return b.bs.PutMany(ctx, toPut) +} + +func (b *idstore) HashOnRead(enabled bool) { + b.bs.HashOnRead(enabled) +} + +func (b *idstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) { + return b.bs.AllKeysChan(ctx) +} + +func (b *idstore) Close() error { + if c, ok := b.bs.(io.Closer); ok { + return c.Close() + } + return nil +} diff --git a/blockstore/idstore_test.go b/blockstore/idstore_test.go new file mode 100644 index 0000000000..f67670cae7 --- /dev/null +++ b/blockstore/idstore_test.go @@ -0,0 +1,162 @@ +package blockstore + +import ( + "context" + "testing" + + cid "github.com/ipfs/go-cid" + ds "github.com/ipfs/go-datastore" + blk "github.com/ipfs/go-libipfs/blocks" + mh "github.com/multiformats/go-multihash" +) + +func createTestStores() (Blockstore, *callbackDatastore) { + cd := &callbackDatastore{f: func() {}, ds: ds.NewMapDatastore()} + ids := NewIdStore(NewBlockstore(cd)) + return ids, cd +} + +func TestIdStore(t *testing.T) { + idhash1, _ := cid.NewPrefixV1(cid.Raw, mh.IDENTITY).Sum([]byte("idhash1")) + idblock1, _ := blk.NewBlockWithCid([]byte("idhash1"), idhash1) + hash1, _ := cid.NewPrefixV1(cid.Raw, mh.SHA2_256).Sum([]byte("hash1")) + block1, _ := blk.NewBlockWithCid([]byte("hash1"), hash1) + emptyHash, _ := cid.NewPrefixV1(cid.Raw, mh.SHA2_256).Sum([]byte("emptyHash")) + emptyBlock, _ := blk.NewBlockWithCid([]byte{}, emptyHash) + + ids, cb := createTestStores() + + have, _ := ids.Has(bg, idhash1) + if !have { + t.Fatal("Has() failed on idhash") + } + + _, err := ids.Get(bg, idhash1) + if err != nil { + t.Fatalf("Get() failed on idhash: %v", err) + } + + noop := func() {} + failIfPassThough := func() { + t.Fatal("operation on identity hash passed though to datastore") + } + + cb.f = failIfPassThough + err = ids.Put(bg, idblock1) + if err != nil { + t.Fatal(err) + } + + cb.f = noop + err = ids.Put(bg, block1) + if err != nil { + t.Fatalf("Put() failed on normal block: %v", err) + } + + have, _ = ids.Has(bg, hash1) + if !have { + t.Fatal("normal block not added to datastore") + } + + blockSize, _ := ids.GetSize(bg, hash1) + if blockSize == -1 { + t.Fatal("normal block not added to datastore") + } + + _, err = ids.Get(bg, hash1) + if err != nil { + t.Fatal(err) + } + + err = ids.Put(bg, emptyBlock) + if err != nil { + t.Fatalf("Put() failed on normal block: %v", err) + } + + have, _ = ids.Has(bg, emptyHash) + if !have { + t.Fatal("normal block not added to datastore") + } + + blockSize, _ = ids.GetSize(bg, emptyHash) + if blockSize != 0 { + t.Fatal("normal block not added to datastore") + } + + cb.f = failIfPassThough + err = ids.DeleteBlock(bg, idhash1) + if err != nil { + t.Fatal(err) + } + + cb.f = noop + err = ids.DeleteBlock(bg, hash1) + if err != nil { + t.Fatal(err) + } + + have, _ = ids.Has(bg, hash1) + if have { + t.Fatal("normal block not deleted from datastore") + } + + blockSize, _ = ids.GetSize(bg, hash1) + if blockSize > -1 { + t.Fatal("normal block not deleted from datastore") + } + + err = ids.DeleteBlock(bg, emptyHash) + if err != nil { + t.Fatal(err) + } + + idhash2, _ := cid.NewPrefixV1(cid.Raw, mh.IDENTITY).Sum([]byte("idhash2")) + idblock2, _ := blk.NewBlockWithCid([]byte("idhash2"), idhash2) + hash2, _ := cid.NewPrefixV1(cid.Raw, mh.SHA2_256).Sum([]byte("hash2")) + block2, _ := blk.NewBlockWithCid([]byte("hash2"), hash2) + + cb.f = failIfPassThough + err = ids.PutMany(bg, []blk.Block{idblock1, idblock2}) + if err != nil { + t.Fatal(err) + } + + opCount := 0 + cb.f = func() { + opCount++ + } + + err = ids.PutMany(bg, []blk.Block{block1, block2}) + if err != nil { + t.Fatal(err) + } + if opCount != 4 { + // one call to Has and Put for each Cid + t.Fatalf("expected exactly 4 operations got %d", opCount) + } + + opCount = 0 + err = ids.PutMany(bg, []blk.Block{idblock1, block1}) + if err != nil { + t.Fatal(err) + } + if opCount != 1 { + // just one call to Put from the normal (non-id) block + t.Fatalf("expected exactly 1 operations got %d", opCount) + } + + ch, err := ids.AllKeysChan(context.TODO()) + if err != nil { + t.Fatal(err) + } + cnt := 0 + for c := range ch { + cnt++ + if c.Prefix().MhType == mh.IDENTITY { + t.Fatalf("block with identity hash found in blockstore") + } + } + if cnt != 2 { + t.Fatalf("expected exactly two keys returned by AllKeysChan got %d", cnt) + } +} diff --git a/chunker/benchmark_test.go b/chunker/benchmark_test.go new file mode 100644 index 0000000000..5069b06536 --- /dev/null +++ b/chunker/benchmark_test.go @@ -0,0 +1,59 @@ +package chunk + +import ( + "bytes" + "io" + "math/rand" + "testing" +) + +type newSplitter func(io.Reader) Splitter + +type bencSpec struct { + size int + name string +} + +var bSizes = []bencSpec{ + {1 << 10, "1K"}, + {1 << 20, "1M"}, + {16 << 20, "16M"}, + {100 << 20, "100M"}, +} + +func benchmarkChunker(b *testing.B, ns newSplitter) { + for _, s := range bSizes { + s := s + b.Run(s.name, func(b *testing.B) { + benchmarkChunkerSize(b, ns, s.size) + }) + } +} + +func benchmarkChunkerSize(b *testing.B, ns newSplitter, size int) { + rng := rand.New(rand.NewSource(1)) + data := make([]byte, size) + rng.Read(data) + + b.SetBytes(int64(size)) + b.ReportAllocs() + b.ResetTimer() + + var res uint64 + + for i := 0; i < b.N; i++ { + r := ns(bytes.NewReader(data)) + + for { + chunk, err := r.NextBytes() + if err != nil { + if err == io.EOF { + break + } + b.Fatal(err) + } + res = res + uint64(len(chunk)) + } + } + Res = Res + res +} diff --git a/chunker/buzhash.go b/chunker/buzhash.go new file mode 100644 index 0000000000..83ab019dd4 --- /dev/null +++ b/chunker/buzhash.go @@ -0,0 +1,151 @@ +package chunk + +import ( + "io" + "math/bits" + + pool "github.com/libp2p/go-buffer-pool" +) + +const ( + buzMin = 128 << 10 + buzMax = 512 << 10 + buzMask = 1<<17 - 1 +) + +type Buzhash struct { + r io.Reader + buf []byte + n int + + err error +} + +func NewBuzhash(r io.Reader) *Buzhash { + return &Buzhash{ + r: r, + buf: pool.Get(buzMax), + } +} + +func (b *Buzhash) Reader() io.Reader { + return b.r +} + +func (b *Buzhash) NextBytes() ([]byte, error) { + if b.err != nil { + return nil, b.err + } + + n, err := io.ReadFull(b.r, b.buf[b.n:]) + if err != nil { + if err == io.ErrUnexpectedEOF || err == io.EOF { + buffered := b.n + n + if buffered < buzMin { + b.err = io.EOF + // Read nothing? Don't return an empty block. + if buffered == 0 { + pool.Put(b.buf) + b.buf = nil + return nil, b.err + } + res := make([]byte, buffered) + copy(res, b.buf) + + pool.Put(b.buf) + b.buf = nil + return res, nil + } + } else { + b.err = err + pool.Put(b.buf) + b.buf = nil + return nil, err + } + } + + i := buzMin - 32 + + var state uint32 = 0 + + if buzMin > len(b.buf) { + panic("this is impossible") + } + + for ; i < buzMin; i++ { + state = bits.RotateLeft32(state, 1) + state = state ^ bytehash[b.buf[i]] + } + + { + max := b.n + n - 32 - 1 + + buf := b.buf + bufshf := b.buf[32:] + i = buzMin - 32 + _ = buf[max] + _ = bufshf[max] + + for ; i <= max; i++ { + if state&buzMask == 0 { + break + } + state = bits.RotateLeft32(state, 1) ^ + bytehash[buf[i]] ^ + bytehash[bufshf[i]] + } + i += 32 + } + + res := make([]byte, i) + copy(res, b.buf) + + b.n = copy(b.buf, b.buf[i:b.n+n]) + + return res, nil +} + +var bytehash = [256]uint32{ + 0x6236e7d5, 0x10279b0b, 0x72818182, 0xdc526514, 0x2fd41e3d, 0x777ef8c8, + 0x83ee5285, 0x2c8f3637, 0x2f049c1a, 0x57df9791, 0x9207151f, 0x9b544818, + 0x74eef658, 0x2028ca60, 0x0271d91a, 0x27ae587e, 0xecf9fa5f, 0x236e71cd, + 0xf43a8a2e, 0xbb13380, 0x9e57912c, 0x89a26cdb, 0x9fcf3d71, 0xa86da6f1, + 0x9c49f376, 0x346aecc7, 0xf094a9ee, 0xea99e9cb, 0xb01713c6, 0x88acffb, + 0x2960a0fb, 0x344a626c, 0x7ff22a46, 0x6d7a1aa5, 0x6a714916, 0x41d454ca, + 0x8325b830, 0xb65f563, 0x447fecca, 0xf9d0ea5e, 0xc1d9d3d4, 0xcb5ec574, + 0x55aae902, 0x86edc0e7, 0xd3a9e33, 0xe70dc1e1, 0xe3c5f639, 0x9b43140a, + 0xc6490ac5, 0x5e4030fb, 0x8e976dd5, 0xa87468ea, 0xf830ef6f, 0xcc1ed5a5, + 0x611f4e78, 0xddd11905, 0xf2613904, 0x566c67b9, 0x905a5ccc, 0x7b37b3a4, + 0x4b53898a, 0x6b8fd29d, 0xaad81575, 0x511be414, 0x3cfac1e7, 0x8029a179, + 0xd40efeda, 0x7380e02, 0xdc9beffd, 0x2d049082, 0x99bc7831, 0xff5002a8, + 0x21ce7646, 0x1cd049b, 0xf43994f, 0xc3c6c5a5, 0xbbda5f50, 0xec15ec7, + 0x9adb19b6, 0xc1e80b9, 0xb9b52968, 0xae162419, 0x2542b405, 0x91a42e9d, + 0x6be0f668, 0x6ed7a6b9, 0xbc2777b4, 0xe162ce56, 0x4266aad5, 0x60fdb704, + 0x66f832a5, 0x9595f6ca, 0xfee83ced, 0x55228d99, 0x12bf0e28, 0x66896459, + 0x789afda, 0x282baa8, 0x2367a343, 0x591491b0, 0x2ff1a4b1, 0x410739b6, + 0x9b7055a0, 0x2e0eb229, 0x24fc8252, 0x3327d3df, 0xb0782669, 0x1c62e069, + 0x7f503101, 0xf50593ae, 0xd9eb275d, 0xe00eb678, 0x5917ccde, 0x97b9660a, + 0xdd06202d, 0xed229e22, 0xa9c735bf, 0xd6316fe6, 0x6fc72e4c, 0x206dfa2, + 0xd6b15c5a, 0x69d87b49, 0x9c97745, 0x13445d61, 0x35a975aa, 0x859aa9b9, + 0x65380013, 0xd1fb6391, 0xc29255fd, 0x784a3b91, 0xb9e74c26, 0x63ce4d40, + 0xc07cbe9e, 0xe6e4529e, 0xfb3632f, 0x9438d9c9, 0x682f94a8, 0xf8fd4611, + 0x257ec1ed, 0x475ce3d6, 0x60ee2db1, 0x2afab002, 0x2b9e4878, 0x86b340de, + 0x1482fdca, 0xfe41b3bf, 0xd4a412b0, 0xe09db98c, 0xc1af5d53, 0x7e55e25f, + 0xd3346b38, 0xb7a12cbd, 0x9c6827ba, 0x71f78bee, 0x8c3a0f52, 0x150491b0, + 0xf26de912, 0x233e3a4e, 0xd309ebba, 0xa0a9e0ff, 0xca2b5921, 0xeeb9893c, + 0x33829e88, 0x9870cc2a, 0x23c4b9d0, 0xeba32ea3, 0xbdac4d22, 0x3bc8c44c, + 0x1e8d0397, 0xf9327735, 0x783b009f, 0xeb83742, 0x2621dc71, 0xed017d03, + 0x5c760aa1, 0x5a69814b, 0x96e3047f, 0xa93c9cde, 0x615c86f5, 0xb4322aa5, + 0x4225534d, 0xd2e2de3, 0xccfccc4b, 0xbac2a57, 0xf0a06d04, 0xbc78d737, + 0xf2d1f766, 0xf5a7953c, 0xbcdfda85, 0x5213b7d5, 0xbce8a328, 0xd38f5f18, + 0xdb094244, 0xfe571253, 0x317fa7ee, 0x4a324f43, 0x3ffc39d9, 0x51b3fa8e, + 0x7a4bee9f, 0x78bbc682, 0x9f5c0350, 0x2fe286c, 0x245ab686, 0xed6bf7d7, + 0xac4988a, 0x3fe010fa, 0xc65fe369, 0xa45749cb, 0x2b84e537, 0xde9ff363, + 0x20540f9a, 0xaa8c9b34, 0x5bc476b3, 0x1d574bd7, 0x929100ad, 0x4721de4d, + 0x27df1b05, 0x58b18546, 0xb7e76764, 0xdf904e58, 0x97af57a1, 0xbd4dc433, + 0xa6256dfd, 0xf63998f3, 0xf1e05833, 0xe20acf26, 0xf57fd9d6, 0x90300b4d, + 0x89df4290, 0x68d01cbc, 0xcf893ee3, 0xcc42a046, 0x778e181b, 0x67265c76, + 0xe981a4c4, 0x82991da1, 0x708f7294, 0xe6e2ae62, 0xfc441870, 0x95e1b0b6, + 0x445f825, 0x5a93b47f, 0x5e9cf4be, 0x84da71e7, 0x9d9582b0, 0x9bf835ef, + 0x591f61e2, 0x43325985, 0x5d2de32e, 0x8d8fbf0f, 0x95b30f38, 0x7ad5b6e, + 0x4e934edf, 0x3cd4990e, 0x9053e259, 0x5c41857d} diff --git a/chunker/buzhash_norace_test.go b/chunker/buzhash_norace_test.go new file mode 100644 index 0000000000..50dc0e5ce2 --- /dev/null +++ b/chunker/buzhash_norace_test.go @@ -0,0 +1,14 @@ +//go:build !race + +package chunk + +import ( + "testing" +) + +func TestFuzzBuzhashChunking(t *testing.T) { + buf := make([]byte, 1024*1024*16) + for i := 0; i < 100; i++ { + testBuzhashChunking(t, buf) + } +} diff --git a/chunker/buzhash_test.go b/chunker/buzhash_test.go new file mode 100644 index 0000000000..384ade8df4 --- /dev/null +++ b/chunker/buzhash_test.go @@ -0,0 +1,91 @@ +package chunk + +import ( + "bytes" + "io" + "testing" + + util "github.com/ipfs/go-libipfs/util" +) + +func testBuzhashChunking(t *testing.T, buf []byte) (chunkCount int) { + n, err := util.NewTimeSeededRand().Read(buf) + if n < len(buf) { + t.Fatalf("expected %d bytes, got %d", len(buf), n) + } + if err != nil { + t.Fatal(err) + } + + r := NewBuzhash(bytes.NewReader(buf)) + + var chunks [][]byte + + for { + chunk, err := r.NextBytes() + if err != nil { + if err == io.EOF { + break + } + t.Fatal(err) + } + + chunks = append(chunks, chunk) + } + chunkCount += len(chunks) + + for i, chunk := range chunks { + if len(chunk) == 0 { + t.Fatalf("chunk %d/%d is empty", i+1, len(chunks)) + } + } + + for i, chunk := range chunks[:len(chunks)-1] { + if len(chunk) < buzMin { + t.Fatalf("chunk %d/%d is less than the minimum size", i+1, len(chunks)) + } + } + + unchunked := bytes.Join(chunks, nil) + if !bytes.Equal(unchunked, buf) { + t.Fatal("data was chunked incorrectly") + } + + return chunkCount +} + +func TestBuzhashChunking(t *testing.T) { + buf := make([]byte, 1024*1024*16) + count := testBuzhashChunking(t, buf) + t.Logf("average block size: %d\n", len(buf)/count) +} + +func TestBuzhashChunkReuse(t *testing.T) { + newBuzhash := func(r io.Reader) Splitter { + return NewBuzhash(r) + } + testReuse(t, newBuzhash) +} + +func BenchmarkBuzhash2(b *testing.B) { + benchmarkChunker(b, func(r io.Reader) Splitter { + return NewBuzhash(r) + }) +} + +func TestBuzhashBitsHashBias(t *testing.T) { + counts := make([]byte, 32) + for _, h := range bytehash { + for i := 0; i < 32; i++ { + if h&1 == 1 { + counts[i]++ + } + h = h >> 1 + } + } + for i, c := range counts { + if c != 128 { + t.Errorf("Bit balance in position %d broken, %d ones", i, c) + } + } +} diff --git a/chunker/gen/main.go b/chunker/gen/main.go new file mode 100644 index 0000000000..9d908544b8 --- /dev/null +++ b/chunker/gen/main.go @@ -0,0 +1,33 @@ +// This file generates bytehash LUT +package main + +import ( + "fmt" + "math/rand" +) + +const nRounds = 200 + +func main() { + rnd := rand.New(rand.NewSource(0)) + + lut := make([]uint32, 256) + for i := 0; i < 256/2; i++ { + lut[i] = 1<<32 - 1 + } + + for r := 0; r < nRounds; r++ { + for b := uint32(0); b < 32; b++ { + mask := uint32(1) << b + nmask := ^mask + for i, j := range rnd.Perm(256) { + li := lut[i] + lj := lut[j] + lut[i] = li&nmask | (lj & mask) + lut[j] = lj&nmask | (li & mask) + } + } + } + + fmt.Printf("%#v", lut) +} diff --git a/chunker/parse.go b/chunker/parse.go new file mode 100644 index 0000000000..d62af509ae --- /dev/null +++ b/chunker/parse.go @@ -0,0 +1,114 @@ +package chunk + +import ( + "errors" + "fmt" + "io" + "strconv" + "strings" +) + +const ( + // DefaultBlockSize is the chunk size that splitters produce (or aim to). + DefaultBlockSize int64 = 1024 * 256 + + // No leaf block should contain more than 1MiB of payload data ( wrapping overhead aside ) + // This effectively mandates the maximum chunk size + // See discussion at https://github.com/ipfs/go-libipfs/chunker/pull/21#discussion_r369124879 for background + ChunkSizeLimit int = 1048576 +) + +var ( + ErrRabinMin = errors.New("rabin min must be greater than 16") + ErrSize = errors.New("chunker size must be greater than 0") + ErrSizeMax = fmt.Errorf("chunker parameters may not exceed the maximum chunk size of %d", ChunkSizeLimit) +) + +// FromString returns a Splitter depending on the given string: +// it supports "default" (""), "size-{size}", "rabin", "rabin-{blocksize}", +// "rabin-{min}-{avg}-{max}" and "buzhash". +func FromString(r io.Reader, chunker string) (Splitter, error) { + switch { + case chunker == "" || chunker == "default": + return DefaultSplitter(r), nil + + case strings.HasPrefix(chunker, "size-"): + sizeStr := strings.Split(chunker, "-")[1] + size, err := strconv.Atoi(sizeStr) + if err != nil { + return nil, err + } else if size <= 0 { + return nil, ErrSize + } else if size > ChunkSizeLimit { + return nil, ErrSizeMax + } + return NewSizeSplitter(r, int64(size)), nil + + case strings.HasPrefix(chunker, "rabin"): + return parseRabinString(r, chunker) + + case chunker == "buzhash": + return NewBuzhash(r), nil + + default: + return nil, fmt.Errorf("unrecognized chunker option: %s", chunker) + } +} + +func parseRabinString(r io.Reader, chunker string) (Splitter, error) { + parts := strings.Split(chunker, "-") + switch len(parts) { + case 1: + return NewRabin(r, uint64(DefaultBlockSize)), nil + case 2: + size, err := strconv.Atoi(parts[1]) + if err != nil { + return nil, err + } else if int(float32(size)*1.5) > ChunkSizeLimit { // FIXME - this will be addressed in a subsequent PR + return nil, ErrSizeMax + } + return NewRabin(r, uint64(size)), nil + case 4: + sub := strings.Split(parts[1], ":") + if len(sub) > 1 && sub[0] != "min" { + return nil, errors.New("first label must be min") + } + min, err := strconv.Atoi(sub[len(sub)-1]) + if err != nil { + return nil, err + } + if min < 16 { + return nil, ErrRabinMin + } + sub = strings.Split(parts[2], ":") + if len(sub) > 1 && sub[0] != "avg" { + log.Error("sub == ", sub) + return nil, errors.New("second label must be avg") + } + avg, err := strconv.Atoi(sub[len(sub)-1]) + if err != nil { + return nil, err + } + + sub = strings.Split(parts[3], ":") + if len(sub) > 1 && sub[0] != "max" { + return nil, errors.New("final label must be max") + } + max, err := strconv.Atoi(sub[len(sub)-1]) + if err != nil { + return nil, err + } + + if min >= avg { + return nil, errors.New("incorrect format: rabin-min must be smaller than rabin-avg") + } else if avg >= max { + return nil, errors.New("incorrect format: rabin-avg must be smaller than rabin-max") + } else if max > ChunkSizeLimit { + return nil, ErrSizeMax + } + + return NewRabinMinMax(r, uint64(min), uint64(avg), uint64(max)), nil + default: + return nil, errors.New("incorrect format (expected 'rabin' 'rabin-[avg]' or 'rabin-[min]-[avg]-[max]'") + } +} diff --git a/chunker/parse_test.go b/chunker/parse_test.go new file mode 100644 index 0000000000..237a2b439a --- /dev/null +++ b/chunker/parse_test.go @@ -0,0 +1,80 @@ +package chunk + +import ( + "bytes" + "fmt" + "testing" +) + +const ( + testTwoThirdsOfChunkLimit = 2 * (float32(ChunkSizeLimit) / float32(3)) +) + +func TestParseRabin(t *testing.T) { + r := bytes.NewReader(randBuf(t, 1000)) + + _, err := FromString(r, "rabin-18-25-32") + if err != nil { + t.Errorf(err.Error()) + } + + _, err = FromString(r, "rabin-15-23-31") + if err != ErrRabinMin { + t.Fatalf("Expected an 'ErrRabinMin' error, got: %#v", err) + } + + _, err = FromString(r, "rabin-20-20-21") + if err == nil || err.Error() != "incorrect format: rabin-min must be smaller than rabin-avg" { + t.Fatalf("Expected an arg-out-of-order error, got: %#v", err) + } + + _, err = FromString(r, "rabin-19-21-21") + if err == nil || err.Error() != "incorrect format: rabin-avg must be smaller than rabin-max" { + t.Fatalf("Expected an arg-out-of-order error, got: %#v", err) + } + + _, err = FromString(r, fmt.Sprintf("rabin-19-21-%d", ChunkSizeLimit)) + if err != nil { + t.Fatalf("Expected success, got: %#v", err) + } + + _, err = FromString(r, fmt.Sprintf("rabin-19-21-%d", 1+ChunkSizeLimit)) + if err != ErrSizeMax { + t.Fatalf("Expected 'ErrSizeMax', got: %#v", err) + } + + _, err = FromString(r, fmt.Sprintf("rabin-%.0f", testTwoThirdsOfChunkLimit)) + if err != nil { + t.Fatalf("Expected success, got: %#v", err) + } + + _, err = FromString(r, fmt.Sprintf("rabin-%.0f", 1+testTwoThirdsOfChunkLimit)) + if err != ErrSizeMax { + t.Fatalf("Expected 'ErrSizeMax', got: %#v", err) + } + +} + +func TestParseSize(t *testing.T) { + r := bytes.NewReader(randBuf(t, 1000)) + + _, err := FromString(r, "size-0") + if err != ErrSize { + t.Fatalf("Expected an 'ErrSize' error, got: %#v", err) + } + + _, err = FromString(r, "size-32") + if err != nil { + t.Fatalf("Expected success, got: %#v", err) + } + + _, err = FromString(r, fmt.Sprintf("size-%d", ChunkSizeLimit)) + if err != nil { + t.Fatalf("Expected success, got: %#v", err) + } + + _, err = FromString(r, fmt.Sprintf("size-%d", 1+ChunkSizeLimit)) + if err != ErrSizeMax { + t.Fatalf("Expected 'ErrSizeMax', got: %#v", err) + } +} diff --git a/chunker/rabin.go b/chunker/rabin.go new file mode 100644 index 0000000000..4247057b2f --- /dev/null +++ b/chunker/rabin.go @@ -0,0 +1,54 @@ +package chunk + +import ( + "hash/fnv" + "io" + + "github.com/whyrusleeping/chunker" +) + +// IpfsRabinPoly is the irreducible polynomial of degree 53 used by for Rabin. +var IpfsRabinPoly = chunker.Pol(17437180132763653) + +// Rabin implements the Splitter interface and splits content with Rabin +// fingerprints. +type Rabin struct { + r *chunker.Chunker + reader io.Reader +} + +// NewRabin creates a new Rabin splitter with the given +// average block size. +func NewRabin(r io.Reader, avgBlkSize uint64) *Rabin { + min := avgBlkSize / 3 + max := avgBlkSize + (avgBlkSize / 2) + + return NewRabinMinMax(r, min, avgBlkSize, max) +} + +// NewRabinMinMax returns a new Rabin splitter which uses +// the given min, average and max block sizes. +func NewRabinMinMax(r io.Reader, min, avg, max uint64) *Rabin { + h := fnv.New32a() + ch := chunker.New(r, IpfsRabinPoly, h, avg, min, max) + + return &Rabin{ + r: ch, + reader: r, + } +} + +// NextBytes reads the next bytes from the reader and returns a slice. +func (r *Rabin) NextBytes() ([]byte, error) { + ch, err := r.r.Next() + if err != nil { + return nil, err + } + + return ch.Data, nil +} + +// Reader returns the io.Reader associated to this Splitter. +func (r *Rabin) Reader() io.Reader { + return r.reader +} diff --git a/chunker/rabin_test.go b/chunker/rabin_test.go new file mode 100644 index 0000000000..5a9eb34d7c --- /dev/null +++ b/chunker/rabin_test.go @@ -0,0 +1,108 @@ +package chunk + +import ( + "bytes" + "fmt" + "io" + "testing" + + blocks "github.com/ipfs/go-libipfs/blocks" + util "github.com/ipfs/go-libipfs/util" +) + +func TestRabinChunking(t *testing.T) { + data := make([]byte, 1024*1024*16) + n, err := util.NewTimeSeededRand().Read(data) + if n < len(data) { + t.Fatalf("expected %d bytes, got %d", len(data), n) + } + if err != nil { + t.Fatal(err) + } + + r := NewRabin(bytes.NewReader(data), 1024*256) + + var chunks [][]byte + + for { + chunk, err := r.NextBytes() + if err != nil { + if err == io.EOF { + break + } + t.Fatal(err) + } + + chunks = append(chunks, chunk) + } + + fmt.Printf("average block size: %d\n", len(data)/len(chunks)) + + unchunked := bytes.Join(chunks, nil) + if !bytes.Equal(unchunked, data) { + fmt.Printf("%d %d\n", len(unchunked), len(data)) + t.Fatal("data was chunked incorrectly") + } +} + +func chunkData(t *testing.T, newC newSplitter, data []byte) map[string]blocks.Block { + r := newC(bytes.NewReader(data)) + + blkmap := make(map[string]blocks.Block) + + for { + blk, err := r.NextBytes() + if err != nil { + if err == io.EOF { + break + } + t.Fatal(err) + } + + b := blocks.NewBlock(blk) + blkmap[b.Cid().KeyString()] = b + } + + return blkmap +} + +func testReuse(t *testing.T, cr newSplitter) { + data := make([]byte, 1024*1024*16) + n, err := util.NewTimeSeededRand().Read(data) + if n < len(data) { + t.Fatalf("expected %d bytes, got %d", len(data), n) + } + if err != nil { + t.Fatal(err) + } + + ch1 := chunkData(t, cr, data[1000:]) + ch2 := chunkData(t, cr, data) + + var extra int + for k := range ch2 { + _, ok := ch1[k] + if !ok { + extra++ + } + } + + if extra > 2 { + t.Logf("too many spare chunks made: %d", extra) + } +} + +func TestRabinChunkReuse(t *testing.T) { + newRabin := func(r io.Reader) Splitter { + return NewRabin(r, 256*1024) + } + testReuse(t, newRabin) +} + +var Res uint64 + +func BenchmarkRabin(b *testing.B) { + benchmarkChunker(b, func(r io.Reader) Splitter { + return NewRabin(r, 256<<10) + }) +} diff --git a/chunker/splitting.go b/chunker/splitting.go new file mode 100644 index 0000000000..a137820ab1 --- /dev/null +++ b/chunker/splitting.go @@ -0,0 +1,102 @@ +// Package chunk implements streaming block splitters. +// Splitters read data from a reader and provide byte slices (chunks) +// The size and contents of these slices depend on the splitting method +// used. +package chunk + +import ( + "io" + + logging "github.com/ipfs/go-log" + pool "github.com/libp2p/go-buffer-pool" +) + +var log = logging.Logger("chunk") + +// A Splitter reads bytes from a Reader and creates "chunks" (byte slices) +// that can be used to build DAG nodes. +type Splitter interface { + Reader() io.Reader + NextBytes() ([]byte, error) +} + +// SplitterGen is a splitter generator, given a reader. +type SplitterGen func(r io.Reader) Splitter + +// DefaultSplitter returns a SizeSplitter with the DefaultBlockSize. +func DefaultSplitter(r io.Reader) Splitter { + return NewSizeSplitter(r, DefaultBlockSize) +} + +// SizeSplitterGen returns a SplitterGen function which will create +// a splitter with the given size when called. +func SizeSplitterGen(size int64) SplitterGen { + return func(r io.Reader) Splitter { + return NewSizeSplitter(r, size) + } +} + +// Chan returns a channel that receives each of the chunks produced +// by a splitter, along with another one for errors. +func Chan(s Splitter) (<-chan []byte, <-chan error) { + out := make(chan []byte) + errs := make(chan error, 1) + go func() { + defer close(out) + defer close(errs) + + // all-chunks loop (keep creating chunks) + for { + b, err := s.NextBytes() + if err != nil { + errs <- err + return + } + + out <- b + } + }() + return out, errs +} + +type sizeSplitterv2 struct { + r io.Reader + size uint32 + err error +} + +// NewSizeSplitter returns a new size-based Splitter with the given block size. +func NewSizeSplitter(r io.Reader, size int64) Splitter { + return &sizeSplitterv2{ + r: r, + size: uint32(size), + } +} + +// NextBytes produces a new chunk. +func (ss *sizeSplitterv2) NextBytes() ([]byte, error) { + if ss.err != nil { + return nil, ss.err + } + + full := pool.Get(int(ss.size)) + n, err := io.ReadFull(ss.r, full) + switch err { + case io.ErrUnexpectedEOF: + ss.err = io.EOF + small := make([]byte, n) + copy(small, full) + pool.Put(full) + return small, nil + case nil: + return full, nil + default: + pool.Put(full) + return nil, err + } +} + +// Reader returns the io.Reader associated to this Splitter. +func (ss *sizeSplitterv2) Reader() io.Reader { + return ss.r +} diff --git a/chunker/splitting_test.go b/chunker/splitting_test.go new file mode 100644 index 0000000000..9f5e5cc3ab --- /dev/null +++ b/chunker/splitting_test.go @@ -0,0 +1,126 @@ +package chunk + +import ( + "bytes" + "io" + "testing" + + u "github.com/ipfs/go-libipfs/util" +) + +func randBuf(t *testing.T, size int) []byte { + buf := make([]byte, size) + if _, err := u.NewTimeSeededRand().Read(buf); err != nil { + t.Fatal("failed to read enough randomness") + } + return buf +} + +func copyBuf(buf []byte) []byte { + cpy := make([]byte, len(buf)) + copy(cpy, buf) + return cpy +} + +func TestSizeSplitterOverAllocate(t *testing.T) { + max := 1000 + r := bytes.NewReader(randBuf(t, max)) + chunksize := int64(1024 * 256) + splitter := NewSizeSplitter(r, chunksize) + chunk, err := splitter.NextBytes() + if err != nil { + t.Fatal(err) + } + if cap(chunk) > len(chunk) { + t.Fatal("chunk capacity too large") + } +} + +func TestSizeSplitterIsDeterministic(t *testing.T) { + if testing.Short() { + t.SkipNow() + } + + test := func() { + bufR := randBuf(t, 10000000) // crank this up to satisfy yourself. + bufA := copyBuf(bufR) + bufB := copyBuf(bufR) + + chunksA, _ := Chan(DefaultSplitter(bytes.NewReader(bufA))) + chunksB, _ := Chan(DefaultSplitter(bytes.NewReader(bufB))) + + for n := 0; ; n++ { + a, moreA := <-chunksA + b, moreB := <-chunksB + + if !moreA { + if moreB { + t.Fatal("A ended, B didnt.") + } + return + } + + if !bytes.Equal(a, b) { + t.Fatalf("chunk %d not equal", n) + } + } + } + + for run := 0; run < 1; run++ { // crank this up to satisfy yourself. + test() + } +} + +func TestSizeSplitterFillsChunks(t *testing.T) { + if testing.Short() { + t.SkipNow() + } + + max := 10000000 + b := randBuf(t, max) + r := &clipReader{r: bytes.NewReader(b), size: 4000} + chunksize := int64(1024 * 256) + c, _ := Chan(NewSizeSplitter(r, chunksize)) + + sofar := 0 + whole := make([]byte, max) + for chunk := range c { + + bc := b[sofar : sofar+len(chunk)] + if !bytes.Equal(bc, chunk) { + t.Fatalf("chunk not correct: (sofar: %d) %d != %d, %v != %v", sofar, len(bc), len(chunk), bc[:100], chunk[:100]) + } + + copy(whole[sofar:], chunk) + + sofar += len(chunk) + if sofar != max && len(chunk) < int(chunksize) { + t.Fatal("sizesplitter split at a smaller size") + } + } + + if !bytes.Equal(b, whole) { + t.Fatal("splitter did not split right") + } +} + +type clipReader struct { + size int + r io.Reader +} + +func (s *clipReader) Read(buf []byte) (int, error) { + + // clip the incoming buffer to produce smaller chunks + if len(buf) > s.size { + buf = buf[:s.size] + } + + return s.r.Read(buf) +} + +func BenchmarkDefault(b *testing.B) { + benchmarkChunker(b, func(r io.Reader) Splitter { + return DefaultSplitter(r) + }) +} diff --git a/coreiface/block.go b/coreiface/block.go new file mode 100644 index 0000000000..78191eaa8c --- /dev/null +++ b/coreiface/block.go @@ -0,0 +1,38 @@ +package iface + +import ( + "context" + "io" + + path "github.com/ipfs/go-libipfs/coreiface/path" + + "github.com/ipfs/go-libipfs/coreiface/options" +) + +// BlockStat contains information about a block +type BlockStat interface { + // Size is the size of a block + Size() int + + // Path returns path to the block + Path() path.Resolved +} + +// BlockAPI specifies the interface to the block layer +type BlockAPI interface { + // Put imports raw block data, hashing it using specified settings. + Put(context.Context, io.Reader, ...options.BlockPutOption) (BlockStat, error) + + // Get attempts to resolve the path and return a reader for data in the block + Get(context.Context, path.Path) (io.Reader, error) + + // Rm removes the block specified by the path from local blockstore. + // By default an error will be returned if the block can't be found locally. + // + // NOTE: If the specified block is pinned it won't be removed and no error + // will be returned + Rm(context.Context, path.Path, ...options.BlockRmOption) error + + // Stat returns information on + Stat(context.Context, path.Path) (BlockStat, error) +} diff --git a/coreiface/coreapi.go b/coreiface/coreapi.go new file mode 100644 index 0000000000..5850122764 --- /dev/null +++ b/coreiface/coreapi.go @@ -0,0 +1,60 @@ +// Package iface defines IPFS Core API which is a set of interfaces used to +// interact with IPFS nodes. +package iface + +import ( + "context" + + path "github.com/ipfs/go-libipfs/coreiface/path" + + "github.com/ipfs/go-libipfs/coreiface/options" + + ipld "github.com/ipfs/go-ipld-format" +) + +// CoreAPI defines an unified interface to IPFS for Go programs +type CoreAPI interface { + // Unixfs returns an implementation of Unixfs API + Unixfs() UnixfsAPI + + // Block returns an implementation of Block API + Block() BlockAPI + + // Dag returns an implementation of Dag API + Dag() APIDagService + + // Name returns an implementation of Name API + Name() NameAPI + + // Key returns an implementation of Key API + Key() KeyAPI + + // Pin returns an implementation of Pin API + Pin() PinAPI + + // Object returns an implementation of Object API + Object() ObjectAPI + + // Dht returns an implementation of Dht API + Dht() DhtAPI + + // Swarm returns an implementation of Swarm API + Swarm() SwarmAPI + + // PubSub returns an implementation of PubSub API + PubSub() PubSubAPI + + // Routing returns an implementation of Routing API + Routing() RoutingAPI + + // ResolvePath resolves the path using Unixfs resolver + ResolvePath(context.Context, path.Path) (path.Resolved, error) + + // ResolveNode resolves the path (if not resolved already) using Unixfs + // resolver, gets and returns the resolved Node + ResolveNode(context.Context, path.Path) (ipld.Node, error) + + // WithOptions creates new instance of CoreAPI based on this instance with + // a set of options applied + WithOptions(...options.ApiOption) (CoreAPI, error) +} diff --git a/coreiface/dag.go b/coreiface/dag.go new file mode 100644 index 0000000000..3cc3aeb4de --- /dev/null +++ b/coreiface/dag.go @@ -0,0 +1,13 @@ +package iface + +import ( + ipld "github.com/ipfs/go-ipld-format" +) + +// APIDagService extends ipld.DAGService +type APIDagService interface { + ipld.DAGService + + // Pinning returns special NodeAdder which recursively pins added nodes + Pinning() ipld.NodeAdder +} diff --git a/coreiface/dht.go b/coreiface/dht.go new file mode 100644 index 0000000000..8f42eefc73 --- /dev/null +++ b/coreiface/dht.go @@ -0,0 +1,27 @@ +package iface + +import ( + "context" + + "github.com/ipfs/go-libipfs/coreiface/path" + + "github.com/ipfs/go-libipfs/coreiface/options" + + "github.com/libp2p/go-libp2p/core/peer" +) + +// DhtAPI specifies the interface to the DHT +// Note: This API will likely get deprecated in near future, see +// https://github.com/ipfs/interface-ipfs-core/issues/249 for more context. +type DhtAPI interface { + // FindPeer queries the DHT for all of the multiaddresses associated with a + // Peer ID + FindPeer(context.Context, peer.ID) (peer.AddrInfo, error) + + // FindProviders finds peers in the DHT who can provide a specific value + // given a key. + FindProviders(context.Context, path.Path, ...options.DhtFindProvidersOption) (<-chan peer.AddrInfo, error) + + // Provide announces to the network that you are providing given values + Provide(context.Context, path.Path, ...options.DhtProvideOption) error +} diff --git a/coreiface/errors.go b/coreiface/errors.go new file mode 100644 index 0000000000..e0bd7805d8 --- /dev/null +++ b/coreiface/errors.go @@ -0,0 +1,10 @@ +package iface + +import "errors" + +var ( + ErrIsDir = errors.New("this dag node is a directory") + ErrNotFile = errors.New("this dag node is not a regular file") + ErrOffline = errors.New("this action must be run in online mode, try running 'ipfs daemon' first") + ErrNotSupported = errors.New("operation not supported") +) diff --git a/coreiface/idfmt.go b/coreiface/idfmt.go new file mode 100644 index 0000000000..80fd0f822f --- /dev/null +++ b/coreiface/idfmt.go @@ -0,0 +1,19 @@ +package iface + +import ( + "github.com/libp2p/go-libp2p/core/peer" + mbase "github.com/multiformats/go-multibase" +) + +func FormatKeyID(id peer.ID) string { + if s, err := peer.ToCid(id).StringOfBase(mbase.Base36); err != nil { + panic(err) + } else { + return s + } +} + +// FormatKey formats the given IPNS key in a canonical way. +func FormatKey(key Key) string { + return FormatKeyID(key.ID()) +} diff --git a/coreiface/key.go b/coreiface/key.go new file mode 100644 index 0000000000..158e5f13ae --- /dev/null +++ b/coreiface/key.go @@ -0,0 +1,43 @@ +package iface + +import ( + "context" + + "github.com/ipfs/go-libipfs/coreiface/path" + + "github.com/ipfs/go-libipfs/coreiface/options" + + "github.com/libp2p/go-libp2p/core/peer" +) + +// Key specifies the interface to Keys in KeyAPI Keystore +type Key interface { + // Key returns key name + Name() string + + // Path returns key path + Path() path.Path + + // ID returns key PeerID + ID() peer.ID +} + +// KeyAPI specifies the interface to Keystore +type KeyAPI interface { + // Generate generates new key, stores it in the keystore under the specified + // name and returns a base58 encoded multihash of it's public key + Generate(ctx context.Context, name string, opts ...options.KeyGenerateOption) (Key, error) + + // Rename renames oldName key to newName. Returns the key and whether another + // key was overwritten, or an error + Rename(ctx context.Context, oldName string, newName string, opts ...options.KeyRenameOption) (Key, bool, error) + + // List lists keys stored in keystore + List(ctx context.Context) ([]Key, error) + + // Self returns the 'main' node key + Self(ctx context.Context) (Key, error) + + // Remove removes keys from keystore. Returns ipns path of the removed key + Remove(ctx context.Context, name string) (Key, error) +} diff --git a/coreiface/name.go b/coreiface/name.go new file mode 100644 index 0000000000..d08f9c460e --- /dev/null +++ b/coreiface/name.go @@ -0,0 +1,48 @@ +package iface + +import ( + "context" + "errors" + + path "github.com/ipfs/go-libipfs/coreiface/path" + + "github.com/ipfs/go-libipfs/coreiface/options" +) + +var ErrResolveFailed = errors.New("could not resolve name") + +// IpnsEntry specifies the interface to IpnsEntries +type IpnsEntry interface { + // Name returns IpnsEntry name + Name() string + // Value returns IpnsEntry value + Value() path.Path +} + +type IpnsResult struct { + path.Path + Err error +} + +// NameAPI specifies the interface to IPNS. +// +// IPNS is a PKI namespace, where names are the hashes of public keys, and the +// private key enables publishing new (signed) values. In both publish and +// resolve, the default name used is the node's own PeerID, which is the hash of +// its public key. +// +// You can use .Key API to list and generate more names and their respective keys. +type NameAPI interface { + // Publish announces new IPNS name + Publish(ctx context.Context, path path.Path, opts ...options.NamePublishOption) (IpnsEntry, error) + + // Resolve attempts to resolve the newest version of the specified name + Resolve(ctx context.Context, name string, opts ...options.NameResolveOption) (path.Path, error) + + // Search is a version of Resolve which outputs paths as they are discovered, + // reducing the time to first entry + // + // Note: by default, all paths read from the channel are considered unsafe, + // except the latest (last path in channel read buffer). + Search(ctx context.Context, name string, opts ...options.NameResolveOption) (<-chan IpnsResult, error) +} diff --git a/coreiface/object.go b/coreiface/object.go new file mode 100644 index 0000000000..59f0285409 --- /dev/null +++ b/coreiface/object.go @@ -0,0 +1,108 @@ +package iface + +import ( + "context" + "io" + + path "github.com/ipfs/go-libipfs/coreiface/path" + + "github.com/ipfs/go-libipfs/coreiface/options" + + "github.com/ipfs/go-cid" + ipld "github.com/ipfs/go-ipld-format" +) + +// ObjectStat provides information about dag nodes +type ObjectStat struct { + // Cid is the CID of the node + Cid cid.Cid + + // NumLinks is number of links the node contains + NumLinks int + + // BlockSize is size of the raw serialized node + BlockSize int + + // LinksSize is size of the links block section + LinksSize int + + // DataSize is the size of data block section + DataSize int + + // CumulativeSize is size of the tree (BlockSize + link sizes) + CumulativeSize int +} + +// ChangeType denotes type of change in ObjectChange +type ChangeType int + +const ( + // DiffAdd is set when a link was added to the graph + DiffAdd ChangeType = iota + + // DiffRemove is set when a link was removed from the graph + DiffRemove + + // DiffMod is set when a link was changed in the graph + DiffMod +) + +// ObjectChange represents a change ia a graph +type ObjectChange struct { + // Type of the change, either: + // * DiffAdd - Added a link + // * DiffRemove - Removed a link + // * DiffMod - Modified a link + Type ChangeType + + // Path to the changed link + Path string + + // Before holds the link path before the change. Note that when a link is + // added, this will be nil. + Before path.Resolved + + // After holds the link path after the change. Note that when a link is + // removed, this will be nil. + After path.Resolved +} + +// ObjectAPI specifies the interface to MerkleDAG and contains useful utilities +// for manipulating MerkleDAG data structures. +type ObjectAPI interface { + // New creates new, empty (by default) dag-node. + New(context.Context, ...options.ObjectNewOption) (ipld.Node, error) + + // Put imports the data into merkledag + Put(context.Context, io.Reader, ...options.ObjectPutOption) (path.Resolved, error) + + // Get returns the node for the path + Get(context.Context, path.Path) (ipld.Node, error) + + // Data returns reader for data of the node + Data(context.Context, path.Path) (io.Reader, error) + + // Links returns lint or links the node contains + Links(context.Context, path.Path) ([]*ipld.Link, error) + + // Stat returns information about the node + Stat(context.Context, path.Path) (*ObjectStat, error) + + // AddLink adds a link under the specified path. child path can point to a + // subdirectory within the patent which must be present (can be overridden + // with WithCreate option). + AddLink(ctx context.Context, base path.Path, name string, child path.Path, opts ...options.ObjectAddLinkOption) (path.Resolved, error) + + // RmLink removes a link from the node + RmLink(ctx context.Context, base path.Path, link string) (path.Resolved, error) + + // AppendData appends data to the node + AppendData(context.Context, path.Path, io.Reader) (path.Resolved, error) + + // SetData sets the data contained in the node + SetData(context.Context, path.Path, io.Reader) (path.Resolved, error) + + // Diff returns a set of changes needed to transform the first object into the + // second. + Diff(context.Context, path.Path, path.Path) ([]ObjectChange, error) +} diff --git a/coreiface/options/block.go b/coreiface/options/block.go new file mode 100644 index 0000000000..130648682f --- /dev/null +++ b/coreiface/options/block.go @@ -0,0 +1,164 @@ +package options + +import ( + "fmt" + + cid "github.com/ipfs/go-cid" + mc "github.com/multiformats/go-multicodec" + mh "github.com/multiformats/go-multihash" +) + +type BlockPutSettings struct { + CidPrefix cid.Prefix + Pin bool +} + +type BlockRmSettings struct { + Force bool +} + +type BlockPutOption func(*BlockPutSettings) error +type BlockRmOption func(*BlockRmSettings) error + +func BlockPutOptions(opts ...BlockPutOption) (*BlockPutSettings, error) { + var cidPrefix cid.Prefix + + // Baseline is CIDv1 raw sha2-255-32 (can be tweaked later via opts) + cidPrefix.Version = 1 + cidPrefix.Codec = uint64(mc.Raw) + cidPrefix.MhType = mh.SHA2_256 + cidPrefix.MhLength = -1 // -1 means len is to be calculated during mh.Sum() + + options := &BlockPutSettings{ + CidPrefix: cidPrefix, + Pin: false, + } + + // Apply any overrides + for _, opt := range opts { + err := opt(options) + if err != nil { + return nil, err + } + } + + return options, nil +} + +func BlockRmOptions(opts ...BlockRmOption) (*BlockRmSettings, error) { + options := &BlockRmSettings{ + Force: false, + } + + for _, opt := range opts { + err := opt(options) + if err != nil { + return nil, err + } + } + return options, nil +} + +type blockOpts struct{} + +var Block blockOpts + +// CidCodec is the modern option for Block.Put which specifies the multicodec to use +// in the CID returned by the Block.Put operation. +// It uses correct codes from go-multicodec and replaces the old Format now with CIDv1 as the default. +func (blockOpts) CidCodec(codecName string) BlockPutOption { + return func(settings *BlockPutSettings) error { + if codecName == "" { + return nil + } + code, err := codeFromName(codecName) + if err != nil { + return err + } + settings.CidPrefix.Codec = uint64(code) + return nil + } +} + +// Map string to code from go-multicodec +func codeFromName(codecName string) (mc.Code, error) { + var cidCodec mc.Code + err := cidCodec.Set(codecName) + return cidCodec, err +} + +// Format is a legacy option for Block.Put which specifies the multicodec to +// use to serialize the object. +// Provided for backward-compatibility only. Use CidCodec instead. +func (blockOpts) Format(format string) BlockPutOption { + return func(settings *BlockPutSettings) error { + if format == "" { + return nil + } + // Opt-in CIDv0 support for backward-compatibility + if format == "v0" { + settings.CidPrefix.Version = 0 + } + + // Fixup a legacy (invalid) names for dag-pb (0x70) + if format == "v0" || format == "protobuf" { + format = "dag-pb" + } + + // Fixup invalid name for dag-cbor (0x71) + if format == "cbor" { + format = "dag-cbor" + } + + // Set code based on name passed as "format" + code, err := codeFromName(format) + if err != nil { + return err + } + settings.CidPrefix.Codec = uint64(code) + + // If CIDv0, ensure all parameters are compatible + // (in theory go-cid would validate this anyway, but we want to provide better errors) + pref := settings.CidPrefix + if pref.Version == 0 { + if pref.Codec != uint64(mc.DagPb) { + return fmt.Errorf("only dag-pb is allowed with CIDv0") + } + if pref.MhType != mh.SHA2_256 || (pref.MhLength != -1 && pref.MhLength != 32) { + return fmt.Errorf("only sha2-255-32 is allowed with CIDv0") + } + } + + return nil + } + +} + +// Hash is an option for Block.Put which specifies the multihash settings to use +// when hashing the object. Default is mh.SHA2_256 (0x12). +// If mhLen is set to -1, default length for the hash will be used +func (blockOpts) Hash(mhType uint64, mhLen int) BlockPutOption { + return func(settings *BlockPutSettings) error { + settings.CidPrefix.MhType = mhType + settings.CidPrefix.MhLength = mhLen + return nil + } +} + +// Pin is an option for Block.Put which specifies whether to (recursively) pin +// added blocks +func (blockOpts) Pin(pin bool) BlockPutOption { + return func(settings *BlockPutSettings) error { + settings.Pin = pin + return nil + } +} + +// Force is an option for Block.Rm which, when set to true, will ignore +// non-existing blocks +func (blockOpts) Force(force bool) BlockRmOption { + return func(settings *BlockRmSettings) error { + settings.Force = force + return nil + } +} diff --git a/coreiface/options/dht.go b/coreiface/options/dht.go new file mode 100644 index 0000000000..e13e160200 --- /dev/null +++ b/coreiface/options/dht.go @@ -0,0 +1,62 @@ +package options + +type DhtProvideSettings struct { + Recursive bool +} + +type DhtFindProvidersSettings struct { + NumProviders int +} + +type DhtProvideOption func(*DhtProvideSettings) error +type DhtFindProvidersOption func(*DhtFindProvidersSettings) error + +func DhtProvideOptions(opts ...DhtProvideOption) (*DhtProvideSettings, error) { + options := &DhtProvideSettings{ + Recursive: false, + } + + for _, opt := range opts { + err := opt(options) + if err != nil { + return nil, err + } + } + return options, nil +} + +func DhtFindProvidersOptions(opts ...DhtFindProvidersOption) (*DhtFindProvidersSettings, error) { + options := &DhtFindProvidersSettings{ + NumProviders: 20, + } + + for _, opt := range opts { + err := opt(options) + if err != nil { + return nil, err + } + } + return options, nil +} + +type dhtOpts struct{} + +var Dht dhtOpts + +// Recursive is an option for Dht.Provide which specifies whether to provide +// the given path recursively +func (dhtOpts) Recursive(recursive bool) DhtProvideOption { + return func(settings *DhtProvideSettings) error { + settings.Recursive = recursive + return nil + } +} + +// NumProviders is an option for Dht.FindProviders which specifies the +// number of peers to look for. Default is 20 +func (dhtOpts) NumProviders(numProviders int) DhtFindProvidersOption { + return func(settings *DhtFindProvidersSettings) error { + settings.NumProviders = numProviders + return nil + } +} diff --git a/coreiface/options/global.go b/coreiface/options/global.go new file mode 100644 index 0000000000..90e2586f10 --- /dev/null +++ b/coreiface/options/global.go @@ -0,0 +1,47 @@ +package options + +type ApiSettings struct { + Offline bool + FetchBlocks bool +} + +type ApiOption func(*ApiSettings) error + +func ApiOptions(opts ...ApiOption) (*ApiSettings, error) { + options := &ApiSettings{ + Offline: false, + FetchBlocks: true, + } + + return ApiOptionsTo(options, opts...) +} + +func ApiOptionsTo(options *ApiSettings, opts ...ApiOption) (*ApiSettings, error) { + for _, opt := range opts { + err := opt(options) + if err != nil { + return nil, err + } + } + return options, nil +} + +type apiOpts struct{} + +var Api apiOpts + +func (apiOpts) Offline(offline bool) ApiOption { + return func(settings *ApiSettings) error { + settings.Offline = offline + return nil + } +} + +// FetchBlocks when set to false prevents api from fetching blocks from the +// network while allowing other services such as IPNS to still be online +func (apiOpts) FetchBlocks(fetch bool) ApiOption { + return func(settings *ApiSettings) error { + settings.FetchBlocks = fetch + return nil + } +} diff --git a/coreiface/options/key.go b/coreiface/options/key.go new file mode 100644 index 0000000000..4bc53a65fe --- /dev/null +++ b/coreiface/options/key.go @@ -0,0 +1,87 @@ +package options + +const ( + RSAKey = "rsa" + Ed25519Key = "ed25519" + + DefaultRSALen = 2048 +) + +type KeyGenerateSettings struct { + Algorithm string + Size int +} + +type KeyRenameSettings struct { + Force bool +} + +type KeyGenerateOption func(*KeyGenerateSettings) error +type KeyRenameOption func(*KeyRenameSettings) error + +func KeyGenerateOptions(opts ...KeyGenerateOption) (*KeyGenerateSettings, error) { + options := &KeyGenerateSettings{ + Algorithm: RSAKey, + Size: -1, + } + + for _, opt := range opts { + err := opt(options) + if err != nil { + return nil, err + } + } + return options, nil +} + +func KeyRenameOptions(opts ...KeyRenameOption) (*KeyRenameSettings, error) { + options := &KeyRenameSettings{ + Force: false, + } + + for _, opt := range opts { + err := opt(options) + if err != nil { + return nil, err + } + } + return options, nil +} + +type keyOpts struct{} + +var Key keyOpts + +// Type is an option for Key.Generate which specifies which algorithm +// should be used for the key. Default is options.RSAKey +// +// Supported key types: +// * options.RSAKey +// * options.Ed25519Key +func (keyOpts) Type(algorithm string) KeyGenerateOption { + return func(settings *KeyGenerateSettings) error { + settings.Algorithm = algorithm + return nil + } +} + +// Size is an option for Key.Generate which specifies the size of the key to +// generated. Default is -1 +// +// value of -1 means 'use default size for key type': +// - 2048 for RSA +func (keyOpts) Size(size int) KeyGenerateOption { + return func(settings *KeyGenerateSettings) error { + settings.Size = size + return nil + } +} + +// Force is an option for Key.Rename which specifies whether to allow to +// replace existing keys. +func (keyOpts) Force(force bool) KeyRenameOption { + return func(settings *KeyRenameSettings) error { + settings.Force = force + return nil + } +} diff --git a/coreiface/options/name.go b/coreiface/options/name.go new file mode 100644 index 0000000000..d12824d296 --- /dev/null +++ b/coreiface/options/name.go @@ -0,0 +1,121 @@ +package options + +import ( + "time" + + ropts "github.com/ipfs/go-libipfs/coreiface/options/namesys" +) + +const ( + DefaultNameValidTime = 24 * time.Hour +) + +type NamePublishSettings struct { + ValidTime time.Duration + Key string + + TTL *time.Duration + + AllowOffline bool +} + +type NameResolveSettings struct { + Cache bool + + ResolveOpts []ropts.ResolveOpt +} + +type NamePublishOption func(*NamePublishSettings) error +type NameResolveOption func(*NameResolveSettings) error + +func NamePublishOptions(opts ...NamePublishOption) (*NamePublishSettings, error) { + options := &NamePublishSettings{ + ValidTime: DefaultNameValidTime, + Key: "self", + + AllowOffline: false, + } + + for _, opt := range opts { + err := opt(options) + if err != nil { + return nil, err + } + } + + return options, nil +} + +func NameResolveOptions(opts ...NameResolveOption) (*NameResolveSettings, error) { + options := &NameResolveSettings{ + Cache: true, + } + + for _, opt := range opts { + err := opt(options) + if err != nil { + return nil, err + } + } + + return options, nil +} + +type nameOpts struct{} + +var Name nameOpts + +// ValidTime is an option for Name.Publish which specifies for how long the +// entry will remain valid. Default value is 24h +func (nameOpts) ValidTime(validTime time.Duration) NamePublishOption { + return func(settings *NamePublishSettings) error { + settings.ValidTime = validTime + return nil + } +} + +// Key is an option for Name.Publish which specifies the key to use for +// publishing. Default value is "self" which is the node's own PeerID. +// The key parameter must be either PeerID or keystore key alias. +// +// You can use KeyAPI to list and generate more names and their respective keys. +func (nameOpts) Key(key string) NamePublishOption { + return func(settings *NamePublishSettings) error { + settings.Key = key + return nil + } +} + +// AllowOffline is an option for Name.Publish which specifies whether to allow +// publishing when the node is offline. Default value is false +func (nameOpts) AllowOffline(allow bool) NamePublishOption { + return func(settings *NamePublishSettings) error { + settings.AllowOffline = allow + return nil + } +} + +// TTL is an option for Name.Publish which specifies the time duration the +// published record should be cached for (caution: experimental). +func (nameOpts) TTL(ttl time.Duration) NamePublishOption { + return func(settings *NamePublishSettings) error { + settings.TTL = &ttl + return nil + } +} + +// Cache is an option for Name.Resolve which specifies if cache should be used. +// Default value is true +func (nameOpts) Cache(cache bool) NameResolveOption { + return func(settings *NameResolveSettings) error { + settings.Cache = cache + return nil + } +} + +func (nameOpts) ResolveOption(opt ropts.ResolveOpt) NameResolveOption { + return func(settings *NameResolveSettings) error { + settings.ResolveOpts = append(settings.ResolveOpts, opt) + return nil + } +} diff --git a/coreiface/options/namesys/opts.go b/coreiface/options/namesys/opts.go new file mode 100644 index 0000000000..0cd1ba7781 --- /dev/null +++ b/coreiface/options/namesys/opts.go @@ -0,0 +1,123 @@ +package nsopts + +import ( + "time" +) + +const ( + // DefaultDepthLimit is the default depth limit used by Resolve. + DefaultDepthLimit = 32 + + // UnlimitedDepth allows infinite recursion in Resolve. You + // probably don't want to use this, but it's here if you absolutely + // trust resolution to eventually complete and can't put an upper + // limit on how many steps it will take. + UnlimitedDepth = 0 + + // DefaultIPNSRecordTTL specifies the time that the record can be cached + // before checking if its validity again. + DefaultIPNSRecordTTL = time.Minute + + // DefaultIPNSRecordEOL specifies the time that the network will cache IPNS + // records after being published. Records should be re-published before this + // interval expires. We use the same default expiration as the DHT. + DefaultIPNSRecordEOL = 48 * time.Hour +) + +// ResolveOpts specifies options for resolving an IPNS path +type ResolveOpts struct { + // Recursion depth limit + Depth uint + // The number of IPNS records to retrieve from the DHT + // (the best record is selected from this set) + DhtRecordCount uint + // The amount of time to wait for DHT records to be fetched + // and verified. A zero value indicates that there is no explicit + // timeout (although there is an implicit timeout due to dial + // timeouts within the DHT) + DhtTimeout time.Duration +} + +// DefaultResolveOpts returns the default options for resolving +// an IPNS path +func DefaultResolveOpts() ResolveOpts { + return ResolveOpts{ + Depth: DefaultDepthLimit, + DhtRecordCount: 16, + DhtTimeout: time.Minute, + } +} + +// ResolveOpt is used to set an option +type ResolveOpt func(*ResolveOpts) + +// Depth is the recursion depth limit +func Depth(depth uint) ResolveOpt { + return func(o *ResolveOpts) { + o.Depth = depth + } +} + +// DhtRecordCount is the number of IPNS records to retrieve from the DHT +func DhtRecordCount(count uint) ResolveOpt { + return func(o *ResolveOpts) { + o.DhtRecordCount = count + } +} + +// DhtTimeout is the amount of time to wait for DHT records to be fetched +// and verified. A zero value indicates that there is no explicit timeout +func DhtTimeout(timeout time.Duration) ResolveOpt { + return func(o *ResolveOpts) { + o.DhtTimeout = timeout + } +} + +// ProcessOpts converts an array of ResolveOpt into a ResolveOpts object +func ProcessOpts(opts []ResolveOpt) ResolveOpts { + rsopts := DefaultResolveOpts() + for _, option := range opts { + option(&rsopts) + } + return rsopts +} + +// PublishOptions specifies options for publishing an IPNS record. +type PublishOptions struct { + EOL time.Time + TTL time.Duration +} + +// DefaultPublishOptions returns the default options for publishing an IPNS record. +func DefaultPublishOptions() PublishOptions { + return PublishOptions{ + EOL: time.Now().Add(DefaultIPNSRecordEOL), + TTL: DefaultIPNSRecordTTL, + } +} + +// PublishOption is used to set an option for PublishOpts. +type PublishOption func(*PublishOptions) + +// PublishWithEOL sets an EOL. +func PublishWithEOL(eol time.Time) PublishOption { + return func(o *PublishOptions) { + o.EOL = eol + } +} + +// PublishWithEOL sets a TTL. +func PublishWithTTL(ttl time.Duration) PublishOption { + return func(o *PublishOptions) { + o.TTL = ttl + } +} + +// ProcessPublishOptions converts an array of PublishOpt into a PublishOpts object. +func ProcessPublishOptions(opts []PublishOption) PublishOptions { + rsopts := DefaultPublishOptions() + for _, option := range opts { + option(&rsopts) + } + return rsopts +} diff --git a/coreiface/options/object.go b/coreiface/options/object.go new file mode 100644 index 0000000000..e484a9f363 --- /dev/null +++ b/coreiface/options/object.go @@ -0,0 +1,124 @@ +package options + +type ObjectNewSettings struct { + Type string +} + +type ObjectPutSettings struct { + InputEnc string + DataType string + Pin bool +} + +type ObjectAddLinkSettings struct { + Create bool +} + +type ObjectNewOption func(*ObjectNewSettings) error +type ObjectPutOption func(*ObjectPutSettings) error +type ObjectAddLinkOption func(*ObjectAddLinkSettings) error + +func ObjectNewOptions(opts ...ObjectNewOption) (*ObjectNewSettings, error) { + options := &ObjectNewSettings{ + Type: "empty", + } + + for _, opt := range opts { + err := opt(options) + if err != nil { + return nil, err + } + } + return options, nil +} + +func ObjectPutOptions(opts ...ObjectPutOption) (*ObjectPutSettings, error) { + options := &ObjectPutSettings{ + InputEnc: "json", + DataType: "text", + Pin: false, + } + + for _, opt := range opts { + err := opt(options) + if err != nil { + return nil, err + } + } + return options, nil +} + +func ObjectAddLinkOptions(opts ...ObjectAddLinkOption) (*ObjectAddLinkSettings, error) { + options := &ObjectAddLinkSettings{ + Create: false, + } + + for _, opt := range opts { + err := opt(options) + if err != nil { + return nil, err + } + } + return options, nil +} + +type objectOpts struct{} + +var Object objectOpts + +// Type is an option for Object.New which allows to change the type of created +// dag node. +// +// Supported types: +// * 'empty' - Empty node +// * 'unixfs-dir' - Empty UnixFS directory +func (objectOpts) Type(t string) ObjectNewOption { + return func(settings *ObjectNewSettings) error { + settings.Type = t + return nil + } +} + +// InputEnc is an option for Object.Put which specifies the input encoding of the +// data. Default is "json". +// +// Supported encodings: +// * "protobuf" +// * "json" +func (objectOpts) InputEnc(e string) ObjectPutOption { + return func(settings *ObjectPutSettings) error { + settings.InputEnc = e + return nil + } +} + +// DataType is an option for Object.Put which specifies the encoding of data +// field when using Json or XML input encoding. +// +// Supported types: +// * "text" (default) +// * "base64" +func (objectOpts) DataType(t string) ObjectPutOption { + return func(settings *ObjectPutSettings) error { + settings.DataType = t + return nil + } +} + +// Pin is an option for Object.Put which specifies whether to pin the added +// objects, default is false +func (objectOpts) Pin(pin bool) ObjectPutOption { + return func(settings *ObjectPutSettings) error { + settings.Pin = pin + return nil + } +} + +// Create is an option for Object.AddLink which specifies whether create required +// directories for the child +func (objectOpts) Create(create bool) ObjectAddLinkOption { + return func(settings *ObjectAddLinkSettings) error { + settings.Create = create + return nil + } +} diff --git a/coreiface/options/pin.go b/coreiface/options/pin.go new file mode 100644 index 0000000000..75c2b8a263 --- /dev/null +++ b/coreiface/options/pin.go @@ -0,0 +1,283 @@ +package options + +import "fmt" + +// PinAddSettings represent the settings for PinAPI.Add +type PinAddSettings struct { + Recursive bool +} + +// PinLsSettings represent the settings for PinAPI.Ls +type PinLsSettings struct { + Type string +} + +// PinIsPinnedSettings represent the settings for PinAPI.IsPinned +type PinIsPinnedSettings struct { + WithType string +} + +// PinRmSettings represents the settings for PinAPI.Rm +type PinRmSettings struct { + Recursive bool +} + +// PinUpdateSettings represent the settings for PinAPI.Update +type PinUpdateSettings struct { + Unpin bool +} + +// PinAddOption is the signature of an option for PinAPI.Add +type PinAddOption func(*PinAddSettings) error + +// PinLsOption is the signature of an option for PinAPI.Ls +type PinLsOption func(*PinLsSettings) error + +// PinIsPinnedOption is the signature of an option for PinAPI.IsPinned +type PinIsPinnedOption func(*PinIsPinnedSettings) error + +// PinRmOption is the signature of an option for PinAPI.Rm +type PinRmOption func(*PinRmSettings) error + +// PinUpdateOption is the signature of an option for PinAPI.Update +type PinUpdateOption func(*PinUpdateSettings) error + +// PinAddOptions compile a series of PinAddOption into a ready to use +// PinAddSettings and set the default values. +func PinAddOptions(opts ...PinAddOption) (*PinAddSettings, error) { + options := &PinAddSettings{ + Recursive: true, + } + + for _, opt := range opts { + err := opt(options) + if err != nil { + return nil, err + } + } + + return options, nil +} + +// PinLsOptions compile a series of PinLsOption into a ready to use +// PinLsSettings and set the default values. +func PinLsOptions(opts ...PinLsOption) (*PinLsSettings, error) { + options := &PinLsSettings{ + Type: "all", + } + + for _, opt := range opts { + err := opt(options) + if err != nil { + return nil, err + } + } + + return options, nil +} + +// PinIsPinnedOptions compile a series of PinIsPinnedOption into a ready to use +// PinIsPinnedSettings and set the default values. +func PinIsPinnedOptions(opts ...PinIsPinnedOption) (*PinIsPinnedSettings, error) { + options := &PinIsPinnedSettings{ + WithType: "all", + } + + for _, opt := range opts { + err := opt(options) + if err != nil { + return nil, err + } + } + + return options, nil +} + +// PinRmOptions compile a series of PinRmOption into a ready to use +// PinRmSettings and set the default values. +func PinRmOptions(opts ...PinRmOption) (*PinRmSettings, error) { + options := &PinRmSettings{ + Recursive: true, + } + + for _, opt := range opts { + if err := opt(options); err != nil { + return nil, err + } + } + + return options, nil +} + +// PinUpdateOptions compile a series of PinUpdateOption into a ready to use +// PinUpdateSettings and set the default values. +func PinUpdateOptions(opts ...PinUpdateOption) (*PinUpdateSettings, error) { + options := &PinUpdateSettings{ + Unpin: true, + } + + for _, opt := range opts { + err := opt(options) + if err != nil { + return nil, err + } + } + + return options, nil +} + +type pinOpts struct { + Ls pinLsOpts + IsPinned pinIsPinnedOpts +} + +// Pin provide an access to all the options for the Pin API. +var Pin pinOpts + +type pinLsOpts struct{} + +// All is an option for Pin.Ls which will make it return all pins. It is +// the default +func (pinLsOpts) All() PinLsOption { + return Pin.Ls.pinType("all") +} + +// Recursive is an option for Pin.Ls which will make it only return recursive +// pins +func (pinLsOpts) Recursive() PinLsOption { + return Pin.Ls.pinType("recursive") +} + +// Direct is an option for Pin.Ls which will make it only return direct (non +// recursive) pins +func (pinLsOpts) Direct() PinLsOption { + return Pin.Ls.pinType("direct") +} + +// Indirect is an option for Pin.Ls which will make it only return indirect pins +// (objects referenced by other recursively pinned objects) +func (pinLsOpts) Indirect() PinLsOption { + return Pin.Ls.pinType("indirect") +} + +// Type is an option for Pin.Ls which will make it only return pins of the given +// type. +// +// Supported values: +// - "direct" - directly pinned objects +// - "recursive" - roots of recursive pins +// - "indirect" - indirectly pinned objects (referenced by recursively pinned +// objects) +// - "all" - all pinned objects (default) +func (pinLsOpts) Type(typeStr string) (PinLsOption, error) { + switch typeStr { + case "all", "direct", "indirect", "recursive": + return Pin.Ls.pinType(typeStr), nil + default: + return nil, fmt.Errorf("invalid type '%s', must be one of {direct, indirect, recursive, all}", typeStr) + } +} + +// pinType is an option for Pin.Ls which allows to specify which pin types should +// be returned +// +// Supported values: +// - "direct" - directly pinned objects +// - "recursive" - roots of recursive pins +// - "indirect" - indirectly pinned objects (referenced by recursively pinned +// objects) +// - "all" - all pinned objects (default) +func (pinLsOpts) pinType(t string) PinLsOption { + return func(settings *PinLsSettings) error { + settings.Type = t + return nil + } +} + +type pinIsPinnedOpts struct{} + +// All is an option for Pin.IsPinned which will make it search in all type of pins. +// It is the default +func (pinIsPinnedOpts) All() PinIsPinnedOption { + return Pin.IsPinned.pinType("all") +} + +// Recursive is an option for Pin.IsPinned which will make it only search in +// recursive pins +func (pinIsPinnedOpts) Recursive() PinIsPinnedOption { + return Pin.IsPinned.pinType("recursive") +} + +// Direct is an option for Pin.IsPinned which will make it only search in direct +// (non recursive) pins +func (pinIsPinnedOpts) Direct() PinIsPinnedOption { + return Pin.IsPinned.pinType("direct") +} + +// Indirect is an option for Pin.IsPinned which will make it only search indirect +// pins (objects referenced by other recursively pinned objects) +func (pinIsPinnedOpts) Indirect() PinIsPinnedOption { + return Pin.IsPinned.pinType("indirect") +} + +// Type is an option for Pin.IsPinned which will make it only search pins of the given +// type. +// +// Supported values: +// - "direct" - directly pinned objects +// - "recursive" - roots of recursive pins +// - "indirect" - indirectly pinned objects (referenced by recursively pinned +// objects) +// - "all" - all pinned objects (default) +func (pinIsPinnedOpts) Type(typeStr string) (PinIsPinnedOption, error) { + switch typeStr { + case "all", "direct", "indirect", "recursive": + return Pin.IsPinned.pinType(typeStr), nil + default: + return nil, fmt.Errorf("invalid type '%s', must be one of {direct, indirect, recursive, all}", typeStr) + } +} + +// pinType is an option for Pin.IsPinned which allows to specify which pin type the given +// pin is expected to be, speeding up the research. +// +// Supported values: +// - "direct" - directly pinned objects +// - "recursive" - roots of recursive pins +// - "indirect" - indirectly pinned objects (referenced by recursively pinned +// objects) +// - "all" - all pinned objects (default) +func (pinIsPinnedOpts) pinType(t string) PinIsPinnedOption { + return func(settings *PinIsPinnedSettings) error { + settings.WithType = t + return nil + } +} + +// Recursive is an option for Pin.Add which specifies whether to pin an entire +// object tree or just one object. Default: true +func (pinOpts) Recursive(recursive bool) PinAddOption { + return func(settings *PinAddSettings) error { + settings.Recursive = recursive + return nil + } +} + +// RmRecursive is an option for Pin.Rm which specifies whether to recursively +// unpin the object linked to by the specified object(s). This does not remove +// indirect pins referenced by other recursive pins. +func (pinOpts) RmRecursive(recursive bool) PinRmOption { + return func(settings *PinRmSettings) error { + settings.Recursive = recursive + return nil + } +} + +// Unpin is an option for Pin.Update which specifies whether to remove the old pin. +// Default is true. +func (pinOpts) Unpin(unpin bool) PinUpdateOption { + return func(settings *PinUpdateSettings) error { + settings.Unpin = unpin + return nil + } +} diff --git a/coreiface/options/pubsub.go b/coreiface/options/pubsub.go new file mode 100644 index 0000000000..c387d613db --- /dev/null +++ b/coreiface/options/pubsub.go @@ -0,0 +1,58 @@ +package options + +type PubSubPeersSettings struct { + Topic string +} + +type PubSubSubscribeSettings struct { + Discover bool +} + +type PubSubPeersOption func(*PubSubPeersSettings) error +type PubSubSubscribeOption func(*PubSubSubscribeSettings) error + +func PubSubPeersOptions(opts ...PubSubPeersOption) (*PubSubPeersSettings, error) { + options := &PubSubPeersSettings{ + Topic: "", + } + + for _, opt := range opts { + err := opt(options) + if err != nil { + return nil, err + } + } + return options, nil +} + +func PubSubSubscribeOptions(opts ...PubSubSubscribeOption) (*PubSubSubscribeSettings, error) { + options := &PubSubSubscribeSettings{ + Discover: false, + } + + for _, opt := range opts { + err := opt(options) + if err != nil { + return nil, err + } + } + return options, nil +} + +type pubsubOpts struct{} + +var PubSub pubsubOpts + +func (pubsubOpts) Topic(topic string) PubSubPeersOption { + return func(settings *PubSubPeersSettings) error { + settings.Topic = topic + return nil + } +} + +func (pubsubOpts) Discover(discover bool) PubSubSubscribeOption { + return func(settings *PubSubSubscribeSettings) error { + settings.Discover = discover + return nil + } +} diff --git a/coreiface/options/unixfs.go b/coreiface/options/unixfs.go new file mode 100644 index 0000000000..cd15991e27 --- /dev/null +++ b/coreiface/options/unixfs.go @@ -0,0 +1,293 @@ +package options + +import ( + "errors" + "fmt" + + cid "github.com/ipfs/go-cid" + dag "github.com/ipfs/go-merkledag" + mh "github.com/multiformats/go-multihash" +) + +type Layout int + +const ( + BalancedLayout Layout = iota + TrickleLayout +) + +type UnixfsAddSettings struct { + CidVersion int + MhType uint64 + + Inline bool + InlineLimit int + RawLeaves bool + RawLeavesSet bool + + Chunker string + Layout Layout + + Pin bool + OnlyHash bool + FsCache bool + NoCopy bool + + Events chan<- interface{} + Silent bool + Progress bool +} + +type UnixfsLsSettings struct { + ResolveChildren bool + UseCumulativeSize bool +} + +type UnixfsAddOption func(*UnixfsAddSettings) error +type UnixfsLsOption func(*UnixfsLsSettings) error + +func UnixfsAddOptions(opts ...UnixfsAddOption) (*UnixfsAddSettings, cid.Prefix, error) { + options := &UnixfsAddSettings{ + CidVersion: -1, + MhType: mh.SHA2_256, + + Inline: false, + InlineLimit: 32, + RawLeaves: false, + RawLeavesSet: false, + + Chunker: "size-262144", + Layout: BalancedLayout, + + Pin: false, + OnlyHash: false, + FsCache: false, + NoCopy: false, + + Events: nil, + Silent: false, + Progress: false, + } + + for _, opt := range opts { + err := opt(options) + if err != nil { + return nil, cid.Prefix{}, err + } + } + + // nocopy -> rawblocks + if options.NoCopy && !options.RawLeaves { + // fixed? + if options.RawLeavesSet { + return nil, cid.Prefix{}, fmt.Errorf("nocopy option requires '--raw-leaves' to be enabled as well") + } + + // No, satisfy mandatory constraint. + options.RawLeaves = true + } + + // (hash != "sha2-256") -> CIDv1 + if options.MhType != mh.SHA2_256 { + switch options.CidVersion { + case 0: + return nil, cid.Prefix{}, errors.New("CIDv0 only supports sha2-256") + case 1, -1: + options.CidVersion = 1 + default: + return nil, cid.Prefix{}, fmt.Errorf("unknown CID version: %d", options.CidVersion) + } + } else { + if options.CidVersion < 0 { + // Default to CIDv0 + options.CidVersion = 0 + } + } + + // cidV1 -> raw blocks (by default) + if options.CidVersion > 0 && !options.RawLeavesSet { + options.RawLeaves = true + } + + prefix, err := dag.PrefixForCidVersion(options.CidVersion) + if err != nil { + return nil, cid.Prefix{}, err + } + + prefix.MhType = options.MhType + prefix.MhLength = -1 + + return options, prefix, nil +} + +func UnixfsLsOptions(opts ...UnixfsLsOption) (*UnixfsLsSettings, error) { + options := &UnixfsLsSettings{ + ResolveChildren: true, + } + + for _, opt := range opts { + err := opt(options) + if err != nil { + return nil, err + } + } + + return options, nil +} + +type unixfsOpts struct{} + +var Unixfs unixfsOpts + +// CidVersion specifies which CID version to use. Defaults to 0 unless an option +// that depends on CIDv1 is passed. +func (unixfsOpts) CidVersion(version int) UnixfsAddOption { + return func(settings *UnixfsAddSettings) error { + settings.CidVersion = version + return nil + } +} + +// Hash function to use. Implies CIDv1 if not set to sha2-256 (default). +// +// Table of functions is declared in https://github.com/multiformats/go-multihash/blob/master/multihash.go +func (unixfsOpts) Hash(mhtype uint64) UnixfsAddOption { + return func(settings *UnixfsAddSettings) error { + settings.MhType = mhtype + return nil + } +} + +// RawLeaves specifies whether to use raw blocks for leaves (data nodes with no +// links) instead of wrapping them with unixfs structures. +func (unixfsOpts) RawLeaves(enable bool) UnixfsAddOption { + return func(settings *UnixfsAddSettings) error { + settings.RawLeaves = enable + settings.RawLeavesSet = true + return nil + } +} + +// Inline tells the adder to inline small blocks into CIDs +func (unixfsOpts) Inline(enable bool) UnixfsAddOption { + return func(settings *UnixfsAddSettings) error { + settings.Inline = enable + return nil + } +} + +// InlineLimit sets the amount of bytes below which blocks will be encoded +// directly into CID instead of being stored and addressed by it's hash. +// Specifying this option won't enable block inlining. For that use `Inline` +// option. Default: 32 bytes +// +// Note that while there is no hard limit on the number of bytes, it should be +// kept at a reasonably low value, such as 64; implementations may choose to +// reject anything larger. +func (unixfsOpts) InlineLimit(limit int) UnixfsAddOption { + return func(settings *UnixfsAddSettings) error { + settings.InlineLimit = limit + return nil + } +} + +// Chunker specifies settings for the chunking algorithm to use. +// +// Default: size-262144, formats: +// size-[bytes] - Simple chunker splitting data into blocks of n bytes +// rabin-[min]-[avg]-[max] - Rabin chunker +func (unixfsOpts) Chunker(chunker string) UnixfsAddOption { + return func(settings *UnixfsAddSettings) error { + settings.Chunker = chunker + return nil + } +} + +// Layout tells the adder how to balance data between leaves. +// options.BalancedLayout is the default, it's optimized for static seekable +// files. +// options.TrickleLayout is optimized for streaming data, +func (unixfsOpts) Layout(layout Layout) UnixfsAddOption { + return func(settings *UnixfsAddSettings) error { + settings.Layout = layout + return nil + } +} + +// Pin tells the adder to pin the file root recursively after adding +func (unixfsOpts) Pin(pin bool) UnixfsAddOption { + return func(settings *UnixfsAddSettings) error { + settings.Pin = pin + return nil + } +} + +// HashOnly will make the adder calculate data hash without storing it in the +// blockstore or announcing it to the network +func (unixfsOpts) HashOnly(hashOnly bool) UnixfsAddOption { + return func(settings *UnixfsAddSettings) error { + settings.OnlyHash = hashOnly + return nil + } +} + +// Events specifies channel which will be used to report events about ongoing +// Add operation. +// +// Note that if this channel blocks it may slowdown the adder +func (unixfsOpts) Events(sink chan<- interface{}) UnixfsAddOption { + return func(settings *UnixfsAddSettings) error { + settings.Events = sink + return nil + } +} + +// Silent reduces event output +func (unixfsOpts) Silent(silent bool) UnixfsAddOption { + return func(settings *UnixfsAddSettings) error { + settings.Silent = silent + return nil + } +} + +// Progress tells the adder whether to enable progress events +func (unixfsOpts) Progress(enable bool) UnixfsAddOption { + return func(settings *UnixfsAddSettings) error { + settings.Progress = enable + return nil + } +} + +// FsCache tells the adder to check the filestore for pre-existing blocks +// +// Experimental +func (unixfsOpts) FsCache(enable bool) UnixfsAddOption { + return func(settings *UnixfsAddSettings) error { + settings.FsCache = enable + return nil + } +} + +// NoCopy tells the adder to add the files using filestore. Implies RawLeaves. +// +// Experimental +func (unixfsOpts) Nocopy(enable bool) UnixfsAddOption { + return func(settings *UnixfsAddSettings) error { + settings.NoCopy = enable + return nil + } +} + +func (unixfsOpts) ResolveChildren(resolve bool) UnixfsLsOption { + return func(settings *UnixfsLsSettings) error { + settings.ResolveChildren = resolve + return nil + } +} + +func (unixfsOpts) UseCumulativeSize(use bool) UnixfsLsOption { + return func(settings *UnixfsLsSettings) error { + settings.UseCumulativeSize = use + return nil + } +} diff --git a/coreiface/path/path.go b/coreiface/path/path.go new file mode 100644 index 0000000000..5c1a533b38 --- /dev/null +++ b/coreiface/path/path.go @@ -0,0 +1,199 @@ +package path + +import ( + "strings" + + cid "github.com/ipfs/go-cid" + ipfspath "github.com/ipfs/go-libipfs/path" +) + +// Path is a generic wrapper for paths used in the API. A path can be resolved +// to a CID using one of Resolve functions in the API. +// +// Paths must be prefixed with a valid prefix: +// +// * /ipfs - Immutable unixfs path (files) +// * /ipld - Immutable ipld path (data) +// * /ipns - Mutable names. Usually resolves to one of the immutable paths +// TODO: /local (MFS) +type Path interface { + // String returns the path as a string. + String() string + + // Namespace returns the first component of the path. + // + // For example path "/ipfs/QmHash", calling Namespace() will return "ipfs" + // + // Calling this method on invalid paths (IsValid() != nil) will result in + // empty string + Namespace() string + + // Mutable returns false if the data pointed to by this path in guaranteed + // to not change. + // + // Note that resolved mutable path can be immutable. + Mutable() bool + + // IsValid checks if this path is a valid ipfs Path, returning nil iff it is + // valid + IsValid() error +} + +// Resolved is a path which was resolved to the last resolvable node. +// ResolvedPaths are guaranteed to return nil from `IsValid` +type Resolved interface { + // Cid returns the CID of the node referenced by the path. Remainder of the + // path is guaranteed to be within the node. + // + // Examples: + // If you have 3 linked objects: QmRoot -> A -> B: + // + // cidB := {"foo": {"bar": 42 }} + // cidA := {"B": {"/": cidB }} + // cidRoot := {"A": {"/": cidA }} + // + // And resolve paths: + // + // * "/ipfs/${cidRoot}" + // * Calling Cid() will return `cidRoot` + // * Calling Root() will return `cidRoot` + // * Calling Remainder() will return `` + // + // * "/ipfs/${cidRoot}/A" + // * Calling Cid() will return `cidA` + // * Calling Root() will return `cidRoot` + // * Calling Remainder() will return `` + // + // * "/ipfs/${cidRoot}/A/B/foo" + // * Calling Cid() will return `cidB` + // * Calling Root() will return `cidRoot` + // * Calling Remainder() will return `foo` + // + // * "/ipfs/${cidRoot}/A/B/foo/bar" + // * Calling Cid() will return `cidB` + // * Calling Root() will return `cidRoot` + // * Calling Remainder() will return `foo/bar` + Cid() cid.Cid + + // Root returns the CID of the root object of the path + // + // Example: + // If you have 3 linked objects: QmRoot -> A -> B, and resolve path + // "/ipfs/QmRoot/A/B", the Root method will return the CID of object QmRoot + // + // For more examples see the documentation of Cid() method + Root() cid.Cid + + // Remainder returns unresolved part of the path + // + // Example: + // If you have 2 linked objects: QmRoot -> A, where A is a CBOR node + // containing the following data: + // + // {"foo": {"bar": 42 }} + // + // When resolving "/ipld/QmRoot/A/foo/bar", Remainder will return "foo/bar" + // + // For more examples see the documentation of Cid() method + Remainder() string + + Path +} + +// path implements coreiface.Path +type path struct { + path string +} + +// resolvedPath implements coreiface.resolvedPath +type resolvedPath struct { + path + cid cid.Cid + root cid.Cid + remainder string +} + +// Join appends provided segments to the base path +func Join(base Path, a ...string) Path { + s := strings.Join(append([]string{base.String()}, a...), "/") + return &path{path: s} +} + +// IpfsPath creates new /ipfs path from the provided CID +func IpfsPath(c cid.Cid) Resolved { + return &resolvedPath{ + path: path{"/ipfs/" + c.String()}, + cid: c, + root: c, + remainder: "", + } +} + +// IpldPath creates new /ipld path from the provided CID +func IpldPath(c cid.Cid) Resolved { + return &resolvedPath{ + path: path{"/ipld/" + c.String()}, + cid: c, + root: c, + remainder: "", + } +} + +// New parses string path to a Path +func New(p string) Path { + if pp, err := ipfspath.ParsePath(p); err == nil { + p = pp.String() + } + + return &path{path: p} +} + +// NewResolvedPath creates new Resolved path. This function performs no checks +// and is intended to be used by resolver implementations. Incorrect inputs may +// cause panics. Handle with care. +func NewResolvedPath(ipath ipfspath.Path, c cid.Cid, root cid.Cid, remainder string) Resolved { + return &resolvedPath{ + path: path{ipath.String()}, + cid: c, + root: root, + remainder: remainder, + } +} + +func (p *path) String() string { + return p.path +} + +func (p *path) Namespace() string { + ip, err := ipfspath.ParsePath(p.path) + if err != nil { + return "" + } + + if len(ip.Segments()) < 1 { + panic("path without namespace") // this shouldn't happen under any scenario + } + return ip.Segments()[0] +} + +func (p *path) Mutable() bool { + // TODO: MFS: check for /local + return p.Namespace() == "ipns" +} + +func (p *path) IsValid() error { + _, err := ipfspath.ParsePath(p.path) + return err +} + +func (p *resolvedPath) Cid() cid.Cid { + return p.cid +} + +func (p *resolvedPath) Root() cid.Cid { + return p.root +} + +func (p *resolvedPath) Remainder() string { + return p.remainder +} diff --git a/coreiface/pin.go b/coreiface/pin.go new file mode 100644 index 0000000000..780652732a --- /dev/null +++ b/coreiface/pin.go @@ -0,0 +1,63 @@ +package iface + +import ( + "context" + + path "github.com/ipfs/go-libipfs/coreiface/path" + + "github.com/ipfs/go-libipfs/coreiface/options" +) + +// Pin holds information about pinned resource +type Pin interface { + // Path to the pinned object + Path() path.Resolved + + // Type of the pin + Type() string + + // if not nil, an error happened. Everything else should be ignored. + Err() error +} + +// PinStatus holds information about pin health +type PinStatus interface { + // Ok indicates whether the pin has been verified to be correct + Ok() bool + + // BadNodes returns any bad (usually missing) nodes from the pin + BadNodes() []BadPinNode +} + +// BadPinNode is a node that has been marked as bad by Pin.Verify +type BadPinNode interface { + // Path is the path of the node + Path() path.Resolved + + // Err is the reason why the node has been marked as bad + Err() error +} + +// PinAPI specifies the interface to pining +type PinAPI interface { + // Add creates new pin, be default recursive - pinning the whole referenced + // tree + Add(context.Context, path.Path, ...options.PinAddOption) error + + // Ls returns list of pinned objects on this node + Ls(context.Context, ...options.PinLsOption) (<-chan Pin, error) + + // IsPinned returns whether or not the given cid is pinned + // and an explanation of why its pinned + IsPinned(context.Context, path.Path, ...options.PinIsPinnedOption) (string, bool, error) + + // Rm removes pin for object specified by the path + Rm(context.Context, path.Path, ...options.PinRmOption) error + + // Update changes one pin to another, skipping checks for matching paths in + // the old tree + Update(ctx context.Context, from path.Path, to path.Path, opts ...options.PinUpdateOption) error + + // Verify verifies the integrity of pinned objects + Verify(context.Context) (<-chan PinStatus, error) +} diff --git a/coreiface/pubsub.go b/coreiface/pubsub.go new file mode 100644 index 0000000000..ab9740ffb1 --- /dev/null +++ b/coreiface/pubsub.go @@ -0,0 +1,48 @@ +package iface + +import ( + "context" + "io" + + "github.com/ipfs/go-libipfs/coreiface/options" + + "github.com/libp2p/go-libp2p/core/peer" +) + +// PubSubSubscription is an active PubSub subscription +type PubSubSubscription interface { + io.Closer + + // Next return the next incoming message + Next(context.Context) (PubSubMessage, error) +} + +// PubSubMessage is a single PubSub message +type PubSubMessage interface { + // From returns id of a peer from which the message has arrived + From() peer.ID + + // Data returns the message body + Data() []byte + + // Seq returns message identifier + Seq() []byte + + // Topics returns list of topics this message was set to + Topics() []string +} + +// PubSubAPI specifies the interface to PubSub +type PubSubAPI interface { + // Ls lists subscribed topics by name + Ls(context.Context) ([]string, error) + + // Peers list peers we are currently pubsubbing with + Peers(context.Context, ...options.PubSubPeersOption) ([]peer.ID, error) + + // Publish a message to a given pubsub topic + Publish(context.Context, string, []byte) error + + // Subscribe to messages on a given topic + Subscribe(context.Context, string, ...options.PubSubSubscribeOption) (PubSubSubscription, error) +} diff --git a/coreiface/routing.go b/coreiface/routing.go new file mode 100644 index 0000000000..a28ceb9e77 --- /dev/null +++ b/coreiface/routing.go @@ -0,0 +1,14 @@ +package iface + +import ( + "context" +) + +// RoutingAPI specifies the interface to the routing layer. +type RoutingAPI interface { + // Get retrieves the best value for a given key + Get(context.Context, string) ([]byte, error) + + // Put sets a value for a given key + Put(ctx context.Context, key string, value []byte) error +} diff --git a/coreiface/swarm.go b/coreiface/swarm.go new file mode 100644 index 0000000000..9aa5466ba4 --- /dev/null +++ b/coreiface/swarm.go @@ -0,0 +1,57 @@ +package iface + +import ( + "context" + "errors" + "time" + + "github.com/libp2p/go-libp2p/core/network" + "github.com/libp2p/go-libp2p/core/peer" + "github.com/libp2p/go-libp2p/core/protocol" + + ma "github.com/multiformats/go-multiaddr" +) + +var ( + ErrNotConnected = errors.New("not connected") + ErrConnNotFound = errors.New("conn not found") +) + +// ConnectionInfo contains information about a peer +type ConnectionInfo interface { + // ID returns PeerID + ID() peer.ID + + // Address returns the multiaddress via which we are connected with the peer + Address() ma.Multiaddr + + // Direction returns which way the connection was established + Direction() network.Direction + + // Latency returns last known round trip time to the peer + Latency() (time.Duration, error) + + // Streams returns list of streams established with the peer + Streams() ([]protocol.ID, error) +} + +// SwarmAPI specifies the interface to libp2p swarm +type SwarmAPI interface { + // Connect to a given peer + Connect(context.Context, peer.AddrInfo) error + + // Disconnect from a given address + Disconnect(context.Context, ma.Multiaddr) error + + // Peers returns the list of peers we are connected to + Peers(context.Context) ([]ConnectionInfo, error) + + // KnownAddrs returns the list of all addresses this node is aware of + KnownAddrs(context.Context) (map[peer.ID][]ma.Multiaddr, error) + + // LocalAddrs returns the list of announced listening addresses + LocalAddrs(context.Context) ([]ma.Multiaddr, error) + + // ListenAddrs returns the list of all listening addresses + ListenAddrs(context.Context) ([]ma.Multiaddr, error) +} diff --git a/coreiface/tests/api.go b/coreiface/tests/api.go new file mode 100644 index 0000000000..d26cda6875 --- /dev/null +++ b/coreiface/tests/api.go @@ -0,0 +1,97 @@ +package tests + +import ( + "context" + "errors" + "testing" + "time" + + coreiface "github.com/ipfs/go-libipfs/coreiface" +) + +var errAPINotImplemented = errors.New("api not implemented") + +func (tp *TestSuite) makeAPI(ctx context.Context) (coreiface.CoreAPI, error) { + api, err := tp.MakeAPISwarm(ctx, false, 1) + if err != nil { + return nil, err + } + + return api[0], nil +} + +type Provider interface { + // Make creates n nodes. fullIdentity set to false can be ignored + MakeAPISwarm(ctx context.Context, fullIdentity bool, n int) ([]coreiface.CoreAPI, error) +} + +func (tp *TestSuite) MakeAPISwarm(ctx context.Context, fullIdentity bool, n int) ([]coreiface.CoreAPI, error) { + if tp.apis != nil { + tp.apis <- 1 + go func() { + <-ctx.Done() + tp.apis <- -1 + }() + } + + return tp.Provider.MakeAPISwarm(ctx, fullIdentity, n) +} + +type TestSuite struct { + Provider + + apis chan int +} + +func TestApi(p Provider) func(t *testing.T) { + running := 1 + apis := make(chan int) + zeroRunning := make(chan struct{}) + go func() { + for i := range apis { + running += i + if running < 1 { + close(zeroRunning) + return + } + } + }() + + tp := &TestSuite{Provider: p, apis: apis} + + return func(t *testing.T) { + t.Run("Block", tp.TestBlock) + t.Run("Dag", tp.TestDag) + t.Run("Dht", tp.TestDht) + t.Run("Key", tp.TestKey) + t.Run("Name", tp.TestName) + t.Run("Object", tp.TestObject) + t.Run("Path", tp.TestPath) + t.Run("Pin", tp.TestPin) + t.Run("PubSub", tp.TestPubSub) + t.Run("Routing", tp.TestRouting) + t.Run("Unixfs", tp.TestUnixfs) + + apis <- -1 + t.Run("TestsCancelCtx", func(t *testing.T) { + select { + case <-zeroRunning: + case <-time.After(time.Second): + t.Errorf("%d test swarms(s) not closed", running) + } + }) + } +} + +func (tp *TestSuite) hasApi(t *testing.T, tf func(coreiface.CoreAPI) error) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + if err := tf(api); err != nil { + t.Fatal(api) + } +} diff --git a/coreiface/tests/block.go b/coreiface/tests/block.go new file mode 100644 index 0000000000..72bf56188f --- /dev/null +++ b/coreiface/tests/block.go @@ -0,0 +1,354 @@ +package tests + +import ( + "bytes" + "context" + "io" + "strings" + "testing" + + ipld "github.com/ipfs/go-ipld-format" + coreiface "github.com/ipfs/go-libipfs/coreiface" + opt "github.com/ipfs/go-libipfs/coreiface/options" + "github.com/ipfs/go-libipfs/coreiface/path" + + mh "github.com/multiformats/go-multihash" +) + +var ( + pbCidV0 = "QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN" // dag-pb + pbCid = "bafybeiffndsajwhk3lwjewwdxqntmjm4b5wxaaanokonsggenkbw6slwk4" // dag-pb + rawCid = "bafkreiffndsajwhk3lwjewwdxqntmjm4b5wxaaanokonsggenkbw6slwk4" // raw bytes + cborCid = "bafyreicnga62zhxnmnlt6ymq5hcbsg7gdhqdu6z4ehu3wpjhvqnflfy6nm" // dag-cbor + cborKCid = "bafyr2qgsohbwdlk7ajmmbb4lhoytmest4wdbe5xnexfvtxeatuyqqmwv3fgxp3pmhpc27gwey2cct56gloqefoqwcf3yqiqzsaqb7p4jefhcw" // dag-cbor keccak-512 +) + +// dag-pb +func pbBlock() io.Reader { + return bytes.NewReader([]byte{10, 12, 8, 2, 18, 6, 104, 101, 108, 108, 111, 10, 24, 6}) +} + +// dag-cbor +func cborBlock() io.Reader { + return bytes.NewReader([]byte{101, 72, 101, 108, 108, 111}) +} + +func (tp *TestSuite) TestBlock(t *testing.T) { + tp.hasApi(t, func(api coreiface.CoreAPI) error { + if api.Block() == nil { + return errAPINotImplemented + } + return nil + }) + + t.Run("TestBlockPut (get raw CIDv1)", tp.TestBlockPut) + t.Run("TestBlockPutCidCodec: dag-pb", tp.TestBlockPutCidCodecDagPb) + t.Run("TestBlockPutCidCodec: dag-cbor", tp.TestBlockPutCidCodecDagCbor) + t.Run("TestBlockPutFormat (legacy): cbor → dag-cbor", tp.TestBlockPutFormatDagCbor) + t.Run("TestBlockPutFormat (legacy): protobuf → dag-pb", tp.TestBlockPutFormatDagPb) + t.Run("TestBlockPutFormat (legacy): v0 → CIDv0", tp.TestBlockPutFormatV0) + t.Run("TestBlockPutHash", tp.TestBlockPutHash) + t.Run("TestBlockGet", tp.TestBlockGet) + t.Run("TestBlockRm", tp.TestBlockRm) + t.Run("TestBlockStat", tp.TestBlockStat) + t.Run("TestBlockPin", tp.TestBlockPin) +} + +// when no opts are passed, produced CID has 'raw' codec +func (tp *TestSuite) TestBlockPut(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + res, err := api.Block().Put(ctx, pbBlock()) + if err != nil { + t.Fatal(err) + } + + if res.Path().Cid().String() != rawCid { + t.Errorf("got wrong cid: %s", res.Path().Cid().String()) + } +} + +// Format is deprecated, it used invalid codec names. +// Confirm 'cbor' gets fixed to 'dag-cbor' +func (tp *TestSuite) TestBlockPutFormatDagCbor(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + res, err := api.Block().Put(ctx, cborBlock(), opt.Block.Format("cbor")) + if err != nil { + t.Fatal(err) + } + + if res.Path().Cid().String() != cborCid { + t.Errorf("got wrong cid: %s", res.Path().Cid().String()) + } +} + +// Format is deprecated, it used invalid codec names. +// Confirm 'protobuf' got fixed to 'dag-pb' +func (tp *TestSuite) TestBlockPutFormatDagPb(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + res, err := api.Block().Put(ctx, pbBlock(), opt.Block.Format("protobuf")) + if err != nil { + t.Fatal(err) + } + + if res.Path().Cid().String() != pbCid { + t.Errorf("got wrong cid: %s", res.Path().Cid().String()) + } +} + +// Format is deprecated, it used invalid codec names. +// Confirm fake codec 'v0' got fixed to CIDv0 (with implicit dag-pb codec) +func (tp *TestSuite) TestBlockPutFormatV0(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + res, err := api.Block().Put(ctx, pbBlock(), opt.Block.Format("v0")) + if err != nil { + t.Fatal(err) + } + + if res.Path().Cid().String() != pbCidV0 { + t.Errorf("got wrong cid: %s", res.Path().Cid().String()) + } +} + +func (tp *TestSuite) TestBlockPutCidCodecDagCbor(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + res, err := api.Block().Put(ctx, cborBlock(), opt.Block.CidCodec("dag-cbor")) + if err != nil { + t.Fatal(err) + } + + if res.Path().Cid().String() != cborCid { + t.Errorf("got wrong cid: %s", res.Path().Cid().String()) + } +} + +func (tp *TestSuite) TestBlockPutCidCodecDagPb(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + res, err := api.Block().Put(ctx, pbBlock(), opt.Block.CidCodec("dag-pb")) + if err != nil { + t.Fatal(err) + } + + if res.Path().Cid().String() != pbCid { + t.Errorf("got wrong cid: %s", res.Path().Cid().String()) + } +} + +func (tp *TestSuite) TestBlockPutHash(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + res, err := api.Block().Put( + ctx, + cborBlock(), + opt.Block.Hash(mh.KECCAK_512, -1), + opt.Block.CidCodec("dag-cbor"), + ) + if err != nil { + t.Fatal(err) + } + + if res.Path().Cid().String() != cborKCid { + t.Errorf("got wrong cid: %s", res.Path().Cid().String()) + } +} + +func (tp *TestSuite) TestBlockGet(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + res, err := api.Block().Put(ctx, strings.NewReader(`Hello`), opt.Block.Format("raw")) + if err != nil { + t.Fatal(err) + } + + r, err := api.Block().Get(ctx, res.Path()) + if err != nil { + t.Fatal(err) + } + + d, err := io.ReadAll(r) + if err != nil { + t.Fatal(err) + } + + if string(d) != "Hello" { + t.Error("didn't get correct data back") + } + + p := path.New("/ipfs/" + res.Path().Cid().String()) + + rp, err := api.ResolvePath(ctx, p) + if err != nil { + t.Fatal(err) + } + if rp.Cid().String() != res.Path().Cid().String() { + t.Error("paths didn't match") + } +} + +func (tp *TestSuite) TestBlockRm(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + res, err := api.Block().Put(ctx, strings.NewReader(`Hello`), opt.Block.Format("raw")) + if err != nil { + t.Fatal(err) + } + + r, err := api.Block().Get(ctx, res.Path()) + if err != nil { + t.Fatal(err) + } + + d, err := io.ReadAll(r) + if err != nil { + t.Fatal(err) + } + + if string(d) != "Hello" { + t.Error("didn't get correct data back") + } + + err = api.Block().Rm(ctx, res.Path()) + if err != nil { + t.Fatal(err) + } + + _, err = api.Block().Get(ctx, res.Path()) + if err == nil { + t.Fatal("expected err to exist") + } + if !ipld.IsNotFound(err) { + t.Errorf("unexpected error; %s", err.Error()) + } + + err = api.Block().Rm(ctx, res.Path()) + if err == nil { + t.Fatal("expected err to exist") + } + if !ipld.IsNotFound(err) { + t.Errorf("unexpected error; %s", err.Error()) + } + + err = api.Block().Rm(ctx, res.Path(), opt.Block.Force(true)) + if err != nil { + t.Fatal(err) + } +} + +func (tp *TestSuite) TestBlockStat(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + res, err := api.Block().Put(ctx, strings.NewReader(`Hello`), opt.Block.Format("raw")) + if err != nil { + t.Fatal(err) + } + + stat, err := api.Block().Stat(ctx, res.Path()) + if err != nil { + t.Fatal(err) + } + + if stat.Path().String() != res.Path().String() { + t.Error("paths don't match") + } + + if stat.Size() != len("Hello") { + t.Error("length doesn't match") + } +} + +func (tp *TestSuite) TestBlockPin(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + _, err = api.Block().Put(ctx, strings.NewReader(`Hello`), opt.Block.Format("raw")) + if err != nil { + t.Fatal(err) + } + + if pins, err := api.Pin().Ls(ctx); err != nil || len(pins) != 0 { + t.Fatal("expected 0 pins") + } + + res, err := api.Block().Put( + ctx, + strings.NewReader(`Hello`), + opt.Block.Pin(true), + opt.Block.Format("raw"), + ) + if err != nil { + t.Fatal(err) + } + + pins, err := accPins(api.Pin().Ls(ctx)) + if err != nil { + t.Fatal(err) + } + if len(pins) != 1 { + t.Fatal("expected 1 pin") + } + if pins[0].Type() != "recursive" { + t.Error("expected a recursive pin") + } + if pins[0].Path().String() != res.Path().String() { + t.Error("pin path didn't match") + } +} diff --git a/coreiface/tests/dag.go b/coreiface/tests/dag.go new file mode 100644 index 0000000000..2423bca039 --- /dev/null +++ b/coreiface/tests/dag.go @@ -0,0 +1,200 @@ +package tests + +import ( + "context" + "math" + gopath "path" + "strings" + "testing" + + path "github.com/ipfs/go-libipfs/coreiface/path" + + coreiface "github.com/ipfs/go-libipfs/coreiface" + + ipldcbor "github.com/ipfs/go-ipld-cbor" + ipld "github.com/ipfs/go-ipld-format" + mh "github.com/multiformats/go-multihash" +) + +func (tp *TestSuite) TestDag(t *testing.T) { + tp.hasApi(t, func(api coreiface.CoreAPI) error { + if api.Dag() == nil { + return errAPINotImplemented + } + return nil + }) + + t.Run("TestPut", tp.TestPut) + t.Run("TestPutWithHash", tp.TestPutWithHash) + t.Run("TestPath", tp.TestDagPath) + t.Run("TestTree", tp.TestTree) + t.Run("TestBatch", tp.TestBatch) +} + +var ( + treeExpected = map[string]struct{}{ + "a": {}, + "b": {}, + "c": {}, + "c/d": {}, + "c/e": {}, + } +) + +func (tp *TestSuite) TestPut(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + nd, err := ipldcbor.FromJSON(strings.NewReader(`"Hello"`), math.MaxUint64, -1) + if err != nil { + t.Fatal(err) + } + + err = api.Dag().Add(ctx, nd) + if err != nil { + t.Fatal(err) + } + + if nd.Cid().String() != "bafyreicnga62zhxnmnlt6ymq5hcbsg7gdhqdu6z4ehu3wpjhvqnflfy6nm" { + t.Errorf("got wrong cid: %s", nd.Cid().String()) + } +} + +func (tp *TestSuite) TestPutWithHash(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + nd, err := ipldcbor.FromJSON(strings.NewReader(`"Hello"`), mh.SHA3_256, -1) + if err != nil { + t.Fatal(err) + } + + err = api.Dag().Add(ctx, nd) + if err != nil { + t.Fatal(err) + } + + if nd.Cid().String() != "bafyrmifu7haikttpqqgc5ewvmp76z3z4ebp7h2ph4memw7dq4nt6btmxny" { + t.Errorf("got wrong cid: %s", nd.Cid().String()) + } +} + +func (tp *TestSuite) TestDagPath(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + snd, err := ipldcbor.FromJSON(strings.NewReader(`"foo"`), math.MaxUint64, -1) + if err != nil { + t.Fatal(err) + } + + err = api.Dag().Add(ctx, snd) + if err != nil { + t.Fatal(err) + } + + nd, err := ipldcbor.FromJSON(strings.NewReader(`{"lnk": {"/": "`+snd.Cid().String()+`"}}`), math.MaxUint64, -1) + if err != nil { + t.Fatal(err) + } + + err = api.Dag().Add(ctx, nd) + if err != nil { + t.Fatal(err) + } + + p := path.New(gopath.Join(nd.Cid().String(), "lnk")) + + rp, err := api.ResolvePath(ctx, p) + if err != nil { + t.Fatal(err) + } + + ndd, err := api.Dag().Get(ctx, rp.Cid()) + if err != nil { + t.Fatal(err) + } + + if ndd.Cid().String() != snd.Cid().String() { + t.Errorf("got unexpected cid %s, expected %s", ndd.Cid().String(), snd.Cid().String()) + } +} + +func (tp *TestSuite) TestTree(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + nd, err := ipldcbor.FromJSON(strings.NewReader(`{"a": 123, "b": "foo", "c": {"d": 321, "e": 111}}`), math.MaxUint64, -1) + if err != nil { + t.Fatal(err) + } + + err = api.Dag().Add(ctx, nd) + if err != nil { + t.Fatal(err) + } + + res, err := api.Dag().Get(ctx, nd.Cid()) + if err != nil { + t.Fatal(err) + } + + lst := res.Tree("", -1) + if len(lst) != len(treeExpected) { + t.Errorf("tree length of %d doesn't match expected %d", len(lst), len(treeExpected)) + } + + for _, ent := range lst { + if _, ok := treeExpected[ent]; !ok { + t.Errorf("unexpected tree entry %s", ent) + } + } +} + +func (tp *TestSuite) TestBatch(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + nd, err := ipldcbor.FromJSON(strings.NewReader(`"Hello"`), math.MaxUint64, -1) + if err != nil { + t.Fatal(err) + } + + if nd.Cid().String() != "bafyreicnga62zhxnmnlt6ymq5hcbsg7gdhqdu6z4ehu3wpjhvqnflfy6nm" { + t.Errorf("got wrong cid: %s", nd.Cid().String()) + } + + _, err = api.Dag().Get(ctx, nd.Cid()) + if err == nil || !strings.Contains(err.Error(), "not found") { + t.Fatal(err) + } + + if err := api.Dag().AddMany(ctx, []ipld.Node{nd}); err != nil { + t.Fatal(err) + } + + _, err = api.Dag().Get(ctx, nd.Cid()) + if err != nil { + t.Fatal(err) + } +} diff --git a/coreiface/tests/dht.go b/coreiface/tests/dht.go new file mode 100644 index 0000000000..d6def089b2 --- /dev/null +++ b/coreiface/tests/dht.go @@ -0,0 +1,166 @@ +package tests + +import ( + "context" + "io" + "testing" + "time" + + iface "github.com/ipfs/go-libipfs/coreiface" + "github.com/ipfs/go-libipfs/coreiface/options" +) + +func (tp *TestSuite) TestDht(t *testing.T) { + tp.hasApi(t, func(api iface.CoreAPI) error { + if api.Dht() == nil { + return errAPINotImplemented + } + return nil + }) + + t.Run("TestDhtFindPeer", tp.TestDhtFindPeer) + t.Run("TestDhtFindProviders", tp.TestDhtFindProviders) + t.Run("TestDhtProvide", tp.TestDhtProvide) +} + +func (tp *TestSuite) TestDhtFindPeer(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + apis, err := tp.MakeAPISwarm(ctx, true, 5) + if err != nil { + t.Fatal(err) + } + + self0, err := apis[0].Key().Self(ctx) + if err != nil { + t.Fatal(err) + } + + laddrs0, err := apis[0].Swarm().LocalAddrs(ctx) + if err != nil { + t.Fatal(err) + } + if len(laddrs0) != 1 { + t.Fatal("unexpected number of local addrs") + } + + time.Sleep(3 * time.Second) + + pi, err := apis[2].Dht().FindPeer(ctx, self0.ID()) + if err != nil { + t.Fatal(err) + } + + if pi.Addrs[0].String() != laddrs0[0].String() { + t.Errorf("got unexpected address from FindPeer: %s", pi.Addrs[0].String()) + } + + self2, err := apis[2].Key().Self(ctx) + if err != nil { + t.Fatal(err) + } + + pi, err = apis[1].Dht().FindPeer(ctx, self2.ID()) + if err != nil { + t.Fatal(err) + } + + laddrs2, err := apis[2].Swarm().LocalAddrs(ctx) + if err != nil { + t.Fatal(err) + } + if len(laddrs2) != 1 { + t.Fatal("unexpected number of local addrs") + } + + if pi.Addrs[0].String() != laddrs2[0].String() { + t.Errorf("got unexpected address from FindPeer: %s", pi.Addrs[0].String()) + } +} + +func (tp *TestSuite) TestDhtFindProviders(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + apis, err := tp.MakeAPISwarm(ctx, true, 5) + if err != nil { + t.Fatal(err) + } + + p, err := addTestObject(ctx, apis[0]) + if err != nil { + t.Fatal(err) + } + + time.Sleep(3 * time.Second) + + out, err := apis[2].Dht().FindProviders(ctx, p, options.Dht.NumProviders(1)) + if err != nil { + t.Fatal(err) + } + + provider := <-out + + self0, err := apis[0].Key().Self(ctx) + if err != nil { + t.Fatal(err) + } + + if provider.ID.String() != self0.ID().String() { + t.Errorf("got wrong provider: %s != %s", provider.ID.String(), self0.ID().String()) + } +} + +func (tp *TestSuite) TestDhtProvide(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + apis, err := tp.MakeAPISwarm(ctx, true, 5) + if err != nil { + t.Fatal(err) + } + + off0, err := apis[0].WithOptions(options.Api.Offline(true)) + if err != nil { + t.Fatal(err) + } + + s, err := off0.Block().Put(ctx, &io.LimitedReader{R: rnd, N: 4092}) + if err != nil { + t.Fatal(err) + } + + p := s.Path() + + time.Sleep(3 * time.Second) + + out, err := apis[2].Dht().FindProviders(ctx, p, options.Dht.NumProviders(1)) + if err != nil { + t.Fatal(err) + } + + _, ok := <-out + + if ok { + t.Fatal("did not expect to find any providers") + } + + self0, err := apis[0].Key().Self(ctx) + if err != nil { + t.Fatal(err) + } + + err = apis[0].Dht().Provide(ctx, p) + if err != nil { + t.Fatal(err) + } + + out, err = apis[2].Dht().FindProviders(ctx, p, options.Dht.NumProviders(1)) + if err != nil { + t.Fatal(err) + } + + provider := <-out + + if provider.ID.String() != self0.ID().String() { + t.Errorf("got wrong provider: %s != %s", provider.ID.String(), self0.ID().String()) + } +} diff --git a/coreiface/tests/key.go b/coreiface/tests/key.go new file mode 100644 index 0000000000..85f5f91e50 --- /dev/null +++ b/coreiface/tests/key.go @@ -0,0 +1,538 @@ +package tests + +import ( + "context" + "strings" + "testing" + + "github.com/ipfs/go-cid" + iface "github.com/ipfs/go-libipfs/coreiface" + opt "github.com/ipfs/go-libipfs/coreiface/options" + mbase "github.com/multiformats/go-multibase" +) + +func (tp *TestSuite) TestKey(t *testing.T) { + tp.hasApi(t, func(api iface.CoreAPI) error { + if api.Key() == nil { + return errAPINotImplemented + } + return nil + }) + + t.Run("TestListSelf", tp.TestListSelf) + t.Run("TestRenameSelf", tp.TestRenameSelf) + t.Run("TestRemoveSelf", tp.TestRemoveSelf) + t.Run("TestGenerate", tp.TestGenerate) + t.Run("TestGenerateSize", tp.TestGenerateSize) + t.Run("TestGenerateType", tp.TestGenerateType) + t.Run("TestGenerateExisting", tp.TestGenerateExisting) + t.Run("TestList", tp.TestList) + t.Run("TestRename", tp.TestRename) + t.Run("TestRenameToSelf", tp.TestRenameToSelf) + t.Run("TestRenameToSelfForce", tp.TestRenameToSelfForce) + t.Run("TestRenameOverwriteNoForce", tp.TestRenameOverwriteNoForce) + t.Run("TestRenameOverwrite", tp.TestRenameOverwrite) + t.Run("TestRenameSameNameNoForce", tp.TestRenameSameNameNoForce) + t.Run("TestRenameSameName", tp.TestRenameSameName) + t.Run("TestRemove", tp.TestRemove) +} + +func (tp *TestSuite) TestListSelf(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + return + } + + self, err := api.Key().Self(ctx) + if err != nil { + t.Fatal(err) + } + + keys, err := api.Key().List(ctx) + if err != nil { + t.Fatalf("failed to list keys: %s", err) + return + } + + if len(keys) != 1 { + t.Fatalf("there should be 1 key (self), got %d", len(keys)) + return + } + + if keys[0].Name() != "self" { + t.Errorf("expected the key to be called 'self', got '%s'", keys[0].Name()) + } + + if keys[0].Path().String() != "/ipns/"+iface.FormatKeyID(self.ID()) { + t.Errorf("expected the key to have path '/ipns/%s', got '%s'", iface.FormatKeyID(self.ID()), keys[0].Path().String()) + } +} + +func (tp *TestSuite) TestRenameSelf(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + return + } + + _, _, err = api.Key().Rename(ctx, "self", "foo") + if err == nil { + t.Error("expected error to not be nil") + } else { + if !strings.Contains(err.Error(), "cannot rename key with name 'self'") { + t.Fatalf("expected error 'cannot rename key with name 'self'', got '%s'", err.Error()) + } + } + + _, _, err = api.Key().Rename(ctx, "self", "foo", opt.Key.Force(true)) + if err == nil { + t.Error("expected error to not be nil") + } else { + if !strings.Contains(err.Error(), "cannot rename key with name 'self'") { + t.Fatalf("expected error 'cannot rename key with name 'self'', got '%s'", err.Error()) + } + } +} + +func (tp *TestSuite) TestRemoveSelf(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + return + } + + _, err = api.Key().Remove(ctx, "self") + if err == nil { + t.Error("expected error to not be nil") + } else { + if !strings.Contains(err.Error(), "cannot remove key with name 'self'") { + t.Fatalf("expected error 'cannot remove key with name 'self'', got '%s'", err.Error()) + } + } +} + +func (tp *TestSuite) TestGenerate(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + k, err := api.Key().Generate(ctx, "foo") + if err != nil { + t.Fatal(err) + return + } + + if k.Name() != "foo" { + t.Errorf("expected the key to be called 'foo', got '%s'", k.Name()) + } + + verifyIPNSPath(t, k.Path().String()) +} + +func verifyIPNSPath(t *testing.T, p string) bool { + t.Helper() + if !strings.HasPrefix(p, "/ipns/") { + t.Errorf("path %q does not look like an IPNS path", p) + return false + } + k := p[len("/ipns/"):] + c, err := cid.Decode(k) + if err != nil { + t.Errorf("failed to decode IPNS key %q (%v)", k, err) + return false + } + b36, err := c.StringOfBase(mbase.Base36) + if err != nil { + t.Fatalf("cid cannot format itself in b36") + return false + } + if b36 != k { + t.Errorf("IPNS key is not base36") + } + return true +} + +func (tp *TestSuite) TestGenerateSize(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + k, err := api.Key().Generate(ctx, "foo", opt.Key.Size(2048)) + if err != nil { + t.Fatal(err) + return + } + + if k.Name() != "foo" { + t.Errorf("expected the key to be called 'foo', got '%s'", k.Name()) + } + + verifyIPNSPath(t, k.Path().String()) +} + +func (tp *TestSuite) TestGenerateType(t *testing.T) { + t.Skip("disabled until libp2p/specs#111 is fixed") + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + k, err := api.Key().Generate(ctx, "bar", opt.Key.Type(opt.Ed25519Key)) + if err != nil { + t.Fatal(err) + return + } + + if k.Name() != "bar" { + t.Errorf("expected the key to be called 'foo', got '%s'", k.Name()) + } + + // Expected to be an inlined identity hash. + if !strings.HasPrefix(k.Path().String(), "/ipns/12") { + t.Errorf("expected the key to be prefixed with '/ipns/12', got '%s'", k.Path().String()) + } +} + +func (tp *TestSuite) TestGenerateExisting(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + _, err = api.Key().Generate(ctx, "foo") + if err != nil { + t.Fatal(err) + return + } + + _, err = api.Key().Generate(ctx, "foo") + if err == nil { + t.Error("expected error to not be nil") + } else { + if !strings.Contains(err.Error(), "key with name 'foo' already exists") { + t.Fatalf("expected error 'key with name 'foo' already exists', got '%s'", err.Error()) + } + } + + _, err = api.Key().Generate(ctx, "self") + if err == nil { + t.Error("expected error to not be nil") + } else { + if !strings.Contains(err.Error(), "cannot create key with name 'self'") { + t.Fatalf("expected error 'cannot create key with name 'self'', got '%s'", err.Error()) + } + } +} + +func (tp *TestSuite) TestList(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + _, err = api.Key().Generate(ctx, "foo") + if err != nil { + t.Fatal(err) + return + } + + l, err := api.Key().List(ctx) + if err != nil { + t.Fatal(err) + return + } + + if len(l) != 2 { + t.Fatalf("expected to get 2 keys, got %d", len(l)) + return + } + + if l[0].Name() != "self" { + t.Fatalf("expected key 0 to be called 'self', got '%s'", l[0].Name()) + return + } + + if l[1].Name() != "foo" { + t.Fatalf("expected key 1 to be called 'foo', got '%s'", l[1].Name()) + return + } + + verifyIPNSPath(t, l[0].Path().String()) + verifyIPNSPath(t, l[1].Path().String()) +} + +func (tp *TestSuite) TestRename(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + _, err = api.Key().Generate(ctx, "foo") + if err != nil { + t.Fatal(err) + return + } + + k, overwrote, err := api.Key().Rename(ctx, "foo", "bar") + if err != nil { + t.Fatal(err) + return + } + + if overwrote { + t.Error("overwrote should be false") + } + + if k.Name() != "bar" { + t.Errorf("returned key should be called 'bar', got '%s'", k.Name()) + } +} + +func (tp *TestSuite) TestRenameToSelf(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + _, err = api.Key().Generate(ctx, "foo") + if err != nil { + t.Fatal(err) + return + } + + _, _, err = api.Key().Rename(ctx, "foo", "self") + if err == nil { + t.Error("expected error to not be nil") + } else { + if !strings.Contains(err.Error(), "cannot overwrite key with name 'self'") { + t.Fatalf("expected error 'cannot overwrite key with name 'self'', got '%s'", err.Error()) + } + } +} + +func (tp *TestSuite) TestRenameToSelfForce(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + _, err = api.Key().Generate(ctx, "foo") + if err != nil { + t.Fatal(err) + return + } + + _, _, err = api.Key().Rename(ctx, "foo", "self", opt.Key.Force(true)) + if err == nil { + t.Error("expected error to not be nil") + } else { + if !strings.Contains(err.Error(), "cannot overwrite key with name 'self'") { + t.Fatalf("expected error 'cannot overwrite key with name 'self'', got '%s'", err.Error()) + } + } +} + +func (tp *TestSuite) TestRenameOverwriteNoForce(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + _, err = api.Key().Generate(ctx, "foo") + if err != nil { + t.Fatal(err) + return + } + + _, err = api.Key().Generate(ctx, "bar") + if err != nil { + t.Fatal(err) + return + } + + _, _, err = api.Key().Rename(ctx, "foo", "bar") + if err == nil { + t.Error("expected error to not be nil") + } else { + if !strings.Contains(err.Error(), "key by that name already exists, refusing to overwrite") { + t.Fatalf("expected error 'key by that name already exists, refusing to overwrite', got '%s'", err.Error()) + } + } +} + +func (tp *TestSuite) TestRenameOverwrite(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + kfoo, err := api.Key().Generate(ctx, "foo") + if err != nil { + t.Fatal(err) + return + } + + _, err = api.Key().Generate(ctx, "bar") + if err != nil { + t.Fatal(err) + return + } + + k, overwrote, err := api.Key().Rename(ctx, "foo", "bar", opt.Key.Force(true)) + if err != nil { + t.Fatal(err) + return + } + + if !overwrote { + t.Error("overwrote should be true") + } + + if k.Name() != "bar" { + t.Errorf("returned key should be called 'bar', got '%s'", k.Name()) + } + + if k.Path().String() != kfoo.Path().String() { + t.Errorf("k and kfoo should have equal paths, '%s'!='%s'", k.Path().String(), kfoo.Path().String()) + } +} + +func (tp *TestSuite) TestRenameSameNameNoForce(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + _, err = api.Key().Generate(ctx, "foo") + if err != nil { + t.Fatal(err) + return + } + + k, overwrote, err := api.Key().Rename(ctx, "foo", "foo") + if err != nil { + t.Fatal(err) + return + } + + if overwrote { + t.Error("overwrote should be false") + } + + if k.Name() != "foo" { + t.Errorf("returned key should be called 'foo', got '%s'", k.Name()) + } +} + +func (tp *TestSuite) TestRenameSameName(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + _, err = api.Key().Generate(ctx, "foo") + if err != nil { + t.Fatal(err) + return + } + + k, overwrote, err := api.Key().Rename(ctx, "foo", "foo", opt.Key.Force(true)) + if err != nil { + t.Fatal(err) + return + } + + if overwrote { + t.Error("overwrote should be false") + } + + if k.Name() != "foo" { + t.Errorf("returned key should be called 'foo', got '%s'", k.Name()) + } +} + +func (tp *TestSuite) TestRemove(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + k, err := api.Key().Generate(ctx, "foo") + if err != nil { + t.Fatal(err) + return + } + + l, err := api.Key().List(ctx) + if err != nil { + t.Fatal(err) + return + } + + if len(l) != 2 { + t.Fatalf("expected to get 2 keys, got %d", len(l)) + return + } + + p, err := api.Key().Remove(ctx, "foo") + if err != nil { + t.Fatal(err) + return + } + + if k.Path().String() != p.Path().String() { + t.Errorf("k and p should have equal paths, '%s'!='%s'", k.Path().String(), p.Path().String()) + } + + l, err = api.Key().List(ctx) + if err != nil { + t.Fatal(err) + return + } + + if len(l) != 1 { + t.Fatalf("expected to get 1 key, got %d", len(l)) + return + } + + if l[0].Name() != "self" { + t.Errorf("expected the key to be called 'self', got '%s'", l[0].Name()) + } +} diff --git a/coreiface/tests/name.go b/coreiface/tests/name.go new file mode 100644 index 0000000000..f8ec7c52ff --- /dev/null +++ b/coreiface/tests/name.go @@ -0,0 +1,274 @@ +package tests + +import ( + "context" + "io" + "math/rand" + gopath "path" + "testing" + "time" + + path "github.com/ipfs/go-libipfs/coreiface/path" + + "github.com/ipfs/go-libipfs/files" + + coreiface "github.com/ipfs/go-libipfs/coreiface" + opt "github.com/ipfs/go-libipfs/coreiface/options" +) + +func (tp *TestSuite) TestName(t *testing.T) { + tp.hasApi(t, func(api coreiface.CoreAPI) error { + if api.Name() == nil { + return errAPINotImplemented + } + return nil + }) + + t.Run("TestPublishResolve", tp.TestPublishResolve) + t.Run("TestBasicPublishResolveKey", tp.TestBasicPublishResolveKey) + t.Run("TestBasicPublishResolveTimeout", tp.TestBasicPublishResolveTimeout) +} + +var rnd = rand.New(rand.NewSource(0x62796532303137)) + +func addTestObject(ctx context.Context, api coreiface.CoreAPI) (path.Path, error) { + return api.Unixfs().Add(ctx, files.NewReaderFile(&io.LimitedReader{R: rnd, N: 4092})) +} + +func appendPath(p path.Path, sub string) path.Path { + return path.New(gopath.Join(p.String(), sub)) +} + +func (tp *TestSuite) TestPublishResolve(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + init := func() (coreiface.CoreAPI, path.Path) { + apis, err := tp.MakeAPISwarm(ctx, true, 5) + if err != nil { + t.Fatal(err) + return nil, nil + } + api := apis[0] + + p, err := addTestObject(ctx, api) + if err != nil { + t.Fatal(err) + return nil, nil + } + return api, p + } + run := func(t *testing.T, ropts []opt.NameResolveOption) { + t.Run("basic", func(t *testing.T) { + api, p := init() + e, err := api.Name().Publish(ctx, p) + if err != nil { + t.Fatal(err) + } + + self, err := api.Key().Self(ctx) + if err != nil { + t.Fatal(err) + } + + if e.Name() != coreiface.FormatKeyID(self.ID()) { + t.Errorf("expected e.Name to equal '%s', got '%s'", coreiface.FormatKeyID(self.ID()), e.Name()) + } + + if e.Value().String() != p.String() { + t.Errorf("expected paths to match, '%s'!='%s'", e.Value().String(), p.String()) + } + + resPath, err := api.Name().Resolve(ctx, e.Name(), ropts...) + if err != nil { + t.Fatal(err) + } + + if resPath.String() != p.String() { + t.Errorf("expected paths to match, '%s'!='%s'", resPath.String(), p.String()) + } + }) + + t.Run("publishPath", func(t *testing.T) { + api, p := init() + e, err := api.Name().Publish(ctx, appendPath(p, "/test")) + if err != nil { + t.Fatal(err) + } + + self, err := api.Key().Self(ctx) + if err != nil { + t.Fatal(err) + } + + if e.Name() != coreiface.FormatKeyID(self.ID()) { + t.Errorf("expected e.Name to equal '%s', got '%s'", coreiface.FormatKeyID(self.ID()), e.Name()) + } + + if e.Value().String() != p.String()+"/test" { + t.Errorf("expected paths to match, '%s'!='%s'", e.Value().String(), p.String()) + } + + resPath, err := api.Name().Resolve(ctx, e.Name(), ropts...) + if err != nil { + t.Fatal(err) + } + + if resPath.String() != p.String()+"/test" { + t.Errorf("expected paths to match, '%s'!='%s'", resPath.String(), p.String()+"/test") + } + }) + + t.Run("revolvePath", func(t *testing.T) { + api, p := init() + e, err := api.Name().Publish(ctx, p) + if err != nil { + t.Fatal(err) + } + + self, err := api.Key().Self(ctx) + if err != nil { + t.Fatal(err) + } + + if e.Name() != coreiface.FormatKeyID(self.ID()) { + t.Errorf("expected e.Name to equal '%s', got '%s'", coreiface.FormatKeyID(self.ID()), e.Name()) + } + + if e.Value().String() != p.String() { + t.Errorf("expected paths to match, '%s'!='%s'", e.Value().String(), p.String()) + } + + resPath, err := api.Name().Resolve(ctx, e.Name()+"/test", ropts...) + if err != nil { + t.Fatal(err) + } + + if resPath.String() != p.String()+"/test" { + t.Errorf("expected paths to match, '%s'!='%s'", resPath.String(), p.String()+"/test") + } + }) + + t.Run("publishRevolvePath", func(t *testing.T) { + api, p := init() + e, err := api.Name().Publish(ctx, appendPath(p, "/a")) + if err != nil { + t.Fatal(err) + } + + self, err := api.Key().Self(ctx) + if err != nil { + t.Fatal(err) + } + + if e.Name() != coreiface.FormatKeyID(self.ID()) { + t.Errorf("expected e.Name to equal '%s', got '%s'", coreiface.FormatKeyID(self.ID()), e.Name()) + } + + if e.Value().String() != p.String()+"/a" { + t.Errorf("expected paths to match, '%s'!='%s'", e.Value().String(), p.String()) + } + + resPath, err := api.Name().Resolve(ctx, e.Name()+"/b", ropts...) + if err != nil { + t.Fatal(err) + } + + if resPath.String() != p.String()+"/a/b" { + t.Errorf("expected paths to match, '%s'!='%s'", resPath.String(), p.String()+"/a/b") + } + }) + } + + t.Run("default", func(t *testing.T) { + run(t, []opt.NameResolveOption{}) + }) + + t.Run("nocache", func(t *testing.T) { + run(t, []opt.NameResolveOption{opt.Name.Cache(false)}) + }) +} + +func (tp *TestSuite) TestBasicPublishResolveKey(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + apis, err := tp.MakeAPISwarm(ctx, true, 5) + if err != nil { + t.Fatal(err) + } + api := apis[0] + + k, err := api.Key().Generate(ctx, "foo") + if err != nil { + t.Fatal(err) + } + + p, err := addTestObject(ctx, api) + if err != nil { + t.Fatal(err) + } + + e, err := api.Name().Publish(ctx, p, opt.Name.Key(k.Name())) + if err != nil { + t.Fatal(err) + } + + if e.Name() != coreiface.FormatKey(k) { + t.Errorf("expected e.Name to equal %s, got '%s'", e.Name(), coreiface.FormatKey(k)) + } + + if e.Value().String() != p.String() { + t.Errorf("expected paths to match, '%s'!='%s'", e.Value().String(), p.String()) + } + + resPath, err := api.Name().Resolve(ctx, e.Name()) + if err != nil { + t.Fatal(err) + } + + if resPath.String() != p.String() { + t.Errorf("expected paths to match, '%s'!='%s'", resPath.String(), p.String()) + } +} + +func (tp *TestSuite) TestBasicPublishResolveTimeout(t *testing.T) { + t.Skip("ValidTime doesn't appear to work at this time resolution") + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + apis, err := tp.MakeAPISwarm(ctx, true, 5) + if err != nil { + t.Fatal(err) + } + api := apis[0] + p, err := addTestObject(ctx, api) + if err != nil { + t.Fatal(err) + } + + e, err := api.Name().Publish(ctx, p, opt.Name.ValidTime(time.Millisecond*100)) + if err != nil { + t.Fatal(err) + } + + self, err := api.Key().Self(ctx) + if err != nil { + t.Fatal(err) + } + + if e.Name() != coreiface.FormatKeyID(self.ID()) { + t.Errorf("expected e.Name to equal '%s', got '%s'", coreiface.FormatKeyID(self.ID()), e.Name()) + } + + if e.Value().String() != p.String() { + t.Errorf("expected paths to match, '%s'!='%s'", e.Value().String(), p.String()) + } + + time.Sleep(time.Second) + + _, err = api.Name().Resolve(ctx, e.Name()) + if err == nil { + t.Fatal("Expected an error") + } +} + +//TODO: When swarm api is created, add multinode tests diff --git a/coreiface/tests/object.go b/coreiface/tests/object.go new file mode 100644 index 0000000000..fe022461ce --- /dev/null +++ b/coreiface/tests/object.go @@ -0,0 +1,467 @@ +package tests + +import ( + "bytes" + "context" + "encoding/hex" + "io" + "strings" + "testing" + + iface "github.com/ipfs/go-libipfs/coreiface" + opt "github.com/ipfs/go-libipfs/coreiface/options" +) + +func (tp *TestSuite) TestObject(t *testing.T) { + tp.hasApi(t, func(api iface.CoreAPI) error { + if api.Object() == nil { + return errAPINotImplemented + } + return nil + }) + + t.Run("TestNew", tp.TestNew) + t.Run("TestObjectPut", tp.TestObjectPut) + t.Run("TestObjectGet", tp.TestObjectGet) + t.Run("TestObjectData", tp.TestObjectData) + t.Run("TestObjectLinks", tp.TestObjectLinks) + t.Run("TestObjectStat", tp.TestObjectStat) + t.Run("TestObjectAddLink", tp.TestObjectAddLink) + t.Run("TestObjectAddLinkCreate", tp.TestObjectAddLinkCreate) + t.Run("TestObjectRmLink", tp.TestObjectRmLink) + t.Run("TestObjectAddData", tp.TestObjectAddData) + t.Run("TestObjectSetData", tp.TestObjectSetData) + t.Run("TestDiffTest", tp.TestDiffTest) +} + +func (tp *TestSuite) TestNew(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + emptyNode, err := api.Object().New(ctx) + if err != nil { + t.Fatal(err) + } + + dirNode, err := api.Object().New(ctx, opt.Object.Type("unixfs-dir")) + if err != nil { + t.Fatal(err) + } + + if emptyNode.String() != "QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n" { + t.Errorf("Unexpected emptyNode path: %s", emptyNode.String()) + } + + if dirNode.String() != "QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn" { + t.Errorf("Unexpected dirNode path: %s", dirNode.String()) + } +} + +func (tp *TestSuite) TestObjectPut(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + p1, err := api.Object().Put(ctx, strings.NewReader(`{"Data":"foo"}`)) + if err != nil { + t.Fatal(err) + } + + p2, err := api.Object().Put(ctx, strings.NewReader(`{"Data":"YmFy"}`), opt.Object.DataType("base64")) //bar + if err != nil { + t.Fatal(err) + } + + pbBytes, err := hex.DecodeString("0a0362617a") + if err != nil { + t.Fatal(err) + } + + p3, err := api.Object().Put(ctx, bytes.NewReader(pbBytes), opt.Object.InputEnc("protobuf")) + if err != nil { + t.Fatal(err) + } + + if p1.String() != "/ipfs/QmQeGyS87nyijii7kFt1zbe4n2PsXTFimzsdxyE9qh9TST" { + t.Errorf("unexpected path: %s", p1.String()) + } + + if p2.String() != "/ipfs/QmNeYRbCibmaMMK6Du6ChfServcLqFvLJF76PzzF76SPrZ" { + t.Errorf("unexpected path: %s", p2.String()) + } + + if p3.String() != "/ipfs/QmZreR7M2t7bFXAdb1V5FtQhjk4t36GnrvueLJowJbQM9m" { + t.Errorf("unexpected path: %s", p3.String()) + } +} + +func (tp *TestSuite) TestObjectGet(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + p1, err := api.Object().Put(ctx, strings.NewReader(`{"Data":"foo"}`)) + if err != nil { + t.Fatal(err) + } + + nd, err := api.Object().Get(ctx, p1) + if err != nil { + t.Fatal(err) + } + + if string(nd.RawData()[len(nd.RawData())-3:]) != "foo" { + t.Fatal("got non-matching data") + } +} + +func (tp *TestSuite) TestObjectData(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + p1, err := api.Object().Put(ctx, strings.NewReader(`{"Data":"foo"}`)) + if err != nil { + t.Fatal(err) + } + + r, err := api.Object().Data(ctx, p1) + if err != nil { + t.Fatal(err) + } + + data, err := io.ReadAll(r) + if err != nil { + t.Fatal(err) + } + + if string(data) != "foo" { + t.Fatal("got non-matching data") + } +} + +func (tp *TestSuite) TestObjectLinks(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + p1, err := api.Object().Put(ctx, strings.NewReader(`{"Data":"foo"}`)) + if err != nil { + t.Fatal(err) + } + + p2, err := api.Object().Put(ctx, strings.NewReader(`{"Links":[{"Name":"bar", "Hash":"`+p1.Cid().String()+`"}]}`)) + if err != nil { + t.Fatal(err) + } + + links, err := api.Object().Links(ctx, p2) + if err != nil { + t.Fatal(err) + } + + if len(links) != 1 { + t.Errorf("unexpected number of links: %d", len(links)) + } + + if links[0].Cid.String() != p1.Cid().String() { + t.Fatal("cids didn't batch") + } + + if links[0].Name != "bar" { + t.Fatal("unexpected link name") + } +} + +func (tp *TestSuite) TestObjectStat(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + p1, err := api.Object().Put(ctx, strings.NewReader(`{"Data":"foo"}`)) + if err != nil { + t.Fatal(err) + } + + p2, err := api.Object().Put(ctx, strings.NewReader(`{"Data":"bazz", "Links":[{"Name":"bar", "Hash":"`+p1.Cid().String()+`", "Size":3}]}`)) + if err != nil { + t.Fatal(err) + } + + stat, err := api.Object().Stat(ctx, p2) + if err != nil { + t.Fatal(err) + } + + if stat.Cid.String() != p2.Cid().String() { + t.Error("unexpected stat.Cid") + } + + if stat.NumLinks != 1 { + t.Errorf("unexpected stat.NumLinks") + } + + if stat.BlockSize != 51 { + t.Error("unexpected stat.BlockSize") + } + + if stat.LinksSize != 47 { + t.Errorf("unexpected stat.LinksSize: %d", stat.LinksSize) + } + + if stat.DataSize != 4 { + t.Error("unexpected stat.DataSize") + } + + if stat.CumulativeSize != 54 { + t.Error("unexpected stat.DataSize") + } +} + +func (tp *TestSuite) TestObjectAddLink(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + p1, err := api.Object().Put(ctx, strings.NewReader(`{"Data":"foo"}`)) + if err != nil { + t.Fatal(err) + } + + p2, err := api.Object().Put(ctx, strings.NewReader(`{"Data":"bazz", "Links":[{"Name":"bar", "Hash":"`+p1.Cid().String()+`", "Size":3}]}`)) + if err != nil { + t.Fatal(err) + } + + p3, err := api.Object().AddLink(ctx, p2, "abc", p2) + if err != nil { + t.Fatal(err) + } + + links, err := api.Object().Links(ctx, p3) + if err != nil { + t.Fatal(err) + } + + if len(links) != 2 { + t.Errorf("unexpected number of links: %d", len(links)) + } + + if links[0].Name != "abc" { + t.Errorf("unexpected link 0 name: %s", links[0].Name) + } + + if links[1].Name != "bar" { + t.Errorf("unexpected link 1 name: %s", links[1].Name) + } +} + +func (tp *TestSuite) TestObjectAddLinkCreate(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + p1, err := api.Object().Put(ctx, strings.NewReader(`{"Data":"foo"}`)) + if err != nil { + t.Fatal(err) + } + + p2, err := api.Object().Put(ctx, strings.NewReader(`{"Data":"bazz", "Links":[{"Name":"bar", "Hash":"`+p1.Cid().String()+`", "Size":3}]}`)) + if err != nil { + t.Fatal(err) + } + + _, err = api.Object().AddLink(ctx, p2, "abc/d", p2) + if err == nil { + t.Fatal("expected an error") + } + if !strings.Contains(err.Error(), "no link by that name") { + t.Fatalf("unexpected error: %s", err.Error()) + } + + p3, err := api.Object().AddLink(ctx, p2, "abc/d", p2, opt.Object.Create(true)) + if err != nil { + t.Fatal(err) + } + + links, err := api.Object().Links(ctx, p3) + if err != nil { + t.Fatal(err) + } + + if len(links) != 2 { + t.Errorf("unexpected number of links: %d", len(links)) + } + + if links[0].Name != "abc" { + t.Errorf("unexpected link 0 name: %s", links[0].Name) + } + + if links[1].Name != "bar" { + t.Errorf("unexpected link 1 name: %s", links[1].Name) + } +} + +func (tp *TestSuite) TestObjectRmLink(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + p1, err := api.Object().Put(ctx, strings.NewReader(`{"Data":"foo"}`)) + if err != nil { + t.Fatal(err) + } + + p2, err := api.Object().Put(ctx, strings.NewReader(`{"Data":"bazz", "Links":[{"Name":"bar", "Hash":"`+p1.Cid().String()+`", "Size":3}]}`)) + if err != nil { + t.Fatal(err) + } + + p3, err := api.Object().RmLink(ctx, p2, "bar") + if err != nil { + t.Fatal(err) + } + + links, err := api.Object().Links(ctx, p3) + if err != nil { + t.Fatal(err) + } + + if len(links) != 0 { + t.Errorf("unexpected number of links: %d", len(links)) + } +} + +func (tp *TestSuite) TestObjectAddData(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + p1, err := api.Object().Put(ctx, strings.NewReader(`{"Data":"foo"}`)) + if err != nil { + t.Fatal(err) + } + + p2, err := api.Object().AppendData(ctx, p1, strings.NewReader("bar")) + if err != nil { + t.Fatal(err) + } + + r, err := api.Object().Data(ctx, p2) + if err != nil { + t.Fatal(err) + } + + data, err := io.ReadAll(r) + if err != nil { + t.Fatal(err) + } + + if string(data) != "foobar" { + t.Error("unexpected data") + } +} + +func (tp *TestSuite) TestObjectSetData(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + p1, err := api.Object().Put(ctx, strings.NewReader(`{"Data":"foo"}`)) + if err != nil { + t.Fatal(err) + } + + p2, err := api.Object().SetData(ctx, p1, strings.NewReader("bar")) + if err != nil { + t.Fatal(err) + } + + r, err := api.Object().Data(ctx, p2) + if err != nil { + t.Fatal(err) + } + + data, err := io.ReadAll(r) + if err != nil { + t.Fatal(err) + } + + if string(data) != "bar" { + t.Error("unexpected data") + } +} + +func (tp *TestSuite) TestDiffTest(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + p1, err := api.Object().Put(ctx, strings.NewReader(`{"Data":"foo"}`)) + if err != nil { + t.Fatal(err) + } + + p2, err := api.Object().Put(ctx, strings.NewReader(`{"Data":"bar"}`)) + if err != nil { + t.Fatal(err) + } + + changes, err := api.Object().Diff(ctx, p1, p2) + if err != nil { + t.Fatal(err) + } + + if len(changes) != 1 { + t.Fatal("unexpected changes len") + } + + if changes[0].Type != iface.DiffMod { + t.Fatal("unexpected change type") + } + + if changes[0].Before.String() != p1.String() { + t.Fatal("unexpected before path") + } + + if changes[0].After.String() != p2.String() { + t.Fatal("unexpected before path") + } +} diff --git a/coreiface/tests/path.go b/coreiface/tests/path.go new file mode 100644 index 0000000000..15fbc40f0f --- /dev/null +++ b/coreiface/tests/path.go @@ -0,0 +1,197 @@ +package tests + +import ( + "context" + "math" + "strings" + "testing" + + "github.com/ipfs/go-libipfs/coreiface/path" + + "github.com/ipfs/go-libipfs/coreiface/options" + + ipldcbor "github.com/ipfs/go-ipld-cbor" +) + +func (tp *TestSuite) TestPath(t *testing.T) { + t.Run("TestMutablePath", tp.TestMutablePath) + t.Run("TestPathRemainder", tp.TestPathRemainder) + t.Run("TestEmptyPathRemainder", tp.TestEmptyPathRemainder) + t.Run("TestInvalidPathRemainder", tp.TestInvalidPathRemainder) + t.Run("TestPathRoot", tp.TestPathRoot) + t.Run("TestPathJoin", tp.TestPathJoin) +} + +func (tp *TestSuite) TestMutablePath(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + blk, err := api.Block().Put(ctx, strings.NewReader(`foo`)) + if err != nil { + t.Fatal(err) + } + + if blk.Path().Mutable() { + t.Error("expected /ipld path to be immutable") + } + + // get self /ipns path + + if api.Key() == nil { + t.Fatal(".Key not implemented") + } + + keys, err := api.Key().List(ctx) + if err != nil { + t.Fatal(err) + } + + if !keys[0].Path().Mutable() { + t.Error("expected self /ipns path to be mutable") + } +} + +func (tp *TestSuite) TestPathRemainder(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + if api.Dag() == nil { + t.Fatal(".Dag not implemented") + } + + nd, err := ipldcbor.FromJSON(strings.NewReader(`{"foo": {"bar": "baz"}}`), math.MaxUint64, -1) + if err != nil { + t.Fatal(err) + } + + if err := api.Dag().Add(ctx, nd); err != nil { + t.Fatal(err) + } + + rp1, err := api.ResolvePath(ctx, path.New(nd.String()+"/foo/bar")) + if err != nil { + t.Fatal(err) + } + + if rp1.Remainder() != "foo/bar" { + t.Error("expected to get path remainder") + } +} + +func (tp *TestSuite) TestEmptyPathRemainder(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + if api.Dag() == nil { + t.Fatal(".Dag not implemented") + } + + nd, err := ipldcbor.FromJSON(strings.NewReader(`{"foo": {"bar": "baz"}}`), math.MaxUint64, -1) + if err != nil { + t.Fatal(err) + } + + if err := api.Dag().Add(ctx, nd); err != nil { + t.Fatal(err) + } + + rp1, err := api.ResolvePath(ctx, path.New(nd.Cid().String())) + if err != nil { + t.Fatal(err) + } + + if rp1.Remainder() != "" { + t.Error("expected the resolved path to not have a remainder") + } +} + +func (tp *TestSuite) TestInvalidPathRemainder(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + if api.Dag() == nil { + t.Fatal(".Dag not implemented") + } + + nd, err := ipldcbor.FromJSON(strings.NewReader(`{"foo": {"bar": "baz"}}`), math.MaxUint64, -1) + if err != nil { + t.Fatal(err) + } + + if err := api.Dag().Add(ctx, nd); err != nil { + t.Fatal(err) + } + + _, err = api.ResolvePath(ctx, path.New("/ipld/"+nd.Cid().String()+"/bar/baz")) + if err == nil || !strings.Contains(err.Error(), `no link named "bar"`) { + t.Fatalf("unexpected error: %s", err) + } +} + +func (tp *TestSuite) TestPathRoot(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + if api.Block() == nil { + t.Fatal(".Block not implemented") + } + + blk, err := api.Block().Put(ctx, strings.NewReader(`foo`), options.Block.Format("raw")) + if err != nil { + t.Fatal(err) + } + + if api.Dag() == nil { + t.Fatal(".Dag not implemented") + } + + nd, err := ipldcbor.FromJSON(strings.NewReader(`{"foo": {"/": "`+blk.Path().Cid().String()+`"}}`), math.MaxUint64, -1) + if err != nil { + t.Fatal(err) + } + + if err := api.Dag().Add(ctx, nd); err != nil { + t.Fatal(err) + } + + rp, err := api.ResolvePath(ctx, path.New("/ipld/"+nd.Cid().String()+"/foo")) + if err != nil { + t.Fatal(err) + } + + if rp.Root().String() != nd.Cid().String() { + t.Error("unexpected path root") + } + + if rp.Cid().String() != blk.Path().Cid().String() { + t.Error("unexpected path cid") + } +} + +func (tp *TestSuite) TestPathJoin(t *testing.T) { + p1 := path.New("/ipfs/QmYNmQKp6SuaVrpgWRsPTgCQCnpxUYGq76YEKBXuj2N4H6/bar/baz") + + if path.Join(p1, "foo").String() != "/ipfs/QmYNmQKp6SuaVrpgWRsPTgCQCnpxUYGq76YEKBXuj2N4H6/bar/baz/foo" { + t.Error("unexpected path") + } +} diff --git a/coreiface/tests/pin.go b/coreiface/tests/pin.go new file mode 100644 index 0000000000..c271db431d --- /dev/null +++ b/coreiface/tests/pin.go @@ -0,0 +1,601 @@ +package tests + +import ( + "context" + "math" + "strings" + "testing" + + iface "github.com/ipfs/go-libipfs/coreiface" + opt "github.com/ipfs/go-libipfs/coreiface/options" + "github.com/ipfs/go-libipfs/coreiface/path" + + "github.com/ipfs/go-cid" + ipldcbor "github.com/ipfs/go-ipld-cbor" + ipld "github.com/ipfs/go-ipld-format" +) + +func (tp *TestSuite) TestPin(t *testing.T) { + tp.hasApi(t, func(api iface.CoreAPI) error { + if api.Pin() == nil { + return errAPINotImplemented + } + return nil + }) + + t.Run("TestPinAdd", tp.TestPinAdd) + t.Run("TestPinSimple", tp.TestPinSimple) + t.Run("TestPinRecursive", tp.TestPinRecursive) + t.Run("TestPinLsIndirect", tp.TestPinLsIndirect) + t.Run("TestPinLsPrecedence", tp.TestPinLsPrecedence) + t.Run("TestPinIsPinned", tp.TestPinIsPinned) +} + +func (tp *TestSuite) TestPinAdd(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + p, err := api.Unixfs().Add(ctx, strFile("foo")()) + if err != nil { + t.Fatal(err) + } + + err = api.Pin().Add(ctx, p) + if err != nil { + t.Fatal(err) + } +} + +func (tp *TestSuite) TestPinSimple(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + p, err := api.Unixfs().Add(ctx, strFile("foo")()) + if err != nil { + t.Fatal(err) + } + + err = api.Pin().Add(ctx, p) + if err != nil { + t.Fatal(err) + } + + list, err := accPins(api.Pin().Ls(ctx)) + if err != nil { + t.Fatal(err) + } + + if len(list) != 1 { + t.Errorf("unexpected pin list len: %d", len(list)) + } + + if list[0].Path().Cid().String() != p.Cid().String() { + t.Error("paths don't match") + } + + if list[0].Type() != "recursive" { + t.Error("unexpected pin type") + } + + assertIsPinned(t, ctx, api, p, "recursive") + + err = api.Pin().Rm(ctx, p) + if err != nil { + t.Fatal(err) + } + + list, err = accPins(api.Pin().Ls(ctx)) + if err != nil { + t.Fatal(err) + } + + if len(list) != 0 { + t.Errorf("unexpected pin list len: %d", len(list)) + } +} + +func (tp *TestSuite) TestPinRecursive(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + p0, err := api.Unixfs().Add(ctx, strFile("foo")()) + if err != nil { + t.Fatal(err) + } + + p1, err := api.Unixfs().Add(ctx, strFile("bar")()) + if err != nil { + t.Fatal(err) + } + + nd2, err := ipldcbor.FromJSON(strings.NewReader(`{"lnk": {"/": "`+p0.Cid().String()+`"}}`), math.MaxUint64, -1) + if err != nil { + t.Fatal(err) + } + + nd3, err := ipldcbor.FromJSON(strings.NewReader(`{"lnk": {"/": "`+p1.Cid().String()+`"}}`), math.MaxUint64, -1) + if err != nil { + t.Fatal(err) + } + + if err := api.Dag().AddMany(ctx, []ipld.Node{nd2, nd3}); err != nil { + t.Fatal(err) + } + + err = api.Pin().Add(ctx, path.IpldPath(nd2.Cid())) + if err != nil { + t.Fatal(err) + } + + err = api.Pin().Add(ctx, path.IpldPath(nd3.Cid()), opt.Pin.Recursive(false)) + if err != nil { + t.Fatal(err) + } + + list, err := accPins(api.Pin().Ls(ctx)) + if err != nil { + t.Fatal(err) + } + + if len(list) != 3 { + t.Errorf("unexpected pin list len: %d", len(list)) + } + + list, err = accPins(api.Pin().Ls(ctx, opt.Pin.Ls.Direct())) + if err != nil { + t.Fatal(err) + } + + if len(list) != 1 { + t.Errorf("unexpected pin list len: %d", len(list)) + } + + if list[0].Path().String() != path.IpldPath(nd3.Cid()).String() { + t.Errorf("unexpected path, %s != %s", list[0].Path().String(), path.IpfsPath(nd3.Cid()).String()) + } + + list, err = accPins(api.Pin().Ls(ctx, opt.Pin.Ls.Recursive())) + if err != nil { + t.Fatal(err) + } + + if len(list) != 1 { + t.Errorf("unexpected pin list len: %d", len(list)) + } + + if list[0].Path().String() != path.IpldPath(nd2.Cid()).String() { + t.Errorf("unexpected path, %s != %s", list[0].Path().String(), path.IpldPath(nd2.Cid()).String()) + } + + list, err = accPins(api.Pin().Ls(ctx, opt.Pin.Ls.Indirect())) + if err != nil { + t.Fatal(err) + } + + if len(list) != 1 { + t.Errorf("unexpected pin list len: %d", len(list)) + } + + if list[0].Path().Cid().String() != p0.Cid().String() { + t.Errorf("unexpected path, %s != %s", list[0].Path().Cid().String(), p0.Cid().String()) + } + + res, err := api.Pin().Verify(ctx) + if err != nil { + t.Fatal(err) + } + n := 0 + for r := range res { + if !r.Ok() { + t.Error("expected pin to be ok") + } + n++ + } + + if n != 1 { + t.Errorf("unexpected verify result count: %d", n) + } + + //TODO: figure out a way to test verify without touching IpfsNode + /* + err = api.Block().Rm(ctx, p0, opt.Block.Force(true)) + if err != nil { + t.Fatal(err) + } + + res, err = api.Pin().Verify(ctx) + if err != nil { + t.Fatal(err) + } + n = 0 + for r := range res { + if r.Ok() { + t.Error("expected pin to not be ok") + } + + if len(r.BadNodes()) != 1 { + t.Fatalf("unexpected badNodes len") + } + + if r.BadNodes()[0].Path().Cid().String() != p0.Cid().String() { + t.Error("unexpected badNode path") + } + + if r.BadNodes()[0].Err().Error() != "merkledag: not found" { + t.Errorf("unexpected badNode error: %s", r.BadNodes()[0].Err().Error()) + } + n++ + } + + if n != 1 { + t.Errorf("unexpected verify result count: %d", n) + } + */ +} + +// TestPinLsIndirect verifies that indirect nodes are listed by pin ls even if a parent node is directly pinned +func (tp *TestSuite) TestPinLsIndirect(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + leaf, parent, grandparent := getThreeChainedNodes(t, ctx, api, "foo") + + err = api.Pin().Add(ctx, path.IpldPath(grandparent.Cid())) + if err != nil { + t.Fatal(err) + } + + err = api.Pin().Add(ctx, path.IpldPath(parent.Cid()), opt.Pin.Recursive(false)) + if err != nil { + t.Fatal(err) + } + + assertPinTypes(t, ctx, api, []cidContainer{grandparent}, []cidContainer{parent}, []cidContainer{leaf}) +} + +// TestPinLsPrecedence verifies the precedence of pins (recursive > direct > indirect) +func (tp *TestSuite) TestPinLsPrecedence(t *testing.T) { + // Testing precedence of recursive, direct and indirect pins + // Results should be recursive > indirect, direct > indirect, and recursive > direct + + t.Run("TestPinLsPredenceRecursiveIndirect", tp.TestPinLsPredenceRecursiveIndirect) + t.Run("TestPinLsPrecedenceDirectIndirect", tp.TestPinLsPrecedenceDirectIndirect) + t.Run("TestPinLsPrecedenceRecursiveDirect", tp.TestPinLsPrecedenceRecursiveDirect) +} + +func (tp *TestSuite) TestPinLsPredenceRecursiveIndirect(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + // Test recursive > indirect + leaf, parent, grandparent := getThreeChainedNodes(t, ctx, api, "recursive > indirect") + + err = api.Pin().Add(ctx, path.IpldPath(grandparent.Cid())) + if err != nil { + t.Fatal(err) + } + + err = api.Pin().Add(ctx, path.IpldPath(parent.Cid())) + if err != nil { + t.Fatal(err) + } + + assertPinTypes(t, ctx, api, []cidContainer{grandparent, parent}, []cidContainer{}, []cidContainer{leaf}) +} + +func (tp *TestSuite) TestPinLsPrecedenceDirectIndirect(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + // Test direct > indirect + leaf, parent, grandparent := getThreeChainedNodes(t, ctx, api, "direct > indirect") + + err = api.Pin().Add(ctx, path.IpldPath(grandparent.Cid())) + if err != nil { + t.Fatal(err) + } + + err = api.Pin().Add(ctx, path.IpldPath(parent.Cid()), opt.Pin.Recursive(false)) + if err != nil { + t.Fatal(err) + } + + assertPinTypes(t, ctx, api, []cidContainer{grandparent}, []cidContainer{parent}, []cidContainer{leaf}) +} + +func (tp *TestSuite) TestPinLsPrecedenceRecursiveDirect(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + // Test recursive > direct + leaf, parent, grandparent := getThreeChainedNodes(t, ctx, api, "recursive + direct = error") + + err = api.Pin().Add(ctx, path.IpldPath(parent.Cid())) + if err != nil { + t.Fatal(err) + } + + err = api.Pin().Add(ctx, path.IpldPath(parent.Cid()), opt.Pin.Recursive(false)) + if err == nil { + t.Fatal("expected error directly pinning a recursively pinned node") + } + + assertPinTypes(t, ctx, api, []cidContainer{parent}, []cidContainer{}, []cidContainer{leaf}) + + err = api.Pin().Add(ctx, path.IpldPath(grandparent.Cid()), opt.Pin.Recursive(false)) + if err != nil { + t.Fatal(err) + } + + err = api.Pin().Add(ctx, path.IpldPath(grandparent.Cid())) + if err != nil { + t.Fatal(err) + } + + assertPinTypes(t, ctx, api, []cidContainer{grandparent, parent}, []cidContainer{}, []cidContainer{leaf}) +} + +func (tp *TestSuite) TestPinIsPinned(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + leaf, parent, grandparent := getThreeChainedNodes(t, ctx, api, "foofoo") + + assertNotPinned(t, ctx, api, path.IpldPath(grandparent.Cid())) + assertNotPinned(t, ctx, api, path.IpldPath(parent.Cid())) + assertNotPinned(t, ctx, api, path.IpldPath(leaf.Cid())) + + err = api.Pin().Add(ctx, path.IpldPath(parent.Cid()), opt.Pin.Recursive(true)) + if err != nil { + t.Fatal(err) + } + + assertNotPinned(t, ctx, api, path.IpldPath(grandparent.Cid())) + assertIsPinned(t, ctx, api, path.IpldPath(parent.Cid()), "recursive") + assertIsPinned(t, ctx, api, path.IpldPath(leaf.Cid()), "indirect") + + err = api.Pin().Add(ctx, path.IpldPath(grandparent.Cid()), opt.Pin.Recursive(false)) + if err != nil { + t.Fatal(err) + } + + assertIsPinned(t, ctx, api, path.IpldPath(grandparent.Cid()), "direct") + assertIsPinned(t, ctx, api, path.IpldPath(parent.Cid()), "recursive") + assertIsPinned(t, ctx, api, path.IpldPath(leaf.Cid()), "indirect") +} + +type cidContainer interface { + Cid() cid.Cid +} + +func getThreeChainedNodes(t *testing.T, ctx context.Context, api iface.CoreAPI, leafData string) (cidContainer, cidContainer, cidContainer) { + leaf, err := api.Unixfs().Add(ctx, strFile(leafData)()) + if err != nil { + t.Fatal(err) + } + + parent, err := ipldcbor.FromJSON(strings.NewReader(`{"lnk": {"/": "`+leaf.Cid().String()+`"}}`), math.MaxUint64, -1) + if err != nil { + t.Fatal(err) + } + + grandparent, err := ipldcbor.FromJSON(strings.NewReader(`{"lnk": {"/": "`+parent.Cid().String()+`"}}`), math.MaxUint64, -1) + if err != nil { + t.Fatal(err) + } + + if err := api.Dag().AddMany(ctx, []ipld.Node{parent, grandparent}); err != nil { + t.Fatal(err) + } + + return leaf, parent, grandparent +} + +func assertPinTypes(t *testing.T, ctx context.Context, api iface.CoreAPI, recusive, direct, indirect []cidContainer) { + assertPinLsAllConsistency(t, ctx, api) + + list, err := accPins(api.Pin().Ls(ctx, opt.Pin.Ls.Recursive())) + if err != nil { + t.Fatal(err) + } + + assertPinCids(t, list, recusive...) + + list, err = accPins(api.Pin().Ls(ctx, opt.Pin.Ls.Direct())) + if err != nil { + t.Fatal(err) + } + + assertPinCids(t, list, direct...) + + list, err = accPins(api.Pin().Ls(ctx, opt.Pin.Ls.Indirect())) + if err != nil { + t.Fatal(err) + } + + assertPinCids(t, list, indirect...) +} + +// assertPinCids verifies that the pins match the expected cids +func assertPinCids(t *testing.T, pins []iface.Pin, cids ...cidContainer) { + t.Helper() + + if expected, actual := len(cids), len(pins); expected != actual { + t.Fatalf("expected pin list to have len %d, was %d", expected, actual) + } + + cSet := cid.NewSet() + for _, c := range cids { + cSet.Add(c.Cid()) + } + + valid := true + for _, p := range pins { + c := p.Path().Cid() + if cSet.Has(c) { + cSet.Remove(c) + } else { + valid = false + break + } + } + + valid = valid && cSet.Len() == 0 + + if !valid { + pinStrs := make([]string, len(pins)) + for i, p := range pins { + pinStrs[i] = p.Path().Cid().String() + } + pathStrs := make([]string, len(cids)) + for i, c := range cids { + pathStrs[i] = c.Cid().String() + } + t.Fatalf("expected: %s \nactual: %s", strings.Join(pathStrs, ", "), strings.Join(pinStrs, ", ")) + } +} + +// assertPinLsAllConsistency verifies that listing all pins gives the same result as listing the pin types individually +func assertPinLsAllConsistency(t *testing.T, ctx context.Context, api iface.CoreAPI) { + t.Helper() + allPins, err := accPins(api.Pin().Ls(ctx)) + if err != nil { + t.Fatal(err) + } + + type pinTypeProps struct { + *cid.Set + opt.PinLsOption + } + + all, recursive, direct, indirect := cid.NewSet(), cid.NewSet(), cid.NewSet(), cid.NewSet() + typeMap := map[string]*pinTypeProps{ + "recursive": {recursive, opt.Pin.Ls.Recursive()}, + "direct": {direct, opt.Pin.Ls.Direct()}, + "indirect": {indirect, opt.Pin.Ls.Indirect()}, + } + + for _, p := range allPins { + if !all.Visit(p.Path().Cid()) { + t.Fatalf("pin ls returned the same cid multiple times") + } + + typeStr := p.Type() + if typeSet, ok := typeMap[p.Type()]; ok { + typeSet.Add(p.Path().Cid()) + } else { + t.Fatalf("unknown pin type: %s", typeStr) + } + } + + for typeStr, pinProps := range typeMap { + pins, err := accPins(api.Pin().Ls(ctx, pinProps.PinLsOption)) + if err != nil { + t.Fatal(err) + } + + if expected, actual := len(pins), pinProps.Set.Len(); expected != actual { + t.Fatalf("pin ls all has %d pins of type %s, but pin ls for the type has %d", expected, typeStr, actual) + } + + for _, p := range pins { + if pinType := p.Type(); pinType != typeStr { + t.Fatalf("returned wrong pin type: expected %s, got %s", typeStr, pinType) + } + + if c := p.Path().Cid(); !pinProps.Has(c) { + t.Fatalf("%s expected to be in pin ls all as type %s", c.String(), typeStr) + } + } + } +} + +func assertIsPinned(t *testing.T, ctx context.Context, api iface.CoreAPI, p path.Path, typeStr string) { + t.Helper() + withType, err := opt.Pin.IsPinned.Type(typeStr) + if err != nil { + t.Fatal("unhandled pin type") + } + + whyPinned, pinned, err := api.Pin().IsPinned(ctx, p, withType) + if err != nil { + t.Fatal(err) + } + + if !pinned { + t.Fatalf("%s expected to be pinned with type %s", p, typeStr) + } + + switch typeStr { + case "recursive", "direct": + if typeStr != whyPinned { + t.Fatalf("reason for pinning expected to be %s for %s, got %s", typeStr, p, whyPinned) + } + case "indirect": + if whyPinned == "" { + t.Fatalf("expected to have a pin reason for %s", p) + } + } +} + +func assertNotPinned(t *testing.T, ctx context.Context, api iface.CoreAPI, p path.Path) { + t.Helper() + + _, pinned, err := api.Pin().IsPinned(ctx, p) + if err != nil { + t.Fatal(err) + } + + if pinned { + t.Fatalf("%s expected to not be pinned", p) + } +} + +func accPins(pins <-chan iface.Pin, err error) ([]iface.Pin, error) { + if err != nil { + return nil, err + } + + var result []iface.Pin + + for pin := range pins { + if pin.Err() != nil { + return nil, pin.Err() + } + result = append(result, pin) + } + + return result, nil +} diff --git a/coreiface/tests/pubsub.go b/coreiface/tests/pubsub.go new file mode 100644 index 0000000000..ebb4bdf8a6 --- /dev/null +++ b/coreiface/tests/pubsub.go @@ -0,0 +1,136 @@ +package tests + +import ( + "context" + "testing" + "time" + + iface "github.com/ipfs/go-libipfs/coreiface" + "github.com/ipfs/go-libipfs/coreiface/options" +) + +func (tp *TestSuite) TestPubSub(t *testing.T) { + tp.hasApi(t, func(api iface.CoreAPI) error { + if api.PubSub() == nil { + return errAPINotImplemented + } + return nil + }) + + t.Run("TestBasicPubSub", tp.TestBasicPubSub) +} + +func (tp *TestSuite) TestBasicPubSub(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + apis, err := tp.MakeAPISwarm(ctx, true, 2) + if err != nil { + t.Fatal(err) + } + + sub, err := apis[0].PubSub().Subscribe(ctx, "testch") + if err != nil { + t.Fatal(err) + } + + done := make(chan struct{}) + go func() { + defer close(done) + + ticker := time.NewTicker(100 * time.Millisecond) + defer ticker.Stop() + + for { + err := apis[1].PubSub().Publish(ctx, "testch", []byte("hello world")) + switch err { + case nil: + case context.Canceled: + return + default: + t.Error(err) + cancel() + return + } + select { + case <-ticker.C: + case <-ctx.Done(): + return + } + } + }() + + // Wait for the sender to finish before we return. + // Otherwise, we can get random errors as publish fails. + defer func() { + cancel() + <-done + }() + + m, err := sub.Next(ctx) + if err != nil { + t.Fatal(err) + } + + if string(m.Data()) != "hello world" { + t.Errorf("got invalid data: %s", string(m.Data())) + } + + self1, err := apis[1].Key().Self(ctx) + if err != nil { + t.Fatal(err) + } + + if m.From() != self1.ID() { + t.Errorf("m.From didn't match") + } + + peers, err := apis[1].PubSub().Peers(ctx, options.PubSub.Topic("testch")) + if err != nil { + t.Fatal(err) + } + + if len(peers) != 1 { + t.Fatalf("got incorrect number of peers: %d", len(peers)) + } + + self0, err := apis[0].Key().Self(ctx) + if err != nil { + t.Fatal(err) + } + + if peers[0] != self0.ID() { + t.Errorf("peer didn't match") + } + + peers, err = apis[1].PubSub().Peers(ctx, options.PubSub.Topic("nottestch")) + if err != nil { + t.Fatal(err) + } + + if len(peers) != 0 { + t.Fatalf("got incorrect number of peers: %d", len(peers)) + } + + topics, err := apis[0].PubSub().Ls(ctx) + if err != nil { + t.Fatal(err) + } + + if len(topics) != 1 { + t.Fatalf("got incorrect number of topics: %d", len(peers)) + } + + if topics[0] != "testch" { + t.Errorf("topic didn't match") + } + + topics, err = apis[1].PubSub().Ls(ctx) + if err != nil { + t.Fatal(err) + } + + if len(topics) != 0 { + t.Fatalf("got incorrect number of topics: %d", len(peers)) + } +} diff --git a/coreiface/tests/routing.go b/coreiface/tests/routing.go new file mode 100644 index 0000000000..6a2cc4fdaa --- /dev/null +++ b/coreiface/tests/routing.go @@ -0,0 +1,92 @@ +package tests + +import ( + "context" + "testing" + "time" + + "github.com/gogo/protobuf/proto" + iface "github.com/ipfs/go-libipfs/coreiface" + ipns_pb "github.com/ipfs/go-libipfs/ipns/pb" +) + +func (tp *TestSuite) TestRouting(t *testing.T) { + tp.hasApi(t, func(api iface.CoreAPI) error { + if api.Routing() == nil { + return errAPINotImplemented + } + return nil + }) + + t.Run("TestRoutingGet", tp.TestRoutingGet) + t.Run("TestRoutingPut", tp.TestRoutingPut) +} + +func (tp *TestSuite) testRoutingPublishKey(t *testing.T, ctx context.Context, api iface.CoreAPI) iface.IpnsEntry { + p, err := addTestObject(ctx, api) + if err != nil { + t.Fatal(err) + } + + entry, err := api.Name().Publish(ctx, p) + if err != nil { + t.Fatal(err) + } + + time.Sleep(3 * time.Second) + return entry +} + +func (tp *TestSuite) TestRoutingGet(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + apis, err := tp.MakeAPISwarm(ctx, true, 2) + if err != nil { + t.Fatal(err) + } + + // Node 1: publishes an IPNS name + ipnsEntry := tp.testRoutingPublishKey(t, ctx, apis[0]) + + // Node 2: retrieves the best value for the IPNS name. + data, err := apis[1].Routing().Get(ctx, "/ipns/"+ipnsEntry.Name()) + if err != nil { + t.Fatal(err) + } + + // Checks if values match. + var entry ipns_pb.IpnsEntry + err = proto.Unmarshal(data, &entry) + if err != nil { + t.Fatal(err) + } + + if string(entry.GetValue()) != ipnsEntry.Value().String() { + t.Fatalf("routing key has wrong value, expected %s, got %s", ipnsEntry.Value().String(), string(entry.GetValue())) + } +} + +func (tp *TestSuite) TestRoutingPut(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + apis, err := tp.MakeAPISwarm(ctx, true, 2) + if err != nil { + t.Fatal(err) + } + + // Create and publish IPNS entry. + ipnsEntry := tp.testRoutingPublishKey(t, ctx, apis[0]) + + // Get valid routing value. + data, err := apis[0].Routing().Get(ctx, "/ipns/"+ipnsEntry.Name()) + if err != nil { + t.Fatal(err) + } + + // Put routing value. + err = apis[1].Routing().Put(ctx, "/ipns/"+ipnsEntry.Name(), data) + if err != nil { + t.Fatal(err) + } +} diff --git a/coreiface/tests/unixfs.go b/coreiface/tests/unixfs.go new file mode 100644 index 0000000000..a8fe728055 --- /dev/null +++ b/coreiface/tests/unixfs.go @@ -0,0 +1,1080 @@ +package tests + +import ( + "bytes" + "context" + "encoding/hex" + "fmt" + "io" + "math" + "math/rand" + "os" + "strconv" + "strings" + "sync" + "testing" + + "github.com/ipfs/go-libipfs/coreiface/path" + + coreiface "github.com/ipfs/go-libipfs/coreiface" + "github.com/ipfs/go-libipfs/coreiface/options" + + "github.com/ipfs/go-cid" + cbor "github.com/ipfs/go-ipld-cbor" + ipld "github.com/ipfs/go-ipld-format" + "github.com/ipfs/go-libipfs/files" + "github.com/ipfs/go-libipfs/unixfs" + "github.com/ipfs/go-libipfs/unixfs/importer/helpers" + mdag "github.com/ipfs/go-merkledag" + mh "github.com/multiformats/go-multihash" +) + +func (tp *TestSuite) TestUnixfs(t *testing.T) { + tp.hasApi(t, func(api coreiface.CoreAPI) error { + if api.Unixfs() == nil { + return errAPINotImplemented + } + return nil + }) + + t.Run("TestAdd", tp.TestAdd) + t.Run("TestAddPinned", tp.TestAddPinned) + t.Run("TestAddHashOnly", tp.TestAddHashOnly) + t.Run("TestGetEmptyFile", tp.TestGetEmptyFile) + t.Run("TestGetDir", tp.TestGetDir) + t.Run("TestGetNonUnixfs", tp.TestGetNonUnixfs) + t.Run("TestLs", tp.TestLs) + t.Run("TestEntriesExpired", tp.TestEntriesExpired) + t.Run("TestLsEmptyDir", tp.TestLsEmptyDir) + t.Run("TestLsNonUnixfs", tp.TestLsNonUnixfs) + t.Run("TestAddCloses", tp.TestAddCloses) + t.Run("TestGetSeek", tp.TestGetSeek) + t.Run("TestGetReadAt", tp.TestGetReadAt) +} + +// `echo -n 'hello, world!' | ipfs add` +var hello = "/ipfs/QmQy2Dw4Wk7rdJKjThjYXzfFJNaRKRHhHP5gHHXroJMYxk" +var helloStr = "hello, world!" + +// `echo -n | ipfs add` +var emptyFile = "/ipfs/QmbFMke1KXqnYyBBWxB74N4c5SBnJMVAiMNRcGu6x1AwQH" + +func strFile(data string) func() files.Node { + return func() files.Node { + return files.NewBytesFile([]byte(data)) + } +} + +func twoLevelDir() func() files.Node { + return func() files.Node { + return files.NewMapDirectory(map[string]files.Node{ + "abc": files.NewMapDirectory(map[string]files.Node{ + "def": files.NewBytesFile([]byte("world")), + }), + + "bar": files.NewBytesFile([]byte("hello2")), + "foo": files.NewBytesFile([]byte("hello1")), + }) + } +} + +func flatDir() files.Node { + return files.NewMapDirectory(map[string]files.Node{ + "bar": files.NewBytesFile([]byte("hello2")), + "foo": files.NewBytesFile([]byte("hello1")), + }) +} + +func wrapped(names ...string) func(f files.Node) files.Node { + return func(f files.Node) files.Node { + for i := range names { + f = files.NewMapDirectory(map[string]files.Node{ + names[len(names)-i-1]: f, + }) + } + return f + } +} + +func (tp *TestSuite) TestAdd(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + p := func(h string) path.Resolved { + c, err := cid.Parse(h) + if err != nil { + t.Fatal(err) + } + return path.IpfsPath(c) + } + + rf, err := os.CreateTemp(os.TempDir(), "unixfs-add-real") + if err != nil { + t.Fatal(err) + } + rfp := rf.Name() + + if _, err := rf.Write([]byte(helloStr)); err != nil { + t.Fatal(err) + } + + stat, err := rf.Stat() + if err != nil { + t.Fatal(err) + } + + if err := rf.Close(); err != nil { + t.Fatal(err) + } + defer os.Remove(rfp) + + realFile := func() files.Node { + n, err := files.NewReaderPathFile(rfp, io.NopCloser(strings.NewReader(helloStr)), stat) + if err != nil { + t.Fatal(err) + } + return n + } + + cases := []struct { + name string + data func() files.Node + expect func(files.Node) files.Node + + apiOpts []options.ApiOption + + path string + err string + + wrap string + + events []coreiface.AddEvent + + opts []options.UnixfsAddOption + }{ + // Simple cases + { + name: "simpleAdd", + data: strFile(helloStr), + path: hello, + opts: []options.UnixfsAddOption{}, + }, + { + name: "addEmpty", + data: strFile(""), + path: emptyFile, + }, + // CIDv1 version / rawLeaves + { + name: "addCidV1", + data: strFile(helloStr), + path: "/ipfs/bafkreidi4zlleupgp2bvrpxyja5lbvi4mym7hz5bvhyoowby2qp7g2hxfa", + opts: []options.UnixfsAddOption{options.Unixfs.CidVersion(1)}, + }, + { + name: "addCidV1NoLeaves", + data: strFile(helloStr), + path: "/ipfs/bafybeibhbcn7k7o2m6xsqkrlfiokod3nxwe47viteynhruh6uqx7hvkjfu", + opts: []options.UnixfsAddOption{options.Unixfs.CidVersion(1), options.Unixfs.RawLeaves(false)}, + }, + // Non sha256 hash vs CID + { + name: "addCidSha3", + data: strFile(helloStr), + path: "/ipfs/bafkrmichjflejeh6aren53o7pig7zk3m3vxqcoc2i5dv326k3x6obh7jry", + opts: []options.UnixfsAddOption{options.Unixfs.Hash(mh.SHA3_256)}, + }, + { + name: "addCidSha3Cid0", + data: strFile(helloStr), + err: "CIDv0 only supports sha2-256", + opts: []options.UnixfsAddOption{options.Unixfs.CidVersion(0), options.Unixfs.Hash(mh.SHA3_256)}, + }, + // Inline + { + name: "addInline", + data: strFile(helloStr), + path: "/ipfs/bafyaafikcmeaeeqnnbswy3dpfqqho33snrsccgan", + opts: []options.UnixfsAddOption{options.Unixfs.Inline(true)}, + }, + { + name: "addInlineLimit", + data: strFile(helloStr), + path: "/ipfs/bafyaafikcmeaeeqnnbswy3dpfqqho33snrsccgan", + opts: []options.UnixfsAddOption{options.Unixfs.InlineLimit(32), options.Unixfs.Inline(true)}, + }, + { + name: "addInlineZero", + data: strFile(""), + path: "/ipfs/bafkqaaa", + opts: []options.UnixfsAddOption{options.Unixfs.InlineLimit(0), options.Unixfs.Inline(true), options.Unixfs.RawLeaves(true)}, + }, + { //TODO: after coreapi add is used in `ipfs add`, consider making this default for inline + name: "addInlineRaw", + data: strFile(helloStr), + path: "/ipfs/bafkqadlimvwgy3zmeb3w64tmmqqq", + opts: []options.UnixfsAddOption{options.Unixfs.InlineLimit(32), options.Unixfs.Inline(true), options.Unixfs.RawLeaves(true)}, + }, + // Chunker / Layout + { + name: "addChunks", + data: strFile(strings.Repeat("aoeuidhtns", 200)), + path: "/ipfs/QmRo11d4QJrST47aaiGVJYwPhoNA4ihRpJ5WaxBWjWDwbX", + opts: []options.UnixfsAddOption{options.Unixfs.Chunker("size-4")}, + }, + { + name: "addChunksTrickle", + data: strFile(strings.Repeat("aoeuidhtns", 200)), + path: "/ipfs/QmNNhDGttafX3M1wKWixGre6PrLFGjnoPEDXjBYpTv93HP", + opts: []options.UnixfsAddOption{options.Unixfs.Chunker("size-4"), options.Unixfs.Layout(options.TrickleLayout)}, + }, + // Local + { + name: "addLocal", // better cases in sharness + data: strFile(helloStr), + path: hello, + apiOpts: []options.ApiOption{options.Api.Offline(true)}, + }, + { + name: "hashOnly", // test (non)fetchability + data: strFile(helloStr), + path: hello, + opts: []options.UnixfsAddOption{options.Unixfs.HashOnly(true)}, + }, + // multi file + { + name: "simpleDirNoWrap", + data: flatDir, + path: "/ipfs/QmRKGpFfR32FVXdvJiHfo4WJ5TDYBsM1P9raAp1p6APWSp", + }, + { + name: "simpleDir", + data: flatDir, + wrap: "t", + expect: wrapped("t"), + path: "/ipfs/Qmc3nGXm1HtUVCmnXLQHvWcNwfdZGpfg2SRm1CxLf7Q2Rm", + }, + { + name: "twoLevelDir", + data: twoLevelDir(), + wrap: "t", + expect: wrapped("t"), + path: "/ipfs/QmPwsL3T5sWhDmmAWZHAzyjKtMVDS9a11aHNRqb3xoVnmg", + }, + // wrapped + { + name: "addWrapped", + path: "/ipfs/QmVE9rNpj5doj7XHzp5zMUxD7BJgXEqx4pe3xZ3JBReWHE", + data: func() files.Node { + return files.NewBytesFile([]byte(helloStr)) + }, + wrap: "foo", + expect: wrapped("foo"), + }, + // hidden + { + name: "hiddenFilesAdded", + data: func() files.Node { + return files.NewMapDirectory(map[string]files.Node{ + ".bar": files.NewBytesFile([]byte("hello2")), + "bar": files.NewBytesFile([]byte("hello2")), + "foo": files.NewBytesFile([]byte("hello1")), + }) + }, + wrap: "t", + expect: wrapped("t"), + path: "/ipfs/QmPXLSBX382vJDLrGakcbrZDkU3grfkjMox7EgSC9KFbtQ", + }, + // NoCopy + { + name: "simpleNoCopy", + data: realFile, + path: "/ipfs/bafkreidi4zlleupgp2bvrpxyja5lbvi4mym7hz5bvhyoowby2qp7g2hxfa", + opts: []options.UnixfsAddOption{options.Unixfs.Nocopy(true)}, + }, + { + name: "noCopyNoRaw", + data: realFile, + path: "/ipfs/bafkreidi4zlleupgp2bvrpxyja5lbvi4mym7hz5bvhyoowby2qp7g2hxfa", + opts: []options.UnixfsAddOption{options.Unixfs.Nocopy(true), options.Unixfs.RawLeaves(false)}, + err: "nocopy option requires '--raw-leaves' to be enabled as well", + }, + { + name: "noCopyNoPath", + data: strFile(helloStr), + path: "/ipfs/bafkreidi4zlleupgp2bvrpxyja5lbvi4mym7hz5bvhyoowby2qp7g2hxfa", + opts: []options.UnixfsAddOption{options.Unixfs.Nocopy(true)}, + err: helpers.ErrMissingFsRef.Error(), + }, + // Events / Progress + { + name: "simpleAddEvent", + data: strFile(helloStr), + path: "/ipfs/bafkreidi4zlleupgp2bvrpxyja5lbvi4mym7hz5bvhyoowby2qp7g2hxfa", + events: []coreiface.AddEvent{ + {Name: "bafkreidi4zlleupgp2bvrpxyja5lbvi4mym7hz5bvhyoowby2qp7g2hxfa", Path: p("bafkreidi4zlleupgp2bvrpxyja5lbvi4mym7hz5bvhyoowby2qp7g2hxfa"), Size: strconv.Itoa(len(helloStr))}, + }, + opts: []options.UnixfsAddOption{options.Unixfs.RawLeaves(true)}, + }, + { + name: "silentAddEvent", + data: twoLevelDir(), + path: "/ipfs/QmVG2ZYCkV1S4TK8URA3a4RupBF17A8yAr4FqsRDXVJASr", + events: []coreiface.AddEvent{ + {Name: "abc", Path: p("QmU7nuGs2djqK99UNsNgEPGh6GV4662p6WtsgccBNGTDxt"), Size: "62"}, + {Name: "", Path: p("QmVG2ZYCkV1S4TK8URA3a4RupBF17A8yAr4FqsRDXVJASr"), Size: "229"}, + }, + opts: []options.UnixfsAddOption{options.Unixfs.Silent(true)}, + }, + { + name: "dirAddEvents", + data: twoLevelDir(), + path: "/ipfs/QmVG2ZYCkV1S4TK8URA3a4RupBF17A8yAr4FqsRDXVJASr", + events: []coreiface.AddEvent{ + {Name: "abc/def", Path: p("QmNyJpQkU1cEkBwMDhDNFstr42q55mqG5GE5Mgwug4xyGk"), Size: "13"}, + {Name: "bar", Path: p("QmS21GuXiRMvJKHos4ZkEmQDmRBqRaF5tQS2CQCu2ne9sY"), Size: "14"}, + {Name: "foo", Path: p("QmfAjGiVpTN56TXi6SBQtstit5BEw3sijKj1Qkxn6EXKzJ"), Size: "14"}, + {Name: "abc", Path: p("QmU7nuGs2djqK99UNsNgEPGh6GV4662p6WtsgccBNGTDxt"), Size: "62"}, + {Name: "", Path: p("QmVG2ZYCkV1S4TK8URA3a4RupBF17A8yAr4FqsRDXVJASr"), Size: "229"}, + }, + }, + { + name: "progress1M", + data: func() files.Node { + return files.NewReaderFile(bytes.NewReader(bytes.Repeat([]byte{0}, 1000000))) + }, + path: "/ipfs/QmXXNNbwe4zzpdMg62ZXvnX1oU7MwSrQ3vAEtuwFKCm1oD", + events: []coreiface.AddEvent{ + {Name: "", Bytes: 262144}, + {Name: "", Bytes: 524288}, + {Name: "", Bytes: 786432}, + {Name: "", Bytes: 1000000}, + {Name: "QmXXNNbwe4zzpdMg62ZXvnX1oU7MwSrQ3vAEtuwFKCm1oD", Path: p("QmXXNNbwe4zzpdMg62ZXvnX1oU7MwSrQ3vAEtuwFKCm1oD"), Size: "1000256"}, + }, + wrap: "", + opts: []options.UnixfsAddOption{options.Unixfs.Progress(true)}, + }, + } + + for _, testCase := range cases { + t.Run(testCase.name, func(t *testing.T) { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + // recursive logic + + data := testCase.data() + if testCase.wrap != "" { + data = files.NewMapDirectory(map[string]files.Node{ + testCase.wrap: data, + }) + } + + // handle events if relevant to test case + + opts := testCase.opts + eventOut := make(chan interface{}) + var evtWg sync.WaitGroup + if len(testCase.events) > 0 { + opts = append(opts, options.Unixfs.Events(eventOut)) + evtWg.Add(1) + + go func() { + defer evtWg.Done() + expected := testCase.events + + for evt := range eventOut { + event, ok := evt.(*coreiface.AddEvent) + if !ok { + t.Error("unexpected event type") + continue + } + + if len(expected) < 1 { + t.Error("got more events than expected") + continue + } + + if expected[0].Size != event.Size { + t.Errorf("Event.Size didn't match, %s != %s", expected[0].Size, event.Size) + } + + if expected[0].Name != event.Name { + t.Errorf("Event.Name didn't match, %s != %s", expected[0].Name, event.Name) + } + + if expected[0].Path != nil && event.Path != nil { + if expected[0].Path.Cid().String() != event.Path.Cid().String() { + t.Errorf("Event.Hash didn't match, %s != %s", expected[0].Path, event.Path) + } + } else if event.Path != expected[0].Path { + t.Errorf("Event.Hash didn't match, %s != %s", expected[0].Path, event.Path) + } + if expected[0].Bytes != event.Bytes { + t.Errorf("Event.Bytes didn't match, %d != %d", expected[0].Bytes, event.Bytes) + } + + expected = expected[1:] + } + + if len(expected) > 0 { + t.Errorf("%d event(s) didn't arrive", len(expected)) + } + }() + } + + tapi, err := api.WithOptions(testCase.apiOpts...) + if err != nil { + t.Fatal(err) + } + + // Add! + + p, err := tapi.Unixfs().Add(ctx, data, opts...) + close(eventOut) + evtWg.Wait() + if testCase.err != "" { + if err == nil { + t.Fatalf("expected an error: %s", testCase.err) + } + if err.Error() != testCase.err { + t.Fatalf("expected an error: '%s' != '%s'", err.Error(), testCase.err) + } + return + } + if err != nil { + t.Fatal(err) + } + + if p.String() != testCase.path { + t.Errorf("expected path %s, got: %s", testCase.path, p) + } + + // compare file structure with Unixfs().Get + + var cmpFile func(origName string, orig files.Node, gotName string, got files.Node) + cmpFile = func(origName string, orig files.Node, gotName string, got files.Node) { + _, origDir := orig.(files.Directory) + _, gotDir := got.(files.Directory) + + if origName != gotName { + t.Errorf("file name mismatch, orig='%s', got='%s'", origName, gotName) + } + + if origDir != gotDir { + t.Fatalf("file type mismatch on %s", origName) + } + + if !gotDir { + defer orig.Close() + defer got.Close() + + do, err := io.ReadAll(orig.(files.File)) + if err != nil { + t.Fatal(err) + } + + dg, err := io.ReadAll(got.(files.File)) + if err != nil { + t.Fatal(err) + } + + if !bytes.Equal(do, dg) { + t.Fatal("data not equal") + } + + return + } + + origIt := orig.(files.Directory).Entries() + gotIt := got.(files.Directory).Entries() + + for { + if origIt.Next() { + if !gotIt.Next() { + t.Fatal("gotIt out of entries before origIt") + } + } else { + if gotIt.Next() { + t.Fatal("origIt out of entries before gotIt") + } + break + } + + cmpFile(origIt.Name(), origIt.Node(), gotIt.Name(), gotIt.Node()) + } + if origIt.Err() != nil { + t.Fatal(origIt.Err()) + } + if gotIt.Err() != nil { + t.Fatal(gotIt.Err()) + } + } + + f, err := tapi.Unixfs().Get(ctx, p) + if err != nil { + t.Fatal(err) + } + + orig := testCase.data() + if testCase.expect != nil { + orig = testCase.expect(orig) + } + + cmpFile("", orig, "", f) + }) + } +} + +func (tp *TestSuite) TestAddPinned(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + _, err = api.Unixfs().Add(ctx, strFile(helloStr)(), options.Unixfs.Pin(true)) + if err != nil { + t.Fatal(err) + } + + pins, err := accPins(api.Pin().Ls(ctx)) + if err != nil { + t.Fatal(err) + } + if len(pins) != 1 { + t.Fatalf("expected 1 pin, got %d", len(pins)) + } + + if pins[0].Path().String() != "/ipld/QmQy2Dw4Wk7rdJKjThjYXzfFJNaRKRHhHP5gHHXroJMYxk" { + t.Fatalf("got unexpected pin: %s", pins[0].Path().String()) + } +} + +func (tp *TestSuite) TestAddHashOnly(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + p, err := api.Unixfs().Add(ctx, strFile(helloStr)(), options.Unixfs.HashOnly(true)) + if err != nil { + t.Fatal(err) + } + + if p.String() != hello { + t.Errorf("unxepected path: %s", p.String()) + } + + _, err = api.Block().Get(ctx, p) + if err == nil { + t.Fatal("expected an error") + } + if !ipld.IsNotFound(err) { + t.Errorf("unxepected error: %s", err.Error()) + } +} + +func (tp *TestSuite) TestGetEmptyFile(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + _, err = api.Unixfs().Add(ctx, files.NewBytesFile([]byte{})) + if err != nil { + t.Fatal(err) + } + + emptyFilePath := path.New(emptyFile) + + r, err := api.Unixfs().Get(ctx, emptyFilePath) + if err != nil { + t.Fatal(err) + } + + buf := make([]byte, 1) // non-zero so that Read() actually tries to read + n, err := io.ReadFull(r.(files.File), buf) + if err != nil && err != io.EOF { + t.Error(err) + } + if !bytes.HasPrefix(buf, []byte{0x00}) { + t.Fatalf("expected empty data, got [%s] [read=%d]", buf, n) + } +} + +func (tp *TestSuite) TestGetDir(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + edir := unixfs.EmptyDirNode() + err = api.Dag().Add(ctx, edir) + if err != nil { + t.Fatal(err) + } + p := path.IpfsPath(edir.Cid()) + + emptyDir, err := api.Object().New(ctx, options.Object.Type("unixfs-dir")) + if err != nil { + t.Fatal(err) + } + + if p.String() != path.IpfsPath(emptyDir.Cid()).String() { + t.Fatalf("expected path %s, got: %s", emptyDir.Cid(), p.String()) + } + + r, err := api.Unixfs().Get(ctx, path.IpfsPath(emptyDir.Cid())) + if err != nil { + t.Fatal(err) + } + + if _, ok := r.(files.Directory); !ok { + t.Fatalf("expected a directory") + } +} + +func (tp *TestSuite) TestGetNonUnixfs(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + nd := new(mdag.ProtoNode) + err = api.Dag().Add(ctx, nd) + if err != nil { + t.Fatal(err) + } + + _, err = api.Unixfs().Get(ctx, path.IpfsPath(nd.Cid())) + if !strings.Contains(err.Error(), "proto: required field") { + t.Fatalf("expected protobuf error, got: %s", err) + } +} + +func (tp *TestSuite) TestLs(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + r := strings.NewReader("content-of-file") + p, err := api.Unixfs().Add(ctx, files.NewMapDirectory(map[string]files.Node{ + "name-of-file": files.NewReaderFile(r), + "name-of-symlink": files.NewLinkFile("/foo/bar", nil), + })) + if err != nil { + t.Fatal(err) + } + + entries, err := api.Unixfs().Ls(ctx, p) + if err != nil { + t.Fatal(err) + } + + entry := <-entries + if entry.Err != nil { + t.Fatal(entry.Err) + } + if entry.Size != 15 { + t.Errorf("expected size = 15, got %d", entry.Size) + } + if entry.Name != "name-of-file" { + t.Errorf("expected name = name-of-file, got %s", entry.Name) + } + if entry.Type != coreiface.TFile { + t.Errorf("wrong type %s", entry.Type) + } + if entry.Cid.String() != "QmX3qQVKxDGz3URVC3861Z3CKtQKGBn6ffXRBBWGMFz9Lr" { + t.Errorf("expected cid = QmX3qQVKxDGz3URVC3861Z3CKtQKGBn6ffXRBBWGMFz9Lr, got %s", entry.Cid) + } + entry = <-entries + if entry.Err != nil { + t.Fatal(entry.Err) + } + if entry.Type != coreiface.TSymlink { + t.Errorf("wrong type %s", entry.Type) + } + if entry.Name != "name-of-symlink" { + t.Errorf("expected name = name-of-symlink, got %s", entry.Name) + } + if entry.Target != "/foo/bar" { + t.Errorf("expected symlink target to be /foo/bar, got %s", entry.Target) + } + + if l, ok := <-entries; ok { + t.Errorf("didn't expect a second link") + if l.Err != nil { + t.Error(l.Err) + } + } +} + +func (tp *TestSuite) TestEntriesExpired(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + r := strings.NewReader("content-of-file") + p, err := api.Unixfs().Add(ctx, files.NewMapDirectory(map[string]files.Node{ + "name-of-file": files.NewReaderFile(r), + })) + if err != nil { + t.Fatal(err) + } + + ctx, cancel = context.WithCancel(ctx) + + nd, err := api.Unixfs().Get(ctx, p) + if err != nil { + t.Fatal(err) + } + cancel() + + it := files.ToDir(nd).Entries() + if it == nil { + t.Fatal("it was nil") + } + + if it.Next() { + t.Fatal("Next succeeded") + } + + if it.Err() != context.Canceled { + t.Fatalf("unexpected error %s", it.Err()) + } + + if it.Next() { + t.Fatal("Next succeeded") + } +} + +func (tp *TestSuite) TestLsEmptyDir(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + _, err = api.Unixfs().Add(ctx, files.NewSliceDirectory([]files.DirEntry{})) + if err != nil { + t.Fatal(err) + } + + emptyDir, err := api.Object().New(ctx, options.Object.Type("unixfs-dir")) + if err != nil { + t.Fatal(err) + } + + links, err := api.Unixfs().Ls(ctx, path.IpfsPath(emptyDir.Cid())) + if err != nil { + t.Fatal(err) + } + + if len(links) != 0 { + t.Fatalf("expected 0 links, got %d", len(links)) + } +} + +// TODO(lgierth) this should test properly, with len(links) > 0 +func (tp *TestSuite) TestLsNonUnixfs(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + nd, err := cbor.WrapObject(map[string]interface{}{"foo": "bar"}, math.MaxUint64, -1) + if err != nil { + t.Fatal(err) + } + + err = api.Dag().Add(ctx, nd) + if err != nil { + t.Fatal(err) + } + + links, err := api.Unixfs().Ls(ctx, path.IpfsPath(nd.Cid())) + if err != nil { + t.Fatal(err) + } + + if len(links) != 0 { + t.Fatalf("expected 0 links, got %d", len(links)) + } +} + +type closeTestF struct { + files.File + closed bool + + t *testing.T +} + +type closeTestD struct { + files.Directory + closed bool + + t *testing.T +} + +func (f *closeTestD) Close() error { + f.t.Helper() + if f.closed { + f.t.Fatal("already closed") + } + f.closed = true + return nil +} + +func (f *closeTestF) Close() error { + if f.closed { + f.t.Fatal("already closed") + } + f.closed = true + return nil +} + +func (tp *TestSuite) TestAddCloses(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + n4 := &closeTestF{files.NewBytesFile([]byte("foo")), false, t} + d3 := &closeTestD{files.NewMapDirectory(map[string]files.Node{ + "sub": n4, + }), false, t} + n2 := &closeTestF{files.NewBytesFile([]byte("bar")), false, t} + n1 := &closeTestF{files.NewBytesFile([]byte("baz")), false, t} + d0 := &closeTestD{files.NewMapDirectory(map[string]files.Node{ + "a": d3, + "b": n1, + "c": n2, + }), false, t} + + _, err = api.Unixfs().Add(ctx, d0) + if err != nil { + t.Fatal(err) + } + + for i, n := range []*closeTestF{n1, n2, n4} { + if !n.closed { + t.Errorf("file %d not closed!", i) + } + } + + for i, n := range []*closeTestD{d0, d3} { + if !n.closed { + t.Errorf("dir %d not closed!", i) + } + } +} + +func (tp *TestSuite) TestGetSeek(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + dataSize := int64(100000) + tf := files.NewReaderFile(io.LimitReader(rand.New(rand.NewSource(1403768328)), dataSize)) + + p, err := api.Unixfs().Add(ctx, tf, options.Unixfs.Chunker("size-100")) + if err != nil { + t.Fatal(err) + } + + r, err := api.Unixfs().Get(ctx, p) + if err != nil { + t.Fatal(err) + } + + f := files.ToFile(r) + if f == nil { + t.Fatal("not a file") + } + + orig := make([]byte, dataSize) + if _, err := io.ReadFull(f, orig); err != nil { + t.Fatal(err) + } + f.Close() + + origR := bytes.NewReader(orig) + + r, err = api.Unixfs().Get(ctx, p) + if err != nil { + t.Fatal(err) + } + + f = files.ToFile(r) + if f == nil { + t.Fatal("not a file") + } + + test := func(offset int64, whence int, read int, expect int64, shouldEof bool) { + t.Run(fmt.Sprintf("seek%d+%d-r%d-%d", whence, offset, read, expect), func(t *testing.T) { + n, err := f.Seek(offset, whence) + if err != nil { + t.Fatal(err) + } + origN, err := origR.Seek(offset, whence) + if err != nil { + t.Fatal(err) + } + + if n != origN { + t.Fatalf("offsets didn't match, expected %d, got %d", origN, n) + } + + buf := make([]byte, read) + origBuf := make([]byte, read) + origRead, err := origR.Read(origBuf) + if err != nil { + t.Fatalf("orig: %s", err) + } + r, err := io.ReadFull(f, buf) + switch { + case shouldEof && err != nil && err != io.ErrUnexpectedEOF: + fallthrough + case !shouldEof && err != nil: + t.Fatalf("f: %s", err) + case shouldEof: + _, err := f.Read([]byte{0}) + if err != io.EOF { + t.Fatal("expected EOF") + } + _, err = origR.Read([]byte{0}) + if err != io.EOF { + t.Fatal("expected EOF (orig)") + } + } + + if int64(r) != expect { + t.Fatal("read wrong amount of data") + } + if r != origRead { + t.Fatal("read different amount of data than bytes.Reader") + } + if !bytes.Equal(buf, origBuf) { + fmt.Fprintf(os.Stderr, "original:\n%s\n", hex.Dump(origBuf)) + fmt.Fprintf(os.Stderr, "got:\n%s\n", hex.Dump(buf)) + t.Fatal("data didn't match") + } + }) + } + + test(3, io.SeekCurrent, 10, 10, false) + test(3, io.SeekCurrent, 10, 10, false) + test(500, io.SeekCurrent, 10, 10, false) + test(350, io.SeekStart, 100, 100, false) + test(-123, io.SeekCurrent, 100, 100, false) + test(0, io.SeekStart, int(dataSize), dataSize, false) + test(dataSize-50, io.SeekStart, 100, 50, true) + test(-5, io.SeekEnd, 100, 5, true) +} + +func (tp *TestSuite) TestGetReadAt(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + dataSize := int64(100000) + tf := files.NewReaderFile(io.LimitReader(rand.New(rand.NewSource(1403768328)), dataSize)) + + p, err := api.Unixfs().Add(ctx, tf, options.Unixfs.Chunker("size-100")) + if err != nil { + t.Fatal(err) + } + + r, err := api.Unixfs().Get(ctx, p) + if err != nil { + t.Fatal(err) + } + + f, ok := r.(interface { + files.File + io.ReaderAt + }) + if !ok { + t.Skip("ReaderAt not implemented") + } + + orig := make([]byte, dataSize) + if _, err := io.ReadFull(f, orig); err != nil { + t.Fatal(err) + } + f.Close() + + origR := bytes.NewReader(orig) + + if _, err := api.Unixfs().Get(ctx, p); err != nil { + t.Fatal(err) + } + + test := func(offset int64, read int, expect int64, shouldEof bool) { + t.Run(fmt.Sprintf("readat%d-r%d-%d", offset, read, expect), func(t *testing.T) { + origBuf := make([]byte, read) + origRead, err := origR.ReadAt(origBuf, offset) + if err != nil && err != io.EOF { + t.Fatalf("orig: %s", err) + } + buf := make([]byte, read) + r, err := f.ReadAt(buf, offset) + if shouldEof { + if err != io.EOF { + t.Fatal("expected EOF, got: ", err) + } + } else if err != nil { + t.Fatal("got: ", err) + } + + if int64(r) != expect { + t.Fatal("read wrong amount of data") + } + if r != origRead { + t.Fatal("read different amount of data than bytes.Reader") + } + if !bytes.Equal(buf, origBuf) { + fmt.Fprintf(os.Stderr, "original:\n%s\n", hex.Dump(origBuf)) + fmt.Fprintf(os.Stderr, "got:\n%s\n", hex.Dump(buf)) + t.Fatal("data didn't match") + } + }) + } + + test(3, 10, 10, false) + test(13, 10, 10, false) + test(513, 10, 10, false) + test(350, 100, 100, false) + test(0, int(dataSize), dataSize, false) + test(dataSize-50, 100, 50, true) +} diff --git a/coreiface/unixfs.go b/coreiface/unixfs.go new file mode 100644 index 0000000000..0fcf803282 --- /dev/null +++ b/coreiface/unixfs.go @@ -0,0 +1,80 @@ +package iface + +import ( + "context" + + "github.com/ipfs/go-libipfs/coreiface/options" + path "github.com/ipfs/go-libipfs/coreiface/path" + + "github.com/ipfs/go-cid" + "github.com/ipfs/go-libipfs/files" +) + +type AddEvent struct { + Name string + Path path.Resolved `json:",omitempty"` + Bytes int64 `json:",omitempty"` + Size string `json:",omitempty"` +} + +// FileType is an enum of possible UnixFS file types. +type FileType int32 + +const ( + // TUnknown means the file type isn't known (e.g., it hasn't been + // resolved). + TUnknown FileType = iota + // TFile is a regular file. + TFile + // TDirectory is a directory. + TDirectory + // TSymlink is a symlink. + TSymlink +) + +func (t FileType) String() string { + switch t { + case TUnknown: + return "unknown" + case TFile: + return "file" + case TDirectory: + return "directory" + case TSymlink: + return "symlink" + default: + return "" + } +} + +// DirEntry is a directory entry returned by `Ls`. +type DirEntry struct { + Name string + Cid cid.Cid + + // Only filled when asked to resolve the directory entry. + Size uint64 // The size of the file in bytes (or the size of the symlink). + Type FileType // The type of the file. + Target string // The symlink target (if a symlink). + + Err error +} + +// UnixfsAPI is the basic interface to immutable files in IPFS +// NOTE: This API is heavily WIP, things are guaranteed to break frequently +type UnixfsAPI interface { + // Add imports the data from the reader into merkledag file + // + // TODO: a long useful comment on how to use this for many different scenarios + Add(context.Context, files.Node, ...options.UnixfsAddOption) (path.Resolved, error) + + // Get returns a read-only handle to a file tree referenced by a path + // + // Note that some implementations of this API may apply the specified context + // to operations performed on the returned file + Get(context.Context, path.Path) (files.Node, error) + + // Ls returns the list of links in a directory. Links aren't guaranteed to be + // returned in order + Ls(context.Context, path.Path, ...options.UnixfsLsOption) (<-chan DirEntry, error) +} diff --git a/coreiface/util.go b/coreiface/util.go new file mode 100644 index 0000000000..6d58bf40d2 --- /dev/null +++ b/coreiface/util.go @@ -0,0 +1,20 @@ +package iface + +import ( + "context" + "io" +) + +type Reader interface { + ReadSeekCloser + Size() uint64 + CtxReadFull(context.Context, []byte) (int, error) +} + +// A ReadSeekCloser implements interfaces to read, copy, seek and close. +type ReadSeekCloser interface { + io.Reader + io.Seeker + io.Closer + io.WriterTo +} diff --git a/datastore/dshelp/.github/ISSUE_TEMPLATE/open_an_issue.md b/datastore/dshelp/.github/ISSUE_TEMPLATE/open_an_issue.md new file mode 100644 index 0000000000..4fcbd00aca --- /dev/null +++ b/datastore/dshelp/.github/ISSUE_TEMPLATE/open_an_issue.md @@ -0,0 +1,19 @@ +--- +name: Open an issue +about: Only for actionable issues relevant to this repository. +title: '' +labels: need/triage +assignees: '' + +--- + diff --git a/datastore/dshelp/key.go b/datastore/dshelp/key.go new file mode 100644 index 0000000000..32b73a61e6 --- /dev/null +++ b/datastore/dshelp/key.go @@ -0,0 +1,51 @@ +// Package dshelp provides utilities for parsing and creating +// datastore keys used by go-ipfs +package dshelp + +import ( + "github.com/ipfs/go-cid" + "github.com/ipfs/go-datastore" + "github.com/multiformats/go-base32" + mh "github.com/multiformats/go-multihash" +) + +// NewKeyFromBinary creates a new key from a byte slice. +func NewKeyFromBinary(rawKey []byte) datastore.Key { + buf := make([]byte, 1+base32.RawStdEncoding.EncodedLen(len(rawKey))) + buf[0] = '/' + base32.RawStdEncoding.Encode(buf[1:], rawKey) + return datastore.RawKey(string(buf)) +} + +// BinaryFromDsKey returns the byte slice corresponding to the given Key. +func BinaryFromDsKey(k datastore.Key) ([]byte, error) { + return base32.RawStdEncoding.DecodeString(k.String()[1:]) +} + +// MultihashToDsKey creates a Key from the given Multihash. +// If working with Cids, you can call cid.Hash() to obtain +// the multihash. Note that different CIDs might represent +// the same multihash. +func MultihashToDsKey(k mh.Multihash) datastore.Key { + return NewKeyFromBinary(k) +} + +// DsKeyToMultihash converts a dsKey to the corresponding Multihash. +func DsKeyToMultihash(dsKey datastore.Key) (mh.Multihash, error) { + kb, err := BinaryFromDsKey(dsKey) + if err != nil { + return nil, err + } + return mh.Cast(kb) +} + +// DsKeyToCidV1Raw converts the given Key (which should be a raw multihash +// key) to a Cid V1 of the given type (see +// https://godoc.org/github.com/ipfs/go-cid#pkg-constants). +func DsKeyToCidV1(dsKey datastore.Key, codecType uint64) (cid.Cid, error) { + hash, err := DsKeyToMultihash(dsKey) + if err != nil { + return cid.Cid{}, err + } + return cid.NewCidV1(codecType, hash), nil +} diff --git a/datastore/dshelp/key_test.go b/datastore/dshelp/key_test.go new file mode 100644 index 0000000000..ff9fcc7d6f --- /dev/null +++ b/datastore/dshelp/key_test.go @@ -0,0 +1,24 @@ +package dshelp + +import ( + "testing" + + cid "github.com/ipfs/go-cid" +) + +func TestKey(t *testing.T) { + c, _ := cid.Decode("QmP63DkAFEnDYNjDYBpyNDfttu1fvUw99x1brscPzpqmmq") + dsKey := MultihashToDsKey(c.Hash()) + mh, err := DsKeyToMultihash(dsKey) + if err != nil { + t.Fatal(err) + } + if string(c.Hash()) != string(mh) { + t.Fatal("should have parsed the same multihash") + } + + c2, err := DsKeyToCidV1(dsKey, cid.Raw) + if err != nil || c.Equals(c2) || c2.Type() != cid.Raw || c2.Version() != 1 { + t.Fatal("should have been converted to CIDv1-raw") + } +} diff --git a/examples/gateway/car/main.go b/examples/gateway/car/main.go index 4ba63d2533..d3d31a2ad0 100644 --- a/examples/gateway/car/main.go +++ b/examples/gateway/car/main.go @@ -8,9 +8,9 @@ import ( "os" "strconv" - "github.com/ipfs/go-blockservice" + "github.com/ipfs/go-libipfs/blockservice" "github.com/ipfs/go-cid" - offline "github.com/ipfs/go-ipfs-exchange-offline" + offline "github.com/ipfs/go-libipfs/exchange/offline" "github.com/ipfs/go-libipfs/examples/gateway/common" "github.com/ipfs/go-libipfs/gateway" carblockstore "github.com/ipld/go-car/v2/blockstore" diff --git a/examples/gateway/common/blocks.go b/examples/gateway/common/blocks.go index a0a0f47025..f9dc510270 100644 --- a/examples/gateway/common/blocks.go +++ b/examples/gateway/common/blocks.go @@ -7,26 +7,26 @@ import ( "net/http" gopath "path" - "github.com/ipfs/go-blockservice" + "github.com/ipfs/go-libipfs/blockservice" "github.com/ipfs/go-cid" - bsfetcher "github.com/ipfs/go-fetcher/impl/blockservice" - blockstore "github.com/ipfs/go-ipfs-blockstore" + bsfetcher "github.com/ipfs/go-libipfs/fetcher/impl/blockservice" + blockstore "github.com/ipfs/go-libipfs/blockstore" format "github.com/ipfs/go-ipld-format" "github.com/ipfs/go-libipfs/blocks" "github.com/ipfs/go-libipfs/files" "github.com/ipfs/go-libipfs/gateway" "github.com/ipfs/go-merkledag" - "github.com/ipfs/go-namesys" - "github.com/ipfs/go-namesys/resolve" - ipfspath "github.com/ipfs/go-path" - "github.com/ipfs/go-path/resolver" - "github.com/ipfs/go-unixfs" - ufile "github.com/ipfs/go-unixfs/file" - uio "github.com/ipfs/go-unixfs/io" + "github.com/ipfs/go-libipfs/namesys" + "github.com/ipfs/go-libipfs/namesys/resolve" + ipfspath "github.com/ipfs/go-libipfs/path" + "github.com/ipfs/go-libipfs/path/resolver" + "github.com/ipfs/go-libipfs/unixfs" + ufile "github.com/ipfs/go-libipfs/unixfs/file" + uio "github.com/ipfs/go-libipfs/unixfs/io" "github.com/ipfs/go-unixfsnode" - iface "github.com/ipfs/interface-go-ipfs-core" - nsopts "github.com/ipfs/interface-go-ipfs-core/options/namesys" - ifacepath "github.com/ipfs/interface-go-ipfs-core/path" + iface "github.com/ipfs/go-libipfs/coreiface" + nsopts "github.com/ipfs/go-libipfs/coreiface/options/namesys" + ifacepath "github.com/ipfs/go-libipfs/coreiface/path" dagpb "github.com/ipld/go-codec-dagpb" "github.com/ipld/go-ipld-prime" "github.com/ipld/go-ipld-prime/node/basicnode" diff --git a/examples/gateway/proxy/blockstore.go b/examples/gateway/proxy/blockstore.go index 5612fdfb0c..f883de6f9e 100644 --- a/examples/gateway/proxy/blockstore.go +++ b/examples/gateway/proxy/blockstore.go @@ -9,7 +9,7 @@ import ( "net/url" "github.com/ipfs/go-cid" - blockstore "github.com/ipfs/go-ipfs-blockstore" + blockstore "github.com/ipfs/go-libipfs/blockstore" "github.com/ipfs/go-libipfs/blocks" ) diff --git a/examples/gateway/proxy/main.go b/examples/gateway/proxy/main.go index 793fee1218..c4fc9082db 100644 --- a/examples/gateway/proxy/main.go +++ b/examples/gateway/proxy/main.go @@ -6,8 +6,8 @@ import ( "net/http" "strconv" - "github.com/ipfs/go-blockservice" - offline "github.com/ipfs/go-ipfs-exchange-offline" + "github.com/ipfs/go-libipfs/blockservice" + offline "github.com/ipfs/go-libipfs/exchange/offline" "github.com/ipfs/go-libipfs/examples/gateway/common" "github.com/ipfs/go-libipfs/gateway" "github.com/prometheus/client_golang/prometheus" diff --git a/examples/gateway/proxy/main_test.go b/examples/gateway/proxy/main_test.go index 453f3e21b4..143ff4fae4 100644 --- a/examples/gateway/proxy/main_test.go +++ b/examples/gateway/proxy/main_test.go @@ -6,8 +6,8 @@ import ( "net/http/httptest" "testing" - "github.com/ipfs/go-blockservice" - offline "github.com/ipfs/go-ipfs-exchange-offline" + "github.com/ipfs/go-libipfs/blockservice" + offline "github.com/ipfs/go-libipfs/exchange/offline" "github.com/ipfs/go-libipfs/blocks" "github.com/ipfs/go-libipfs/examples/gateway/common" "github.com/stretchr/testify/assert" diff --git a/examples/gateway/proxy/routing.go b/examples/gateway/proxy/routing.go index 57d0c1492e..f5cf1df65b 100644 --- a/examples/gateway/proxy/routing.go +++ b/examples/gateway/proxy/routing.go @@ -9,8 +9,8 @@ import ( "strings" "github.com/gogo/protobuf/proto" - "github.com/ipfs/go-ipns" - ipns_pb "github.com/ipfs/go-ipns/pb" + "github.com/ipfs/go-libipfs/ipns" + ipns_pb "github.com/ipfs/go-libipfs/ipns/pb" ic "github.com/libp2p/go-libp2p/core/crypto" "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/core/routing" diff --git a/examples/unixfs-file-cid/main.go b/examples/unixfs-file-cid/main.go index 6ab3e96875..16f1159ff1 100644 --- a/examples/unixfs-file-cid/main.go +++ b/examples/unixfs-file-cid/main.go @@ -24,14 +24,14 @@ import ( "github.com/multiformats/go-multiaddr" "github.com/multiformats/go-multicodec" - "github.com/ipfs/go-blockservice" - blockstore "github.com/ipfs/go-ipfs-blockstore" - chunker "github.com/ipfs/go-ipfs-chunker" - offline "github.com/ipfs/go-ipfs-exchange-offline" + "github.com/ipfs/go-libipfs/blockservice" + blockstore "github.com/ipfs/go-libipfs/blockstore" + chunker "github.com/ipfs/go-libipfs/chunker" + offline "github.com/ipfs/go-libipfs/exchange/offline" "github.com/ipfs/go-merkledag" - unixfile "github.com/ipfs/go-unixfs/file" - "github.com/ipfs/go-unixfs/importer/balanced" - uih "github.com/ipfs/go-unixfs/importer/helpers" + unixfile "github.com/ipfs/go-libipfs/unixfs/file" + "github.com/ipfs/go-libipfs/unixfs/importer/balanced" + uih "github.com/ipfs/go-libipfs/unixfs/importer/helpers" "github.com/libp2p/go-libp2p-routing-helpers" bsclient "github.com/ipfs/go-libipfs/bitswap/client" diff --git a/exchange/interface.go b/exchange/interface.go new file mode 100644 index 0000000000..b7089399de --- /dev/null +++ b/exchange/interface.go @@ -0,0 +1,40 @@ +// Package exchange defines the IPFS exchange interface +package exchange + +import ( + "context" + "io" + + cid "github.com/ipfs/go-cid" + blocks "github.com/ipfs/go-libipfs/blocks" +) + +// Interface defines the functionality of the IPFS block exchange protocol. +type Interface interface { // type Exchanger interface + Fetcher + + // NotifyNewBlocks tells the exchange that new blocks are available and can be served. + NotifyNewBlocks(ctx context.Context, blocks ...blocks.Block) error + + io.Closer +} + +// Fetcher is an object that can be used to retrieve blocks +type Fetcher interface { + // GetBlock returns the block associated with a given cid. + GetBlock(context.Context, cid.Cid) (blocks.Block, error) + // GetBlocks returns the blocks associated with the given cids. + // If the requested blocks are not found immediately, this function should hang until + // they are found. If they can't be found later, it's also acceptable to terminate. + GetBlocks(context.Context, []cid.Cid) (<-chan blocks.Block, error) +} + +// SessionExchange is an exchange.Interface which supports +// sessions. +type SessionExchange interface { + Interface + // NewSession generates a new exchange session. You should use this, rather + // that calling GetBlocks, any time you intend to do several related calls + // in a row. The exchange can leverage that to be more efficient. + NewSession(context.Context) Fetcher +} diff --git a/exchange/offline/offline.go b/exchange/offline/offline.go new file mode 100644 index 0000000000..07aeee2265 --- /dev/null +++ b/exchange/offline/offline.go @@ -0,0 +1,74 @@ +// package offline implements an object that implements the exchange +// interface but returns nil values to every request. +package offline + +import ( + "context" + "fmt" + + cid "github.com/ipfs/go-cid" + ipld "github.com/ipfs/go-ipld-format" + blocks "github.com/ipfs/go-libipfs/blocks" + blockstore "github.com/ipfs/go-libipfs/blockstore" + exchange "github.com/ipfs/go-libipfs/exchange" +) + +func Exchange(bs blockstore.Blockstore) exchange.Interface { + return &offlineExchange{bs: bs} +} + +// offlineExchange implements the Exchange interface but doesn't return blocks. +// For use in offline mode. +type offlineExchange struct { + bs blockstore.Blockstore +} + +// GetBlock returns nil to signal that a block could not be retrieved for the +// given key. +// NB: This function may return before the timeout expires. +func (e *offlineExchange) GetBlock(ctx context.Context, k cid.Cid) (blocks.Block, error) { + blk, err := e.bs.Get(ctx, k) + if ipld.IsNotFound(err) { + return nil, fmt.Errorf("block was not found locally (offline): %w", err) + } + return blk, err +} + +// NotifyNewBlocks tells the exchange that new blocks are available and can be served. +func (e *offlineExchange) NotifyNewBlocks(ctx context.Context, blocks ...blocks.Block) error { + // as an offline exchange we have nothing to do + return nil +} + +// Close always returns nil. +func (e *offlineExchange) Close() error { + // NB: exchange doesn't own the blockstore's underlying datastore, so it is + // not responsible for closing it. + return nil +} + +func (e *offlineExchange) GetBlocks(ctx context.Context, ks []cid.Cid) (<-chan blocks.Block, error) { + out := make(chan blocks.Block) + go func() { + defer close(out) + for _, k := range ks { + hit, err := e.bs.Get(ctx, k) + if err != nil { + // a long line of misses should abort when context is cancelled. + select { + // TODO case send misses down channel + case <-ctx.Done(): + return + default: + continue + } + } + select { + case out <- hit: + case <-ctx.Done(): + return + } + } + }() + return out, nil +} diff --git a/exchange/offline/offline_test.go b/exchange/offline/offline_test.go new file mode 100644 index 0000000000..47f412efcd --- /dev/null +++ b/exchange/offline/offline_test.go @@ -0,0 +1,66 @@ +package offline + +import ( + "context" + "testing" + + cid "github.com/ipfs/go-cid" + ds "github.com/ipfs/go-datastore" + ds_sync "github.com/ipfs/go-datastore/sync" + blocksutil "github.com/ipfs/go-ipfs-blocksutil" + blockstore "github.com/ipfs/go-libipfs/blockstore" + u "github.com/ipfs/go-libipfs/util" +) + +func TestBlockReturnsErr(t *testing.T) { + off := Exchange(bstore()) + c := cid.NewCidV0(u.Hash([]byte("foo"))) + _, err := off.GetBlock(context.Background(), c) + if err != nil { + return // as desired + } + t.Fail() +} + +func TestGetBlocks(t *testing.T) { + store := bstore() + ex := Exchange(store) + g := blocksutil.NewBlockGenerator() + + expected := g.Blocks(2) + + for _, b := range expected { + if err := store.Put(context.Background(), b); err != nil { + t.Fatal(err) + } + if err := ex.NotifyNewBlocks(context.Background(), b); err != nil { + t.Fail() + } + } + + request := func() []cid.Cid { + var ks []cid.Cid + + for _, b := range expected { + ks = append(ks, b.Cid()) + } + return ks + }() + + received, err := ex.GetBlocks(context.Background(), request) + if err != nil { + t.Fatal(err) + } + + var count int + for range received { + count++ + } + if len(expected) != count { + t.Fail() + } +} + +func bstore() blockstore.Blockstore { + return blockstore.NewBlockstore(ds_sync.MutexWrap(ds.NewMapDatastore())) +} diff --git a/fetcher/.gitignore b/fetcher/.gitignore new file mode 100644 index 0000000000..485dee64bc --- /dev/null +++ b/fetcher/.gitignore @@ -0,0 +1 @@ +.idea diff --git a/fetcher/fetcher.go b/fetcher/fetcher.go new file mode 100644 index 0000000000..f332d9f121 --- /dev/null +++ b/fetcher/fetcher.go @@ -0,0 +1,51 @@ +package fetcher + +import ( + "context" + + "github.com/ipld/go-ipld-prime" +) + +// Fetcher is an interface for reading from a dag. Reads may be local or remote, and may employ data exchange +// protocols like graphsync and bitswap +type Fetcher interface { + // NodeMatching traverses a node graph starting with the provided root node using the given selector node and + // possibly crossing block boundaries. Each matched node is passed as FetchResult to the callback. Errors returned + // from callback will halt the traversal. The sequence of events is: NodeMatching begins, the callback is called zero + // or more times with a FetchResult, then NodeMatching returns. + NodeMatching(ctx context.Context, root ipld.Node, selector ipld.Node, cb FetchCallback) error + + // BlockOfType fetches a node graph of the provided type corresponding to single block by link. + BlockOfType(ctx context.Context, link ipld.Link, nodePrototype ipld.NodePrototype) (ipld.Node, error) + + // BlockMatchingOfType traverses a node graph starting with the given root link using the given selector node and + // possibly crossing block boundaries. The nodes will be typed using the provided prototype. Each matched node is + // passed as a FetchResult to the callback. Errors returned from callback will halt the traversal. + // The sequence of events is: BlockMatchingOfType begins, the callback is called zero or more times with a + // FetchResult, then BlockMatchingOfType returns. + BlockMatchingOfType( + ctx context.Context, + root ipld.Link, + selector ipld.Node, + nodePrototype ipld.NodePrototype, + cb FetchCallback) error + + // Uses the given link to pick a prototype to build the linked node. + PrototypeFromLink(link ipld.Link) (ipld.NodePrototype, error) +} + +// FetchResult is a single node read as part of a dag operation called on a fetcher +type FetchResult struct { + Node ipld.Node + Path ipld.Path + LastBlockPath ipld.Path + LastBlockLink ipld.Link +} + +// FetchCallback is called for each node traversed during a fetch +type FetchCallback func(result FetchResult) error + +// Factory is anything that can create new sessions of the fetcher +type Factory interface { + NewSession(ctx context.Context) Fetcher +} diff --git a/fetcher/helpers/block_visitor.go b/fetcher/helpers/block_visitor.go new file mode 100644 index 0000000000..1b82c80612 --- /dev/null +++ b/fetcher/helpers/block_visitor.go @@ -0,0 +1,43 @@ +package helpers + +import ( + "github.com/ipfs/go-cid" + "github.com/ipfs/go-libipfs/fetcher" + "github.com/ipld/go-ipld-prime" + cidlink "github.com/ipld/go-ipld-prime/linking/cid" +) + +// BlockResult specifies a node at the top of a block boundary +type BlockResult struct { + Node ipld.Node + Link ipld.Link +} + +// BlockCallback is a callback for visiting blocks +type BlockCallback func(BlockResult) error + +// OnBlocks produces a fetch call back that only gets called when visiting blocks during a fetch +func OnBlocks(bv BlockCallback) fetcher.FetchCallback { + return func(fr fetcher.FetchResult) error { + if fr.LastBlockPath.String() == fr.Path.String() { + return bv(BlockResult{ + Node: fr.Node, + Link: fr.LastBlockLink, + }) + } + return nil + } +} + +// OnUniqueBlocks is a callback that only gets called visiting each block once +func OnUniqueBlocks(bv BlockCallback) fetcher.FetchCallback { + set := cid.NewSet() + return OnBlocks(func(br BlockResult) error { + c := br.Link.(cidlink.Link).Cid + if set.Has(c) { + return nil + } + set.Add(c) + return bv(br) + }) +} diff --git a/fetcher/helpers/block_visitor_test.go b/fetcher/helpers/block_visitor_test.go new file mode 100644 index 0000000000..cc7b45d794 --- /dev/null +++ b/fetcher/helpers/block_visitor_test.go @@ -0,0 +1,142 @@ +package helpers_test + +import ( + "context" + "testing" + "time" + + delay "github.com/ipfs/go-ipfs-delay" + testinstance "github.com/ipfs/go-libipfs/bitswap/testinstance" + tn "github.com/ipfs/go-libipfs/bitswap/testnet" + "github.com/ipfs/go-libipfs/blockservice" + "github.com/ipfs/go-libipfs/fetcher/helpers" + bsfetcher "github.com/ipfs/go-libipfs/fetcher/impl/blockservice" + "github.com/ipfs/go-libipfs/fetcher/testutil" + mockrouting "github.com/ipfs/go-libipfs/routing/mock" + "github.com/ipld/go-ipld-prime" + "github.com/ipld/go-ipld-prime/fluent" + cidlink "github.com/ipld/go-ipld-prime/linking/cid" + basicnode "github.com/ipld/go-ipld-prime/node/basic" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +var bg = context.Background() + +func TestFetchGraphToBlocks(t *testing.T) { + block3, node3, link3 := testutil.EncodeBlock(fluent.MustBuildMap(basicnode.Prototype__Map{}, 1, func(na fluent.MapAssembler) { + na.AssembleEntry("three").AssignBool(true) + })) + block4, node4, link4 := testutil.EncodeBlock(fluent.MustBuildMap(basicnode.Prototype__Map{}, 1, func(na fluent.MapAssembler) { + na.AssembleEntry("four").AssignBool(true) + })) + block2, node2, link2 := testutil.EncodeBlock(fluent.MustBuildMap(basicnode.Prototype__Map{}, 2, func(na fluent.MapAssembler) { + na.AssembleEntry("link3").AssignLink(link3) + na.AssembleEntry("link4").AssignLink(link4) + })) + block1, node1, _ := testutil.EncodeBlock(fluent.MustBuildMap(basicnode.Prototype__Map{}, 3, func(na fluent.MapAssembler) { + na.AssembleEntry("foo").AssignBool(true) + na.AssembleEntry("bar").AssignBool(false) + na.AssembleEntry("nested").CreateMap(2, func(na fluent.MapAssembler) { + na.AssembleEntry("link2").AssignLink(link2) + na.AssembleEntry("nonlink").AssignString("zoo") + }) + })) + + net := tn.VirtualNetwork(mockrouting.NewServer(), delay.Fixed(0*time.Millisecond)) + ig := testinstance.NewTestInstanceGenerator(net, nil, nil) + defer ig.Close() + + peers := ig.Instances(2) + hasBlock := peers[0] + defer hasBlock.Exchange.Close() + + err := hasBlock.Exchange.HasBlock(bg, block1) + require.NoError(t, err) + err = hasBlock.Exchange.HasBlock(bg, block2) + require.NoError(t, err) + err = hasBlock.Exchange.HasBlock(bg, block3) + require.NoError(t, err) + err = hasBlock.Exchange.HasBlock(bg, block4) + require.NoError(t, err) + + wantsBlock := peers[1] + defer wantsBlock.Exchange.Close() + + wantsGetter := blockservice.New(wantsBlock.Blockstore(), wantsBlock.Exchange) + fetcherConfig := bsfetcher.NewFetcherConfig(wantsGetter) + session := fetcherConfig.NewSession(context.Background()) + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + + results := []helpers.BlockResult{} + err = helpers.BlockAll(ctx, session, cidlink.Link{Cid: block1.Cid()}, helpers.OnBlocks(func(res helpers.BlockResult) error { + results = append(results, res) + return nil + })) + require.NoError(t, err) + + assertBlocksInOrder(t, results, 4, map[int]ipld.Node{0: node1, 1: node2, 2: node3, 3: node4}) +} + +func TestFetchGraphToUniqueBlocks(t *testing.T) { + block3, node3, link3 := testutil.EncodeBlock(fluent.MustBuildMap(basicnode.Prototype__Map{}, 1, func(na fluent.MapAssembler) { + na.AssembleEntry("three").AssignBool(true) + })) + block2, node2, link2 := testutil.EncodeBlock(fluent.MustBuildMap(basicnode.Prototype__Map{}, 2, func(na fluent.MapAssembler) { + na.AssembleEntry("link3").AssignLink(link3) + })) + block1, node1, _ := testutil.EncodeBlock(fluent.MustBuildMap(basicnode.Prototype__Map{}, 3, func(na fluent.MapAssembler) { + na.AssembleEntry("foo").AssignBool(true) + na.AssembleEntry("bar").AssignBool(false) + na.AssembleEntry("nested").CreateMap(2, func(na fluent.MapAssembler) { + na.AssembleEntry("link2").AssignLink(link2) + na.AssembleEntry("link3").AssignLink(link3) + na.AssembleEntry("nonlink").AssignString("zoo") + }) + })) + + net := tn.VirtualNetwork(mockrouting.NewServer(), delay.Fixed(0*time.Millisecond)) + ig := testinstance.NewTestInstanceGenerator(net, nil, nil) + defer ig.Close() + + peers := ig.Instances(2) + hasBlock := peers[0] + defer hasBlock.Exchange.Close() + + err := hasBlock.Exchange.HasBlock(bg, block1) + require.NoError(t, err) + err = hasBlock.Exchange.HasBlock(bg, block2) + require.NoError(t, err) + err = hasBlock.Exchange.HasBlock(bg, block3) + require.NoError(t, err) + + wantsBlock := peers[1] + defer wantsBlock.Exchange.Close() + + wantsGetter := blockservice.New(wantsBlock.Blockstore(), wantsBlock.Exchange) + fetcherConfig := bsfetcher.NewFetcherConfig(wantsGetter) + session := fetcherConfig.NewSession(context.Background()) + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + + results := []helpers.BlockResult{} + err = helpers.BlockAll(ctx, session, cidlink.Link{Cid: block1.Cid()}, helpers.OnUniqueBlocks(func(res helpers.BlockResult) error { + results = append(results, res) + return nil + })) + require.NoError(t, err) + + assertBlocksInOrder(t, results, 3, map[int]ipld.Node{0: node1, 1: node2, 2: node3}) +} + +func assertBlocksInOrder(t *testing.T, results []helpers.BlockResult, nodeCount int, nodes map[int]ipld.Node) { + for order, res := range results { + expectedNode, ok := nodes[order] + if ok { + assert.Equal(t, expectedNode, res.Node) + } + } + + assert.Equal(t, nodeCount, len(results)) +} diff --git a/fetcher/helpers/traversal.go b/fetcher/helpers/traversal.go new file mode 100644 index 0000000000..9916d841e4 --- /dev/null +++ b/fetcher/helpers/traversal.go @@ -0,0 +1,42 @@ +package helpers + +import ( + "context" + + "github.com/ipfs/go-libipfs/fetcher" + "github.com/ipld/go-ipld-prime" + basicnode "github.com/ipld/go-ipld-prime/node/basic" + "github.com/ipld/go-ipld-prime/traversal/selector" + "github.com/ipld/go-ipld-prime/traversal/selector/builder" +) + +var matchAllSelector ipld.Node + +func init() { + ssb := builder.NewSelectorSpecBuilder(basicnode.Prototype.Any) + matchAllSelector = ssb.ExploreRecursive(selector.RecursionLimitNone(), ssb.ExploreUnion( + ssb.Matcher(), + ssb.ExploreAll(ssb.ExploreRecursiveEdge()), + )).Node() +} + +// Block fetches a schemaless node graph corresponding to single block by link. +func Block(ctx context.Context, f fetcher.Fetcher, link ipld.Link) (ipld.Node, error) { + prototype, err := f.PrototypeFromLink(link) + if err != nil { + return nil, err + } + return f.BlockOfType(ctx, link, prototype) +} + +// BlockMatching traverses a schemaless node graph starting with the given link using the given selector and possibly crossing +// block boundaries. Each matched node is sent to the FetchResult channel. +func BlockMatching(ctx context.Context, f fetcher.Fetcher, root ipld.Link, match ipld.Node, cb fetcher.FetchCallback) error { + return f.BlockMatchingOfType(ctx, root, match, nil, cb) +} + +// BlockAll traverses all nodes in the graph linked by root. The nodes will be untyped and send over the results +// channel. +func BlockAll(ctx context.Context, f fetcher.Fetcher, root ipld.Link, cb fetcher.FetchCallback) error { + return f.BlockMatchingOfType(ctx, root, matchAllSelector, nil, cb) +} diff --git a/fetcher/impl/blockservice/fetcher.go b/fetcher/impl/blockservice/fetcher.go new file mode 100644 index 0000000000..50c5808232 --- /dev/null +++ b/fetcher/impl/blockservice/fetcher.go @@ -0,0 +1,148 @@ +package bsfetcher + +import ( + "bytes" + "context" + "fmt" + "io" + + "github.com/ipfs/go-libipfs/blockservice" + "github.com/ipfs/go-libipfs/fetcher" + "github.com/ipld/go-ipld-prime" + cidlink "github.com/ipld/go-ipld-prime/linking/cid" + basicnode "github.com/ipld/go-ipld-prime/node/basic" + "github.com/ipld/go-ipld-prime/schema" + "github.com/ipld/go-ipld-prime/traversal" + "github.com/ipld/go-ipld-prime/traversal/selector" +) + +type fetcherSession struct { + linkSystem ipld.LinkSystem + protoChooser traversal.LinkTargetNodePrototypeChooser +} + +// FetcherConfig defines a configuration object from which Fetcher instances are constructed +type FetcherConfig struct { + blockService blockservice.BlockService + NodeReifier ipld.NodeReifier + PrototypeChooser traversal.LinkTargetNodePrototypeChooser +} + +// NewFetcherConfig creates a FetchConfig from which session may be created and nodes retrieved. +func NewFetcherConfig(blockService blockservice.BlockService) FetcherConfig { + return FetcherConfig{ + blockService: blockService, + PrototypeChooser: DefaultPrototypeChooser, + } +} + +// NewSession creates a session from which nodes may be retrieved. +// The session ends when the provided context is canceled. +func (fc FetcherConfig) NewSession(ctx context.Context) fetcher.Fetcher { + return fc.FetcherWithSession(ctx, blockservice.NewSession(ctx, fc.blockService)) +} + +func (fc FetcherConfig) FetcherWithSession(ctx context.Context, s *blockservice.Session) fetcher.Fetcher { + ls := cidlink.DefaultLinkSystem() + // while we may be loading blocks remotely, they are already hash verified by the time they load + // into ipld-prime + ls.TrustedStorage = true + ls.StorageReadOpener = blockOpener(ctx, s) + ls.NodeReifier = fc.NodeReifier + + protoChooser := fc.PrototypeChooser + return &fetcherSession{linkSystem: ls, protoChooser: protoChooser} +} + +// WithReifier derives a different fetcher factory from the same source but +// with a chosen NodeReifier for pathing semantics. +func (fc FetcherConfig) WithReifier(nr ipld.NodeReifier) fetcher.Factory { + return FetcherConfig{ + blockService: fc.blockService, + NodeReifier: nr, + PrototypeChooser: fc.PrototypeChooser, + } +} + +// interface check +var _ fetcher.Factory = FetcherConfig{} + +// BlockOfType fetches a node graph of the provided type corresponding to single block by link. +func (f *fetcherSession) BlockOfType(ctx context.Context, link ipld.Link, ptype ipld.NodePrototype) (ipld.Node, error) { + return f.linkSystem.Load(ipld.LinkContext{}, link, ptype) +} + +func (f *fetcherSession) nodeMatching(ctx context.Context, initialProgress traversal.Progress, node ipld.Node, match ipld.Node, cb fetcher.FetchCallback) error { + matchSelector, err := selector.ParseSelector(match) + if err != nil { + return err + } + return initialProgress.WalkMatching(node, matchSelector, func(prog traversal.Progress, n ipld.Node) error { + return cb(fetcher.FetchResult{ + Node: n, + Path: prog.Path, + LastBlockPath: prog.LastBlock.Path, + LastBlockLink: prog.LastBlock.Link, + }) + }) +} + +func (f *fetcherSession) blankProgress(ctx context.Context) traversal.Progress { + return traversal.Progress{ + Cfg: &traversal.Config{ + LinkSystem: f.linkSystem, + LinkTargetNodePrototypeChooser: f.protoChooser, + }, + } +} + +func (f *fetcherSession) NodeMatching(ctx context.Context, node ipld.Node, match ipld.Node, cb fetcher.FetchCallback) error { + return f.nodeMatching(ctx, f.blankProgress(ctx), node, match, cb) +} + +func (f *fetcherSession) BlockMatchingOfType(ctx context.Context, root ipld.Link, match ipld.Node, + _ ipld.NodePrototype, cb fetcher.FetchCallback) error { + + // retrieve first node + prototype, err := f.PrototypeFromLink(root) + if err != nil { + return err + } + node, err := f.BlockOfType(ctx, root, prototype) + if err != nil { + return err + } + + progress := f.blankProgress(ctx) + progress.LastBlock.Link = root + return f.nodeMatching(ctx, progress, node, match, cb) +} + +func (f *fetcherSession) PrototypeFromLink(lnk ipld.Link) (ipld.NodePrototype, error) { + return f.protoChooser(lnk, ipld.LinkContext{}) +} + +// DefaultPrototypeChooser supports choosing the prototype from the link and falling +// back to a basicnode.Any builder +var DefaultPrototypeChooser = func(lnk ipld.Link, lnkCtx ipld.LinkContext) (ipld.NodePrototype, error) { + if tlnkNd, ok := lnkCtx.LinkNode.(schema.TypedLinkNode); ok { + return tlnkNd.LinkTargetNodePrototype(), nil + } + return basicnode.Prototype.Any, nil +} + +func blockOpener(ctx context.Context, bs *blockservice.Session) ipld.BlockReadOpener { + return func(_ ipld.LinkContext, lnk ipld.Link) (io.Reader, error) { + cidLink, ok := lnk.(cidlink.Link) + if !ok { + return nil, fmt.Errorf("invalid link type for loading: %v", lnk) + } + + blk, err := bs.GetBlock(ctx, cidLink.Cid) + if err != nil { + return nil, err + } + + return bytes.NewReader(blk.RawData()), nil + } +} diff --git a/fetcher/impl/blockservice/fetcher_test.go b/fetcher/impl/blockservice/fetcher_test.go new file mode 100644 index 0000000000..04b724fb4e --- /dev/null +++ b/fetcher/impl/blockservice/fetcher_test.go @@ -0,0 +1,369 @@ +package bsfetcher_test + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/ipld/go-ipld-prime/traversal/selector" + "github.com/ipld/go-ipld-prime/traversal/selector/builder" + + delay "github.com/ipfs/go-ipfs-delay" + testinstance "github.com/ipfs/go-libipfs/bitswap/testinstance" + tn "github.com/ipfs/go-libipfs/bitswap/testnet" + blocks "github.com/ipfs/go-libipfs/blocks" + "github.com/ipfs/go-libipfs/blockservice" + "github.com/ipfs/go-libipfs/fetcher" + "github.com/ipfs/go-libipfs/fetcher/helpers" + bsfetcher "github.com/ipfs/go-libipfs/fetcher/impl/blockservice" + "github.com/ipfs/go-libipfs/fetcher/testutil" + mockrouting "github.com/ipfs/go-libipfs/routing/mock" + "github.com/ipld/go-ipld-prime" + "github.com/ipld/go-ipld-prime/fluent" + cidlink "github.com/ipld/go-ipld-prime/linking/cid" + basicnode "github.com/ipld/go-ipld-prime/node/basic" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +var bg = context.Background() + +func TestFetchIPLDPrimeNode(t *testing.T) { + block, node, _ := testutil.EncodeBlock(fluent.MustBuildMap(basicnode.Prototype__Map{}, 3, func(na fluent.MapAssembler) { + na.AssembleEntry("foo").AssignBool(true) + na.AssembleEntry("bar").AssignBool(false) + na.AssembleEntry("nested").CreateMap(2, func(na fluent.MapAssembler) { + na.AssembleEntry("nonlink").AssignString("zoo") + }) + })) + + net := tn.VirtualNetwork(mockrouting.NewServer(), delay.Fixed(0*time.Millisecond)) + ig := testinstance.NewTestInstanceGenerator(net, nil, nil) + defer ig.Close() + + peers := ig.Instances(2) + hasBlock := peers[0] + defer hasBlock.Exchange.Close() + + err := hasBlock.Exchange.HasBlock(bg, block) + require.NoError(t, err) + + wantsBlock := peers[1] + defer wantsBlock.Exchange.Close() + + wantsGetter := blockservice.New(wantsBlock.Blockstore(), wantsBlock.Exchange) + fetcherConfig := bsfetcher.NewFetcherConfig(wantsGetter) + session := fetcherConfig.NewSession(context.Background()) + + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + + retrievedNode, err := helpers.Block(ctx, session, cidlink.Link{Cid: block.Cid()}) + require.NoError(t, err) + assert.Equal(t, node, retrievedNode) +} + +func TestFetchIPLDGraph(t *testing.T) { + block3, node3, link3 := testutil.EncodeBlock(fluent.MustBuildMap(basicnode.Prototype__Map{}, 1, func(na fluent.MapAssembler) { + na.AssembleEntry("three").AssignBool(true) + })) + block4, node4, link4 := testutil.EncodeBlock(fluent.MustBuildMap(basicnode.Prototype__Map{}, 1, func(na fluent.MapAssembler) { + na.AssembleEntry("four").AssignBool(true) + })) + block2, node2, link2 := testutil.EncodeBlock(fluent.MustBuildMap(basicnode.Prototype__Map{}, 2, func(na fluent.MapAssembler) { + na.AssembleEntry("link3").AssignLink(link3) + na.AssembleEntry("link4").AssignLink(link4) + })) + block1, node1, _ := testutil.EncodeBlock(fluent.MustBuildMap(basicnode.Prototype__Map{}, 3, func(na fluent.MapAssembler) { + na.AssembleEntry("foo").AssignBool(true) + na.AssembleEntry("bar").AssignBool(false) + na.AssembleEntry("nested").CreateMap(2, func(na fluent.MapAssembler) { + na.AssembleEntry("link2").AssignLink(link2) + na.AssembleEntry("nonlink").AssignString("zoo") + }) + })) + + net := tn.VirtualNetwork(mockrouting.NewServer(), delay.Fixed(0*time.Millisecond)) + ig := testinstance.NewTestInstanceGenerator(net, nil, nil) + defer ig.Close() + + peers := ig.Instances(2) + hasBlock := peers[0] + defer hasBlock.Exchange.Close() + + err := hasBlock.Exchange.HasBlock(bg, block1) + require.NoError(t, err) + err = hasBlock.Exchange.HasBlock(bg, block2) + require.NoError(t, err) + err = hasBlock.Exchange.HasBlock(bg, block3) + require.NoError(t, err) + err = hasBlock.Exchange.HasBlock(bg, block4) + require.NoError(t, err) + + wantsBlock := peers[1] + defer wantsBlock.Exchange.Close() + + wantsGetter := blockservice.New(wantsBlock.Blockstore(), wantsBlock.Exchange) + fetcherConfig := bsfetcher.NewFetcherConfig(wantsGetter) + session := fetcherConfig.NewSession(context.Background()) + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + + results := []fetcher.FetchResult{} + err = helpers.BlockAll(ctx, session, cidlink.Link{Cid: block1.Cid()}, func(res fetcher.FetchResult) error { + results = append(results, res) + return nil + }) + require.NoError(t, err) + + assertNodesInOrder(t, results, 10, map[int]ipld.Node{0: node1, 4: node2, 5: node3, 7: node4}) +} + +func TestFetchIPLDPath(t *testing.T) { + block5, node5, link5 := testutil.EncodeBlock(fluent.MustBuildMap(basicnode.Prototype__Map{}, 1, func(na fluent.MapAssembler) { + na.AssembleEntry("five").AssignBool(true) + })) + block3, _, link3 := testutil.EncodeBlock(fluent.MustBuildMap(basicnode.Prototype__Map{}, 1, func(na fluent.MapAssembler) { + na.AssembleEntry("three").AssignLink(link5) + })) + block4, _, link4 := testutil.EncodeBlock(fluent.MustBuildMap(basicnode.Prototype__Map{}, 1, func(na fluent.MapAssembler) { + na.AssembleEntry("four").AssignBool(true) + })) + block2, _, link2 := testutil.EncodeBlock(fluent.MustBuildMap(basicnode.Prototype__Map{}, 2, func(na fluent.MapAssembler) { + na.AssembleEntry("link3").AssignLink(link3) + na.AssembleEntry("link4").AssignLink(link4) + })) + block1, _, _ := testutil.EncodeBlock(fluent.MustBuildMap(basicnode.Prototype__Map{}, 3, func(na fluent.MapAssembler) { + na.AssembleEntry("foo").AssignBool(true) + na.AssembleEntry("bar").AssignBool(false) + na.AssembleEntry("nested").CreateMap(2, func(na fluent.MapAssembler) { + na.AssembleEntry("link2").AssignLink(link2) + na.AssembleEntry("nonlink").AssignString("zoo") + }) + })) + + net := tn.VirtualNetwork(mockrouting.NewServer(), delay.Fixed(0*time.Millisecond)) + ig := testinstance.NewTestInstanceGenerator(net, nil, nil) + defer ig.Close() + + peers := ig.Instances(2) + hasBlock := peers[0] + defer hasBlock.Exchange.Close() + + for _, blk := range []blocks.Block{block1, block2, block3, block4, block5} { + err := hasBlock.Exchange.HasBlock(bg, blk) + require.NoError(t, err) + } + + wantsBlock := peers[1] + defer wantsBlock.Exchange.Close() + + wantsGetter := blockservice.New(wantsBlock.Blockstore(), wantsBlock.Exchange) + fetcherConfig := bsfetcher.NewFetcherConfig(wantsGetter) + session := fetcherConfig.NewSession(context.Background()) + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + + path := strings.Split("nested/link2/link3/three", "/") + ssb := builder.NewSelectorSpecBuilder(basicnode.Prototype.Any) + spec := ssb.Matcher() + explorePath := func(p string, s builder.SelectorSpec) builder.SelectorSpec { + return ssb.ExploreFields(func(efsb builder.ExploreFieldsSpecBuilder) { efsb.Insert(p, s) }) + } + for i := len(path) - 1; i >= 0; i-- { + spec = explorePath(path[i], spec) + } + sel := spec.Node() + + results := []fetcher.FetchResult{} + err := helpers.BlockMatching(ctx, session, cidlink.Link{Cid: block1.Cid()}, sel, func(res fetcher.FetchResult) error { + results = append(results, res) + return nil + }) + require.NoError(t, err) + + assertNodesInOrder(t, results, 1, map[int]ipld.Node{0: node5}) +} + +func TestHelpers(t *testing.T) { + block3, node3, link3 := testutil.EncodeBlock(fluent.MustBuildMap(basicnode.Prototype__Map{}, 1, func(na fluent.MapAssembler) { + na.AssembleEntry("three").AssignBool(true) + })) + block4, node4, link4 := testutil.EncodeBlock(fluent.MustBuildMap(basicnode.Prototype__Map{}, 1, func(na fluent.MapAssembler) { + na.AssembleEntry("four").AssignBool(true) + })) + block2, node2, link2 := testutil.EncodeBlock(fluent.MustBuildMap(basicnode.Prototype__Map{}, 2, func(na fluent.MapAssembler) { + na.AssembleEntry("link3").AssignLink(link3) + na.AssembleEntry("link4").AssignLink(link4) + })) + block1, node1, _ := testutil.EncodeBlock(fluent.MustBuildMap(basicnode.Prototype__Map{}, 3, func(na fluent.MapAssembler) { + na.AssembleEntry("foo").AssignBool(true) + na.AssembleEntry("bar").AssignBool(false) + na.AssembleEntry("nested").CreateMap(2, func(na fluent.MapAssembler) { + na.AssembleEntry("link2").AssignLink(link2) + na.AssembleEntry("nonlink").AssignString("zoo") + }) + })) + + net := tn.VirtualNetwork(mockrouting.NewServer(), delay.Fixed(0*time.Millisecond)) + ig := testinstance.NewTestInstanceGenerator(net, nil, nil) + defer ig.Close() + + peers := ig.Instances(2) + hasBlock := peers[0] + defer hasBlock.Exchange.Close() + + err := hasBlock.Exchange.HasBlock(bg, block1) + require.NoError(t, err) + err = hasBlock.Exchange.HasBlock(bg, block2) + require.NoError(t, err) + err = hasBlock.Exchange.HasBlock(bg, block3) + require.NoError(t, err) + err = hasBlock.Exchange.HasBlock(bg, block4) + require.NoError(t, err) + + wantsBlock := peers[1] + defer wantsBlock.Exchange.Close() + + wantsGetter := blockservice.New(wantsBlock.Blockstore(), wantsBlock.Exchange) + + t.Run("Block retrieves node", func(t *testing.T) { + fetcherConfig := bsfetcher.NewFetcherConfig(wantsGetter) + session := fetcherConfig.NewSession(context.Background()) + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + + node, err := helpers.Block(ctx, session, cidlink.Link{Cid: block1.Cid()}) + require.NoError(t, err) + + assert.Equal(t, node, node1) + }) + + t.Run("BlockMatching retrieves nodes matching selector", func(t *testing.T) { + // limit recursion depth to 2 nodes and expect to get only 2 blocks (4 nodes) + ssb := builder.NewSelectorSpecBuilder(basicnode.Prototype__Any{}) + sel := ssb.ExploreRecursive(selector.RecursionLimitDepth(2), ssb.ExploreUnion( + ssb.Matcher(), + ssb.ExploreAll(ssb.ExploreRecursiveEdge()), + )).Node() + + fetcherConfig := bsfetcher.NewFetcherConfig(wantsGetter) + session := fetcherConfig.NewSession(context.Background()) + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + + results := []fetcher.FetchResult{} + err = helpers.BlockMatching(ctx, session, cidlink.Link{Cid: block1.Cid()}, sel, func(res fetcher.FetchResult) error { + results = append(results, res) + return nil + }) + require.NoError(t, err) + + assertNodesInOrder(t, results, 4, map[int]ipld.Node{0: node1, 4: node2}) + }) + + t.Run("BlockAllOfType retrieves all nodes with a schema", func(t *testing.T) { + // limit recursion depth to 2 nodes and expect to get only 2 blocks (4 nodes) + fetcherConfig := bsfetcher.NewFetcherConfig(wantsGetter) + session := fetcherConfig.NewSession(context.Background()) + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + + results := []fetcher.FetchResult{} + err = helpers.BlockAll(ctx, session, cidlink.Link{Cid: block1.Cid()}, func(res fetcher.FetchResult) error { + results = append(results, res) + return nil + }) + require.NoError(t, err) + + assertNodesInOrder(t, results, 10, map[int]ipld.Node{0: node1, 4: node2, 5: node3, 7: node4}) + }) +} + +func assertNodesInOrder(t *testing.T, results []fetcher.FetchResult, nodeCount int, nodes map[int]ipld.Node) { + for order, res := range results { + expectedNode, ok := nodes[order] + if ok { + assert.Equal(t, expectedNode, res.Node) + } + } + + assert.Equal(t, nodeCount, len(results)) +} + +type selfLoader struct { + ipld.Node + ctx context.Context + ls *ipld.LinkSystem +} + +func (sl *selfLoader) LookupByString(key string) (ipld.Node, error) { + nd, err := sl.Node.LookupByString(key) + if err != nil { + return nd, err + } + if nd.Kind() == ipld.Kind_Link { + lnk, _ := nd.AsLink() + nd, err = sl.ls.Load(ipld.LinkContext{Ctx: sl.ctx}, lnk, basicnode.Prototype.Any) + } + return nd, err +} + +func TestNodeReification(t *testing.T) { + // demonstrates how to use the augment chooser to build an ADL that self loads its own nodes + block3, node3, link3 := testutil.EncodeBlock(fluent.MustBuildMap(basicnode.Prototype__Map{}, 1, func(na fluent.MapAssembler) { + na.AssembleEntry("three").AssignBool(true) + })) + block4, node4, link4 := testutil.EncodeBlock(fluent.MustBuildMap(basicnode.Prototype__Map{}, 1, func(na fluent.MapAssembler) { + na.AssembleEntry("four").AssignBool(true) + })) + block2, _, _ := testutil.EncodeBlock(fluent.MustBuildMap(basicnode.Prototype__Map{}, 2, func(na fluent.MapAssembler) { + na.AssembleEntry("link3").AssignLink(link3) + na.AssembleEntry("link4").AssignLink(link4) + })) + + net := tn.VirtualNetwork(mockrouting.NewServer(), delay.Fixed(0*time.Millisecond)) + ig := testinstance.NewTestInstanceGenerator(net, nil, nil) + defer ig.Close() + + peers := ig.Instances(2) + hasBlock := peers[0] + defer hasBlock.Exchange.Close() + + err := hasBlock.Exchange.HasBlock(bg, block2) + require.NoError(t, err) + err = hasBlock.Exchange.HasBlock(bg, block3) + require.NoError(t, err) + err = hasBlock.Exchange.HasBlock(bg, block4) + require.NoError(t, err) + + wantsBlock := peers[1] + defer wantsBlock.Exchange.Close() + + wantsGetter := blockservice.New(wantsBlock.Blockstore(), wantsBlock.Exchange) + fetcherConfig := bsfetcher.NewFetcherConfig(wantsGetter) + nodeReifier := func(lnkCtx ipld.LinkContext, nd ipld.Node, ls *ipld.LinkSystem) (ipld.Node, error) { + return &selfLoader{Node: nd, ctx: lnkCtx.Ctx, ls: ls}, nil + } + fetcherConfig.NodeReifier = nodeReifier + session := fetcherConfig.NewSession(context.Background()) + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + + retrievedNode, err := helpers.Block(ctx, session, cidlink.Link{Cid: block2.Cid()}) + require.NoError(t, err) + + // instead of getting links back, we automatically load the nodes + + retrievedNode3, err := retrievedNode.LookupByString("link3") + require.NoError(t, err) + underlying3 := retrievedNode3.(*selfLoader).Node + assert.Equal(t, node3, underlying3) + + retrievedNode4, err := retrievedNode.LookupByString("link4") + require.NoError(t, err) + underlying4 := retrievedNode4.(*selfLoader).Node + assert.Equal(t, node4, underlying4) + +} diff --git a/fetcher/testutil/testutil.go b/fetcher/testutil/testutil.go new file mode 100644 index 0000000000..1478f5ec9e --- /dev/null +++ b/fetcher/testutil/testutil.go @@ -0,0 +1,51 @@ +package testutil + +import ( + "bytes" + "fmt" + "io" + + "github.com/ipfs/go-cid" + blocks "github.com/ipfs/go-libipfs/blocks" + "github.com/ipld/go-ipld-prime" + + // used to make sure we have dagcbor encoding + _ "github.com/ipld/go-ipld-prime/codec/dagcbor" + cidlink "github.com/ipld/go-ipld-prime/linking/cid" +) + +// EncodeBlock produces an encoded block from a node +func EncodeBlock(n ipld.Node) (blocks.Block, ipld.Node, ipld.Link) { + ls := cidlink.DefaultLinkSystem() + var b blocks.Block + lb := cidlink.LinkPrototype{Prefix: cid.Prefix{ + Version: 1, + Codec: 0x71, + MhType: 0x17, + MhLength: 20, + }} + ls.StorageReadOpener = func(ipld.LinkContext, ipld.Link) (io.Reader, error) { + return bytes.NewReader(b.RawData()), nil + } + ls.StorageWriteOpener = func(ipld.LinkContext) (io.Writer, ipld.BlockWriteCommitter, error) { + buf := bytes.Buffer{} + return &buf, func(lnk ipld.Link) error { + clnk, ok := lnk.(cidlink.Link) + if !ok { + return fmt.Errorf("incorrect link type %v", lnk) + } + var err error + b, err = blocks.NewBlockWithCid(buf.Bytes(), clnk.Cid) + return err + }, nil + } + lnk, err := ls.Store(ipld.LinkContext{}, lb, n) + if err != nil { + panic(err) + } + ln, err := ls.Load(ipld.LinkContext{}, lnk, n.Prototype()) + if err != nil { + panic(err) + } + return b, ln, lnk +} diff --git a/filestore/filestore.go b/filestore/filestore.go new file mode 100644 index 0000000000..2623504af2 --- /dev/null +++ b/filestore/filestore.go @@ -0,0 +1,245 @@ +// Package filestore implements a Blockstore which is able to read certain +// blocks of data directly from its original location in the filesystem. +// +// In a Filestore, object leaves are stored as FilestoreNodes. FilestoreNodes +// include a filesystem path and an offset, allowing a Blockstore dealing with +// such blocks to avoid storing the whole contents and reading them from their +// filesystem location instead. +package filestore + +import ( + "context" + "errors" + + cid "github.com/ipfs/go-cid" + dsq "github.com/ipfs/go-datastore/query" + ipld "github.com/ipfs/go-ipld-format" + blocks "github.com/ipfs/go-libipfs/blocks" + blockstore "github.com/ipfs/go-libipfs/blockstore" + posinfo "github.com/ipfs/go-libipfs/filestore/posinfo" + logging "github.com/ipfs/go-log" +) + +var logger = logging.Logger("filestore") + +var ErrFilestoreNotEnabled = errors.New("filestore is not enabled, see https://git.io/vNItf") +var ErrUrlstoreNotEnabled = errors.New("urlstore is not enabled") + +// Filestore implements a Blockstore by combining a standard Blockstore +// to store regular blocks and a special Blockstore called +// FileManager to store blocks which data exists in an external file. +type Filestore struct { + fm *FileManager + bs blockstore.Blockstore +} + +// FileManager returns the FileManager in Filestore. +func (f *Filestore) FileManager() *FileManager { + return f.fm +} + +// MainBlockstore returns the standard Blockstore in the Filestore. +func (f *Filestore) MainBlockstore() blockstore.Blockstore { + return f.bs +} + +// NewFilestore creates one using the given Blockstore and FileManager. +func NewFilestore(bs blockstore.Blockstore, fm *FileManager) *Filestore { + return &Filestore{fm, bs} +} + +// AllKeysChan returns a channel from which to read the keys stored in +// the blockstore. If the given context is cancelled the channel will be closed. +func (f *Filestore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) { + ctx, cancel := context.WithCancel(ctx) + + a, err := f.bs.AllKeysChan(ctx) + if err != nil { + cancel() + return nil, err + } + + out := make(chan cid.Cid, dsq.KeysOnlyBufSize) + go func() { + defer cancel() + defer close(out) + + var done bool + for !done { + select { + case c, ok := <-a: + if !ok { + done = true + continue + } + select { + case out <- c: + case <-ctx.Done(): + return + } + case <-ctx.Done(): + return + } + } + + // Can't do these at the same time because the abstractions around + // leveldb make us query leveldb for both operations. We apparently + // cant query leveldb concurrently + b, err := f.fm.AllKeysChan(ctx) + if err != nil { + logger.Error("error querying filestore: ", err) + return + } + + done = false + for !done { + select { + case c, ok := <-b: + if !ok { + done = true + continue + } + select { + case out <- c: + case <-ctx.Done(): + return + } + case <-ctx.Done(): + return + } + } + }() + return out, nil +} + +// DeleteBlock deletes the block with the given key from the +// blockstore. As expected, in the case of FileManager blocks, only the +// reference is deleted, not its contents. It may return +// ErrNotFound when the block is not stored. +func (f *Filestore) DeleteBlock(ctx context.Context, c cid.Cid) error { + err1 := f.bs.DeleteBlock(ctx, c) + if err1 != nil && !ipld.IsNotFound(err1) { + return err1 + } + + err2 := f.fm.DeleteBlock(ctx, c) + + // if we successfully removed something from the blockstore, but the + // filestore didnt have it, return success + if !ipld.IsNotFound(err2) { + return err2 + } + + if ipld.IsNotFound(err1) { + return err1 + } + + return nil +} + +// Get retrieves the block with the given Cid. It may return +// ErrNotFound when the block is not stored. +func (f *Filestore) Get(ctx context.Context, c cid.Cid) (blocks.Block, error) { + blk, err := f.bs.Get(ctx, c) + if ipld.IsNotFound(err) { + return f.fm.Get(ctx, c) + } + return blk, err +} + +// GetSize returns the size of the requested block. It may return ErrNotFound +// when the block is not stored. +func (f *Filestore) GetSize(ctx context.Context, c cid.Cid) (int, error) { + size, err := f.bs.GetSize(ctx, c) + if err != nil { + if ipld.IsNotFound(err) { + return f.fm.GetSize(ctx, c) + } + return -1, err + } + return size, nil +} + +// Has returns true if the block with the given Cid is +// stored in the Filestore. +func (f *Filestore) Has(ctx context.Context, c cid.Cid) (bool, error) { + has, err := f.bs.Has(ctx, c) + if err != nil { + return false, err + } + + if has { + return true, nil + } + + return f.fm.Has(ctx, c) +} + +// Put stores a block in the Filestore. For blocks of +// underlying type FilestoreNode, the operation is +// delegated to the FileManager, while the rest of blocks +// are handled by the regular blockstore. +func (f *Filestore) Put(ctx context.Context, b blocks.Block) error { + has, err := f.Has(ctx, b.Cid()) + if err != nil { + return err + } + + if has { + return nil + } + + switch b := b.(type) { + case *posinfo.FilestoreNode: + return f.fm.Put(ctx, b) + default: + return f.bs.Put(ctx, b) + } +} + +// PutMany is like Put(), but takes a slice of blocks, allowing +// the underlying blockstore to perform batch transactions. +func (f *Filestore) PutMany(ctx context.Context, bs []blocks.Block) error { + var normals []blocks.Block + var fstores []*posinfo.FilestoreNode + + for _, b := range bs { + has, err := f.Has(ctx, b.Cid()) + if err != nil { + return err + } + + if has { + continue + } + + switch b := b.(type) { + case *posinfo.FilestoreNode: + fstores = append(fstores, b) + default: + normals = append(normals, b) + } + } + + if len(normals) > 0 { + err := f.bs.PutMany(ctx, normals) + if err != nil { + return err + } + } + + if len(fstores) > 0 { + err := f.fm.PutMany(ctx, fstores) + if err != nil { + return err + } + } + return nil +} + +// HashOnRead calls blockstore.HashOnRead. +func (f *Filestore) HashOnRead(enabled bool) { + f.bs.HashOnRead(enabled) +} + +var _ blockstore.Blockstore = (*Filestore)(nil) diff --git a/filestore/filestore_test.go b/filestore/filestore_test.go new file mode 100644 index 0000000000..cc6c7dffc0 --- /dev/null +++ b/filestore/filestore_test.go @@ -0,0 +1,180 @@ +package filestore + +import ( + "bytes" + "context" + "math/rand" + "os" + "testing" + + dag "github.com/ipfs/go-merkledag" + + cid "github.com/ipfs/go-cid" + ds "github.com/ipfs/go-datastore" + ipld "github.com/ipfs/go-ipld-format" + blockstore "github.com/ipfs/go-libipfs/blockstore" + posinfo "github.com/ipfs/go-libipfs/filestore/posinfo" +) + +var bg = context.Background() + +func newTestFilestore(t *testing.T) (string, *Filestore) { + mds := ds.NewMapDatastore() + + testdir, err := os.MkdirTemp("", "filestore-test") + if err != nil { + t.Fatal(err) + } + fm := NewFileManager(mds, testdir) + fm.AllowFiles = true + + bs := blockstore.NewBlockstore(mds) + fstore := NewFilestore(bs, fm) + return testdir, fstore +} + +func makeFile(dir string, data []byte) (string, error) { + f, err := os.CreateTemp(dir, "file") + if err != nil { + return "", err + } + + _, err = f.Write(data) + if err != nil { + return "", err + } + + return f.Name(), nil +} + +func TestBasicFilestore(t *testing.T) { + dir, fs := newTestFilestore(t) + + buf := make([]byte, 1000) + rand.Read(buf) + + fname, err := makeFile(dir, buf) + if err != nil { + t.Fatal(err) + } + + var cids []cid.Cid + for i := 0; i < 100; i++ { + n := &posinfo.FilestoreNode{ + PosInfo: &posinfo.PosInfo{ + FullPath: fname, + Offset: uint64(i * 10), + }, + Node: dag.NewRawNode(buf[i*10 : (i+1)*10]), + } + + err := fs.Put(bg, n) + if err != nil { + t.Fatal(err) + } + cids = append(cids, n.Node.Cid()) + } + + for i, c := range cids { + blk, err := fs.Get(bg, c) + if err != nil { + t.Fatal(err) + } + + if !bytes.Equal(blk.RawData(), buf[i*10:(i+1)*10]) { + t.Fatal("data didnt match on the way out") + } + } + + kch, err := fs.AllKeysChan(context.Background()) + if err != nil { + t.Fatal(err) + } + + out := make(map[string]struct{}) + for c := range kch { + out[c.KeyString()] = struct{}{} + } + + if len(out) != len(cids) { + t.Fatal("mismatch in number of entries") + } + + for _, c := range cids { + if _, ok := out[c.KeyString()]; !ok { + t.Fatal("missing cid: ", c) + } + } +} + +func randomFileAdd(t *testing.T, fs *Filestore, dir string, size int) (string, []cid.Cid) { + buf := make([]byte, size) + rand.Read(buf) + + fname, err := makeFile(dir, buf) + if err != nil { + t.Fatal(err) + } + + var out []cid.Cid + for i := 0; i < size/10; i++ { + n := &posinfo.FilestoreNode{ + PosInfo: &posinfo.PosInfo{ + FullPath: fname, + Offset: uint64(i * 10), + }, + Node: dag.NewRawNode(buf[i*10 : (i+1)*10]), + } + err := fs.Put(bg, n) + if err != nil { + t.Fatal(err) + } + out = append(out, n.Cid()) + } + + return fname, out +} + +func TestDeletes(t *testing.T) { + dir, fs := newTestFilestore(t) + _, cids := randomFileAdd(t, fs, dir, 100) + todelete := cids[:4] + for _, c := range todelete { + err := fs.DeleteBlock(bg, c) + if err != nil { + t.Fatal(err) + } + } + + deleted := make(map[string]bool) + for _, c := range todelete { + _, err := fs.Get(bg, c) + if !ipld.IsNotFound(err) { + t.Fatal("expected blockstore not found error") + } + deleted[c.KeyString()] = true + } + + keys, err := fs.AllKeysChan(context.Background()) + if err != nil { + t.Fatal(err) + } + + for c := range keys { + if deleted[c.KeyString()] { + t.Fatal("shouldnt have reference to this key anymore") + } + } +} + +func TestIsURL(t *testing.T) { + if !IsURL("http://www.example.com") { + t.Fatal("IsURL failed: http://www.example.com") + } + if !IsURL("https://www.example.com") { + t.Fatal("IsURL failed: https://www.example.com") + } + if IsURL("adir/afile") || IsURL("http:/ /afile") || IsURL("http:/a/file") { + t.Fatal("IsURL recognized non-url") + } +} diff --git a/filestore/fsrefstore.go b/filestore/fsrefstore.go new file mode 100644 index 0000000000..694e48f4e1 --- /dev/null +++ b/filestore/fsrefstore.go @@ -0,0 +1,340 @@ +package filestore + +import ( + "context" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + + pb "github.com/ipfs/go-libipfs/filestore/pb" + + proto "github.com/gogo/protobuf/proto" + cid "github.com/ipfs/go-cid" + ds "github.com/ipfs/go-datastore" + dsns "github.com/ipfs/go-datastore/namespace" + dsq "github.com/ipfs/go-datastore/query" + ipld "github.com/ipfs/go-ipld-format" + blocks "github.com/ipfs/go-libipfs/blocks" + dshelp "github.com/ipfs/go-libipfs/datastore/dshelp" + posinfo "github.com/ipfs/go-libipfs/filestore/posinfo" + mh "github.com/multiformats/go-multihash" +) + +// FilestorePrefix identifies the key prefix for FileManager blocks. +var FilestorePrefix = ds.NewKey("filestore") + +// FileManager is a blockstore implementation which stores special +// blocks FilestoreNode type. These nodes only contain a reference +// to the actual location of the block data in the filesystem +// (a path and an offset). +type FileManager struct { + AllowFiles bool + AllowUrls bool + ds ds.Batching + root string +} + +// CorruptReferenceError implements the error interface. +// It is used to indicate that the block contents pointed +// by the referencing blocks cannot be retrieved (i.e. the +// file is not found, or the data changed as it was being read). +type CorruptReferenceError struct { + Code Status + Err error +} + +// Error() returns the error message in the CorruptReferenceError +// as a string. +func (c CorruptReferenceError) Error() string { + return c.Err.Error() +} + +// NewFileManager initializes a new file manager with the given +// datastore and root. All FilestoreNodes paths are relative to the +// root path given here, which is prepended for any operations. +func NewFileManager(ds ds.Batching, root string) *FileManager { + return &FileManager{ds: dsns.Wrap(ds, FilestorePrefix), root: root} +} + +// AllKeysChan returns a channel from which to read the keys stored in +// the FileManager. If the given context is cancelled the channel will be +// closed. +// +// All CIDs returned are of type Raw. +func (f *FileManager) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) { + q := dsq.Query{KeysOnly: true} + + res, err := f.ds.Query(ctx, q) + if err != nil { + return nil, err + } + + out := make(chan cid.Cid, dsq.KeysOnlyBufSize) + go func() { + defer close(out) + for { + v, ok := res.NextSync() + if !ok { + return + } + + k := ds.RawKey(v.Key) + mhash, err := dshelp.DsKeyToMultihash(k) + if err != nil { + logger.Errorf("decoding cid from filestore: %s", err) + continue + } + + select { + case out <- cid.NewCidV1(cid.Raw, mhash): + case <-ctx.Done(): + return + } + } + }() + + return out, nil +} + +// DeleteBlock deletes the reference-block from the underlying +// datastore. It does not touch the referenced data. +func (f *FileManager) DeleteBlock(ctx context.Context, c cid.Cid) error { + err := f.ds.Delete(ctx, dshelp.MultihashToDsKey(c.Hash())) + if err == ds.ErrNotFound { + return ipld.ErrNotFound{Cid: c} + } + return err +} + +// Get reads a block from the datastore. Reading a block +// is done in two steps: the first step retrieves the reference +// block from the datastore. The second step uses the stored +// path and offsets to read the raw block data directly from disk. +func (f *FileManager) Get(ctx context.Context, c cid.Cid) (blocks.Block, error) { + dobj, err := f.getDataObj(ctx, c.Hash()) + if err != nil { + return nil, err + } + out, err := f.readDataObj(ctx, c.Hash(), dobj) + if err != nil { + return nil, err + } + + return blocks.NewBlockWithCid(out, c) +} + +// GetSize gets the size of the block from the datastore. +// +// This method may successfully return the size even if returning the block +// would fail because the associated file is no longer available. +func (f *FileManager) GetSize(ctx context.Context, c cid.Cid) (int, error) { + dobj, err := f.getDataObj(ctx, c.Hash()) + if err != nil { + return -1, err + } + return int(dobj.GetSize_()), nil +} + +func (f *FileManager) readDataObj(ctx context.Context, m mh.Multihash, d *pb.DataObj) ([]byte, error) { + if IsURL(d.GetFilePath()) { + return f.readURLDataObj(ctx, m, d) + } + return f.readFileDataObj(m, d) +} + +func (f *FileManager) getDataObj(ctx context.Context, m mh.Multihash) (*pb.DataObj, error) { + o, err := f.ds.Get(ctx, dshelp.MultihashToDsKey(m)) + switch err { + case ds.ErrNotFound: + return nil, ipld.ErrNotFound{Cid: cid.NewCidV1(cid.Raw, m)} + case nil: + // + default: + return nil, err + } + + return unmarshalDataObj(o) +} + +func unmarshalDataObj(data []byte) (*pb.DataObj, error) { + var dobj pb.DataObj + if err := proto.Unmarshal(data, &dobj); err != nil { + return nil, err + } + + return &dobj, nil +} + +func (f *FileManager) readFileDataObj(m mh.Multihash, d *pb.DataObj) ([]byte, error) { + if !f.AllowFiles { + return nil, ErrFilestoreNotEnabled + } + + p := filepath.FromSlash(d.GetFilePath()) + abspath := filepath.Join(f.root, p) + + fi, err := os.Open(abspath) + if os.IsNotExist(err) { + return nil, &CorruptReferenceError{StatusFileNotFound, err} + } else if err != nil { + return nil, &CorruptReferenceError{StatusFileError, err} + } + defer fi.Close() + + _, err = fi.Seek(int64(d.GetOffset()), io.SeekStart) + if err != nil { + return nil, &CorruptReferenceError{StatusFileError, err} + } + + outbuf := make([]byte, d.GetSize_()) + _, err = io.ReadFull(fi, outbuf) + if err == io.EOF || err == io.ErrUnexpectedEOF { + return nil, &CorruptReferenceError{StatusFileChanged, err} + } else if err != nil { + return nil, &CorruptReferenceError{StatusFileError, err} + } + + // Work with CIDs for this, as they are a nice wrapper and things + // will not break if multihashes underlying types change. + origCid := cid.NewCidV1(cid.Raw, m) + outcid, err := origCid.Prefix().Sum(outbuf) + if err != nil { + return nil, err + } + + if !origCid.Equals(outcid) { + return nil, &CorruptReferenceError{StatusFileChanged, + fmt.Errorf("data in file did not match. %s offset %d", d.GetFilePath(), d.GetOffset())} + } + + return outbuf, nil +} + +// reads and verifies the block from URL +func (f *FileManager) readURLDataObj(ctx context.Context, m mh.Multihash, d *pb.DataObj) ([]byte, error) { + if !f.AllowUrls { + return nil, ErrUrlstoreNotEnabled + } + + req, err := http.NewRequestWithContext(ctx, "GET", d.GetFilePath(), nil) + if err != nil { + return nil, err + } + + req.Header.Add("Range", fmt.Sprintf("bytes=%d-%d", d.GetOffset(), d.GetOffset()+d.GetSize_()-1)) + + res, err := http.DefaultClient.Do(req) + if err != nil { + return nil, &CorruptReferenceError{StatusFileError, err} + } + if res.StatusCode != http.StatusOK && res.StatusCode != http.StatusPartialContent { + return nil, &CorruptReferenceError{StatusFileError, + fmt.Errorf("expected HTTP 200 or 206 got %d", res.StatusCode)} + } + + outbuf := make([]byte, d.GetSize_()) + _, err = io.ReadFull(res.Body, outbuf) + if err == io.EOF || err == io.ErrUnexpectedEOF { + return nil, &CorruptReferenceError{StatusFileChanged, err} + } else if err != nil { + return nil, &CorruptReferenceError{StatusFileError, err} + } + res.Body.Close() + + // Work with CIDs for this, as they are a nice wrapper and things + // will not break if multihashes underlying types change. + origCid := cid.NewCidV1(cid.Raw, m) + outcid, err := origCid.Prefix().Sum(outbuf) + if err != nil { + return nil, err + } + + if !origCid.Equals(outcid) { + return nil, &CorruptReferenceError{StatusFileChanged, + fmt.Errorf("data in file did not match. %s offset %d", d.GetFilePath(), d.GetOffset())} + } + + return outbuf, nil +} + +// Has returns if the FileManager is storing a block reference. It does not +// validate the data, nor checks if the reference is valid. +func (f *FileManager) Has(ctx context.Context, c cid.Cid) (bool, error) { + // NOTE: interesting thing to consider. Has doesnt validate the data. + // So the data on disk could be invalid, and we could think we have it. + dsk := dshelp.MultihashToDsKey(c.Hash()) + return f.ds.Has(ctx, dsk) +} + +type putter interface { + Put(context.Context, ds.Key, []byte) error +} + +// Put adds a new reference block to the FileManager. It does not check +// that the reference is valid. +func (f *FileManager) Put(ctx context.Context, b *posinfo.FilestoreNode) error { + return f.putTo(ctx, b, f.ds) +} + +func (f *FileManager) putTo(ctx context.Context, b *posinfo.FilestoreNode, to putter) error { + var dobj pb.DataObj + + if IsURL(b.PosInfo.FullPath) { + if !f.AllowUrls { + return ErrUrlstoreNotEnabled + } + dobj.FilePath = b.PosInfo.FullPath + } else { + if !f.AllowFiles { + return ErrFilestoreNotEnabled + } + //lint:ignore SA1019 // ignore staticcheck + if !filepath.HasPrefix(b.PosInfo.FullPath, f.root) { + return fmt.Errorf("cannot add filestore references outside ipfs root (%s)", f.root) + } + + p, err := filepath.Rel(f.root, b.PosInfo.FullPath) + if err != nil { + return err + } + + dobj.FilePath = filepath.ToSlash(p) + } + dobj.Offset = b.PosInfo.Offset + dobj.Size_ = uint64(len(b.RawData())) + + data, err := proto.Marshal(&dobj) + if err != nil { + return err + } + + return to.Put(ctx, dshelp.MultihashToDsKey(b.Cid().Hash()), data) +} + +// PutMany is like Put() but takes a slice of blocks instead, +// allowing it to create a batch transaction. +func (f *FileManager) PutMany(ctx context.Context, bs []*posinfo.FilestoreNode) error { + batch, err := f.ds.Batch(ctx) + if err != nil { + return err + } + + for _, b := range bs { + if err := f.putTo(ctx, b, batch); err != nil { + return err + } + } + + return batch.Commit(ctx) +} + +// IsURL returns true if the string represents a valid URL that the +// urlstore can handle. More specifically it returns true if a string +// begins with 'http://' or 'https://'. +func IsURL(str string) bool { + return (len(str) > 7 && str[0] == 'h' && str[1] == 't' && str[2] == 't' && str[3] == 'p') && + ((len(str) > 8 && str[4] == 's' && str[5] == ':' && str[6] == '/' && str[7] == '/') || + (str[4] == ':' && str[5] == '/' && str[6] == '/')) +} diff --git a/filestore/pb/dataobj.pb.go b/filestore/pb/dataobj.pb.go new file mode 100644 index 0000000000..d342cabe51 --- /dev/null +++ b/filestore/pb/dataobj.pb.go @@ -0,0 +1,376 @@ +// Code generated by protoc-gen-gogo. DO NOT EDIT. +// source: dataobj.proto + +package datastore_pb + +import ( + fmt "fmt" + io "io" + math "math" + math_bits "math/bits" + + proto "github.com/gogo/protobuf/proto" +) + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package + +type DataObj struct { + FilePath string `protobuf:"bytes,1,opt,name=FilePath" json:"FilePath"` + Offset uint64 `protobuf:"varint,2,opt,name=Offset" json:"Offset"` + Size_ uint64 `protobuf:"varint,3,opt,name=Size" json:"Size"` +} + +func (m *DataObj) Reset() { *m = DataObj{} } +func (m *DataObj) String() string { return proto.CompactTextString(m) } +func (*DataObj) ProtoMessage() {} +func (*DataObj) Descriptor() ([]byte, []int) { + return fileDescriptor_a76cb282d869d683, []int{0} +} +func (m *DataObj) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *DataObj) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_DataObj.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *DataObj) XXX_Merge(src proto.Message) { + xxx_messageInfo_DataObj.Merge(m, src) +} +func (m *DataObj) XXX_Size() int { + return m.Size() +} +func (m *DataObj) XXX_DiscardUnknown() { + xxx_messageInfo_DataObj.DiscardUnknown(m) +} + +var xxx_messageInfo_DataObj proto.InternalMessageInfo + +func (m *DataObj) GetFilePath() string { + if m != nil { + return m.FilePath + } + return "" +} + +func (m *DataObj) GetOffset() uint64 { + if m != nil { + return m.Offset + } + return 0 +} + +func (m *DataObj) GetSize_() uint64 { + if m != nil { + return m.Size_ + } + return 0 +} + +func init() { + proto.RegisterType((*DataObj)(nil), "datastore.pb.DataObj") +} + +func init() { proto.RegisterFile("dataobj.proto", fileDescriptor_a76cb282d869d683) } + +var fileDescriptor_a76cb282d869d683 = []byte{ + // 150 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xe2, 0x4d, 0x49, 0x2c, 0x49, + 0xcc, 0x4f, 0xca, 0xd2, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0xe2, 0x01, 0x71, 0x8b, 0x4b, 0xf2, + 0x8b, 0x52, 0xf5, 0x0a, 0x92, 0x94, 0x92, 0xb9, 0xd8, 0x5d, 0x12, 0x4b, 0x12, 0xfd, 0x93, 0xb2, + 0x84, 0x14, 0xb8, 0x38, 0xdc, 0x32, 0x73, 0x52, 0x03, 0x12, 0x4b, 0x32, 0x24, 0x18, 0x15, 0x18, + 0x35, 0x38, 0x9d, 0x58, 0x4e, 0xdc, 0x93, 0x67, 0x08, 0x82, 0x8b, 0x0a, 0xc9, 0x70, 0xb1, 0xf9, + 0xa7, 0xa5, 0x15, 0xa7, 0x96, 0x48, 0x30, 0x29, 0x30, 0x6a, 0xb0, 0x40, 0xe5, 0xa1, 0x62, 0x42, + 0x12, 0x5c, 0x2c, 0xc1, 0x99, 0x55, 0xa9, 0x12, 0xcc, 0x48, 0x72, 0x60, 0x11, 0x27, 0x89, 0x13, + 0x8f, 0xe4, 0x18, 0x2f, 0x3c, 0x92, 0x63, 0x7c, 0xf0, 0x48, 0x8e, 0x71, 0xc2, 0x63, 0x39, 0x86, + 0x0b, 0x8f, 0xe5, 0x18, 0x6e, 0x3c, 0x96, 0x63, 0x00, 0x04, 0x00, 0x00, 0xff, 0xff, 0x5d, 0x4a, + 0x76, 0xa0, 0x9c, 0x00, 0x00, 0x00, +} + +func (m *DataObj) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *DataObj) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *DataObj) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + i = encodeVarintDataobj(dAtA, i, uint64(m.Size_)) + i-- + dAtA[i] = 0x18 + i = encodeVarintDataobj(dAtA, i, uint64(m.Offset)) + i-- + dAtA[i] = 0x10 + i -= len(m.FilePath) + copy(dAtA[i:], m.FilePath) + i = encodeVarintDataobj(dAtA, i, uint64(len(m.FilePath))) + i-- + dAtA[i] = 0xa + return len(dAtA) - i, nil +} + +func encodeVarintDataobj(dAtA []byte, offset int, v uint64) int { + offset -= sovDataobj(v) + base := offset + for v >= 1<<7 { + dAtA[offset] = uint8(v&0x7f | 0x80) + v >>= 7 + offset++ + } + dAtA[offset] = uint8(v) + return base +} +func (m *DataObj) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.FilePath) + n += 1 + l + sovDataobj(uint64(l)) + n += 1 + sovDataobj(uint64(m.Offset)) + n += 1 + sovDataobj(uint64(m.Size_)) + return n +} + +func sovDataobj(x uint64) (n int) { + return (math_bits.Len64(x|1) + 6) / 7 +} +func sozDataobj(x uint64) (n int) { + return sovDataobj(uint64((x << 1) ^ uint64((int64(x) >> 63)))) +} +func (m *DataObj) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowDataobj + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: DataObj: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: DataObj: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field FilePath", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowDataobj + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthDataobj + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthDataobj + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.FilePath = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Offset", wireType) + } + m.Offset = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowDataobj + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Offset |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Size_", wireType) + } + m.Size_ = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowDataobj + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Size_ |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipDataobj(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthDataobj + } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthDataobj + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func skipDataobj(dAtA []byte) (n int, err error) { + l := len(dAtA) + iNdEx := 0 + depth := 0 + for iNdEx < l { + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowDataobj + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + wireType := int(wire & 0x7) + switch wireType { + case 0: + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowDataobj + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + iNdEx++ + if dAtA[iNdEx-1] < 0x80 { + break + } + } + case 1: + iNdEx += 8 + case 2: + var length int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowDataobj + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + length |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if length < 0 { + return 0, ErrInvalidLengthDataobj + } + iNdEx += length + case 3: + depth++ + case 4: + if depth == 0 { + return 0, ErrUnexpectedEndOfGroupDataobj + } + depth-- + case 5: + iNdEx += 4 + default: + return 0, fmt.Errorf("proto: illegal wireType %d", wireType) + } + if iNdEx < 0 { + return 0, ErrInvalidLengthDataobj + } + if depth == 0 { + return iNdEx, nil + } + } + return 0, io.ErrUnexpectedEOF +} + +var ( + ErrInvalidLengthDataobj = fmt.Errorf("proto: negative length found during unmarshaling") + ErrIntOverflowDataobj = fmt.Errorf("proto: integer overflow") + ErrUnexpectedEndOfGroupDataobj = fmt.Errorf("proto: unexpected end of group") +) diff --git a/filestore/pb/dataobj.proto b/filestore/pb/dataobj.proto new file mode 100644 index 0000000000..909d22b77f --- /dev/null +++ b/filestore/pb/dataobj.proto @@ -0,0 +1,9 @@ +syntax = "proto2"; + +package datastore.pb; + +message DataObj { + optional string FilePath = 1; + optional uint64 Offset = 2; + optional uint64 Size = 3; +} diff --git a/filestore/posinfo/.github/ISSUE_TEMPLATE/open_an_issue.md b/filestore/posinfo/.github/ISSUE_TEMPLATE/open_an_issue.md new file mode 100644 index 0000000000..4fcbd00aca --- /dev/null +++ b/filestore/posinfo/.github/ISSUE_TEMPLATE/open_an_issue.md @@ -0,0 +1,19 @@ +--- +name: Open an issue +about: Only for actionable issues relevant to this repository. +title: '' +labels: need/triage +assignees: '' + +--- + diff --git a/filestore/posinfo/.gitignore b/filestore/posinfo/.gitignore new file mode 100644 index 0000000000..a1338d6851 --- /dev/null +++ b/filestore/posinfo/.gitignore @@ -0,0 +1,14 @@ +# Binaries for programs and plugins +*.exe +*.dll +*.so +*.dylib + +# Test binary, build with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Project-local glide cache, RE: https://github.com/Masterminds/glide/issues/736 +.glide/ diff --git a/filestore/posinfo/Makefile b/filestore/posinfo/Makefile new file mode 100644 index 0000000000..24d71558e7 --- /dev/null +++ b/filestore/posinfo/Makefile @@ -0,0 +1,18 @@ +all: deps +gx: + go get github.com/whyrusleeping/gx + go get github.com/whyrusleeping/gx-go +deps: gx + gx --verbose install --global + gx-go rewrite +test: deps + go test -v -covermode count -coverprofile=coverage.out . +rw: + gx-go rewrite +rwundo: + gx-go rewrite --undo +publish: rwundo + gx publish +.PHONY: all gx deps test rw rwundo publish + + diff --git a/filestore/posinfo/README.md b/filestore/posinfo/README.md new file mode 100644 index 0000000000..bd509c17e0 --- /dev/null +++ b/filestore/posinfo/README.md @@ -0,0 +1,37 @@ +# go-ipfs-posinfo + +[![](https://img.shields.io/badge/made%20by-Protocol%20Labs-blue.svg?style=flat-square)](http://ipn.io) +[![](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](http://ipfs.io/) +[![standard-readme compliant](https://img.shields.io/badge/standard--readme-OK-green.svg?style=flat-square)](https://github.com/RichardLitt/standard-readme) +[![GoDoc](https://godoc.org/github.com/ipfs/go-ipfs-posinfo?status.svg)](https://godoc.org/github.com/ipfs/go-ipfs-posinfo) +[![Build Status](https://travis-ci.org/ipfs/go-ipfs-posinfo.svg?branch=master)](https://travis-ci.org/ipfs/go-ipfs-posinfo) + +> Posinfo wraps offset information for ipfs filestore nodes + +## Table of Contents + +- [Install](#install) +- [Usage](#usage) +- [Contribute](#contribute) +- [License](#license) + +## Install + +``` +go get github.com/ipfs/go-ipfs-posinfo +``` + +## Usage + +See the [GoDoc documentation](https://godoc.org/github.com/ipfs/go-ipfs-posinfo) + + +## Contribute + +PRs accepted. + +Small note: If editing the README, please conform to the [standard-readme](https://github.com/RichardLitt/standard-readme) specification. + +## License + +MIT © Protocol Labs, Inc. diff --git a/filestore/posinfo/posinfo.go b/filestore/posinfo/posinfo.go new file mode 100644 index 0000000000..0b32c89da1 --- /dev/null +++ b/filestore/posinfo/posinfo.go @@ -0,0 +1,23 @@ +// Package posinfo wraps offset information used by ipfs filestore nodes +package posinfo + +import ( + "os" + + ipld "github.com/ipfs/go-ipld-format" +) + +// PosInfo stores information about the file offset, its path and +// stat. +type PosInfo struct { + Offset uint64 + FullPath string + Stat os.FileInfo // can be nil +} + +// FilestoreNode is an ipld.Node which arries PosInfo with it +// allowing to map it directly to a filesystem object. +type FilestoreNode struct { + ipld.Node + PosInfo *PosInfo +} diff --git a/filestore/util.go b/filestore/util.go new file mode 100644 index 0000000000..6f3c7d12f9 --- /dev/null +++ b/filestore/util.go @@ -0,0 +1,292 @@ +package filestore + +import ( + "context" + "fmt" + "sort" + + pb "github.com/ipfs/go-libipfs/filestore/pb" + + cid "github.com/ipfs/go-cid" + ds "github.com/ipfs/go-datastore" + dsq "github.com/ipfs/go-datastore/query" + ipld "github.com/ipfs/go-ipld-format" + dshelp "github.com/ipfs/go-libipfs/datastore/dshelp" + mh "github.com/multiformats/go-multihash" +) + +// Status is used to identify the state of the block data referenced +// by a FilestoreNode. Among other places, it is used by CorruptReferenceError. +type Status int32 + +// These are the supported Status codes. +const ( + StatusOk Status = 0 + StatusFileError Status = 10 // Backing File Error + StatusFileNotFound Status = 11 // Backing File Not Found + StatusFileChanged Status = 12 // Contents of the file changed + StatusOtherError Status = 20 // Internal Error, likely corrupt entry + StatusKeyNotFound Status = 30 +) + +// String provides a human-readable representation for Status codes. +func (s Status) String() string { + switch s { + case StatusOk: + return "ok" + case StatusFileError: + return "error" + case StatusFileNotFound: + return "no-file" + case StatusFileChanged: + return "changed" + case StatusOtherError: + return "ERROR" + case StatusKeyNotFound: + return "missing" + default: + return "???" + } +} + +// Format returns the status formatted as a string +// with leading 0s. +func (s Status) Format() string { + return fmt.Sprintf("%-7s", s.String()) +} + +// ListRes wraps the response of the List*() functions, which +// allows to obtain and verify blocks stored by the FileManager +// of a Filestore. It includes information about the referenced +// block. +type ListRes struct { + Status Status + ErrorMsg string + Key cid.Cid + FilePath string + Offset uint64 + Size uint64 +} + +// FormatLong returns a human readable string for a ListRes object +func (r *ListRes) FormatLong(enc func(cid.Cid) string) string { + if enc == nil { + enc = (cid.Cid).String + } + switch { + case !r.Key.Defined(): + return "" + case r.FilePath == "": + return r.Key.String() + default: + return fmt.Sprintf("%-50s %6d %s %d", enc(r.Key), r.Size, r.FilePath, r.Offset) + } +} + +// List fetches the block with the given key from the Filemanager +// of the given Filestore and returns a ListRes object with the information. +// List does not verify that the reference is valid or whether the +// raw data is accesible. See Verify(). +func List(ctx context.Context, fs *Filestore, key cid.Cid) *ListRes { + return list(ctx, fs, false, key.Hash()) +} + +// ListAll returns a function as an iterator which, once invoked, returns +// one by one each block in the Filestore's FileManager. +// ListAll does not verify that the references are valid or whether +// the raw data is accessible. See VerifyAll(). +func ListAll(ctx context.Context, fs *Filestore, fileOrder bool) (func(context.Context) *ListRes, error) { + if fileOrder { + return listAllFileOrder(ctx, fs, false) + } + return listAll(ctx, fs, false) +} + +// Verify fetches the block with the given key from the Filemanager +// of the given Filestore and returns a ListRes object with the information. +// Verify makes sure that the reference is valid and the block data can be +// read. +func Verify(ctx context.Context, fs *Filestore, key cid.Cid) *ListRes { + return list(ctx, fs, true, key.Hash()) +} + +// VerifyAll returns a function as an iterator which, once invoked, +// returns one by one each block in the Filestore's FileManager. +// VerifyAll checks that the reference is valid and that the block data +// can be read. +func VerifyAll(ctx context.Context, fs *Filestore, fileOrder bool) (func(context.Context) *ListRes, error) { + if fileOrder { + return listAllFileOrder(ctx, fs, true) + } + return listAll(ctx, fs, true) +} + +func list(ctx context.Context, fs *Filestore, verify bool, key mh.Multihash) *ListRes { + dobj, err := fs.fm.getDataObj(ctx, key) + if err != nil { + return mkListRes(key, nil, err) + } + if verify { + _, err = fs.fm.readDataObj(ctx, key, dobj) + } + return mkListRes(key, dobj, err) +} + +func listAll(ctx context.Context, fs *Filestore, verify bool) (func(context.Context) *ListRes, error) { + q := dsq.Query{} + qr, err := fs.fm.ds.Query(ctx, q) + if err != nil { + return nil, err + } + + return func(ctx context.Context) *ListRes { + mhash, dobj, err := next(qr) + if dobj == nil && err == nil { + return nil + } else if err == nil && verify { + _, err = fs.fm.readDataObj(ctx, mhash, dobj) + } + return mkListRes(mhash, dobj, err) + }, nil +} + +func next(qr dsq.Results) (mh.Multihash, *pb.DataObj, error) { + v, ok := qr.NextSync() + if !ok { + return nil, nil, nil + } + + k := ds.RawKey(v.Key) + mhash, err := dshelp.DsKeyToMultihash(k) + if err != nil { + return nil, nil, fmt.Errorf("decoding multihash from filestore: %s", err) + } + + dobj, err := unmarshalDataObj(v.Value) + if err != nil { + return mhash, nil, err + } + + return mhash, dobj, nil +} + +func listAllFileOrder(ctx context.Context, fs *Filestore, verify bool) (func(context.Context) *ListRes, error) { + q := dsq.Query{} + qr, err := fs.fm.ds.Query(ctx, q) + if err != nil { + return nil, err + } + + var entries listEntries + + for { + v, ok := qr.NextSync() + if !ok { + break + } + dobj, err := unmarshalDataObj(v.Value) + if err != nil { + entries = append(entries, &listEntry{ + dsKey: v.Key, + err: err, + }) + } else { + entries = append(entries, &listEntry{ + dsKey: v.Key, + filePath: dobj.GetFilePath(), + offset: dobj.GetOffset(), + size: dobj.GetSize_(), + }) + } + } + sort.Sort(entries) + + i := 0 + return func(ctx context.Context) *ListRes { + if i >= len(entries) { + return nil + } + v := entries[i] + i++ + // attempt to convert the datastore key to a Multihash, + // store the error but don't use it yet + mhash, keyErr := dshelp.DsKeyToMultihash(ds.RawKey(v.dsKey)) + // first if they listRes already had an error return that error + if v.err != nil { + return mkListRes(mhash, nil, v.err) + } + // now reconstruct the DataObj + dobj := pb.DataObj{ + FilePath: v.filePath, + Offset: v.offset, + Size_: v.size, + } + // now if we could not convert the datastore key return that + // error + if keyErr != nil { + return mkListRes(mhash, &dobj, keyErr) + } + // finally verify the dataobj if requested + var err error + if verify { + _, err = fs.fm.readDataObj(ctx, mhash, &dobj) + } + return mkListRes(mhash, &dobj, err) + }, nil +} + +type listEntry struct { + filePath string + offset uint64 + dsKey string + size uint64 + err error +} + +type listEntries []*listEntry + +func (l listEntries) Len() int { return len(l) } +func (l listEntries) Swap(i, j int) { l[i], l[j] = l[j], l[i] } +func (l listEntries) Less(i, j int) bool { + if l[i].filePath == l[j].filePath { + if l[i].offset == l[j].offset { + return l[i].dsKey < l[j].dsKey + } + return l[i].offset < l[j].offset + } + return l[i].filePath < l[j].filePath +} + +func mkListRes(m mh.Multihash, d *pb.DataObj, err error) *ListRes { + status := StatusOk + errorMsg := "" + if err != nil { + if err == ds.ErrNotFound || ipld.IsNotFound(err) { + status = StatusKeyNotFound + } else if err, ok := err.(*CorruptReferenceError); ok { + status = err.Code + } else { + status = StatusOtherError + } + errorMsg = err.Error() + } + + c := cid.NewCidV1(cid.Raw, m) + + if d == nil { + return &ListRes{ + Status: status, + ErrorMsg: errorMsg, + Key: c, + } + } + + return &ListRes{ + Status: status, + ErrorMsg: errorMsg, + Key: c, + FilePath: d.FilePath, + Size: d.Size_, + Offset: d.Offset, + } +} diff --git a/gateway/assets/assets.go b/gateway/assets/assets.go index 2e442dd134..fd0fe73cc9 100644 --- a/gateway/assets/assets.go +++ b/gateway/assets/assets.go @@ -15,7 +15,7 @@ import ( "github.com/cespare/xxhash" - ipfspath "github.com/ipfs/go-path" + ipfspath "github.com/ipfs/go-libipfs/path" ) //go:embed dag-index.html directory-index.html knownIcons.txt diff --git a/gateway/errors.go b/gateway/errors.go index 47f40cf5b6..3fe053d3bb 100644 --- a/gateway/errors.go +++ b/gateway/errors.go @@ -9,7 +9,7 @@ import ( "time" ipld "github.com/ipfs/go-ipld-format" - "github.com/ipfs/go-path/resolver" + "github.com/ipfs/go-libipfs/path" ) var ( diff --git a/gateway/gateway.go b/gateway/gateway.go index 977c6b960f..a1c4c29480 100644 --- a/gateway/gateway.go +++ b/gateway/gateway.go @@ -7,9 +7,9 @@ import ( cid "github.com/ipfs/go-cid" "github.com/ipfs/go-libipfs/blocks" + "github.com/ipfs/go-libipfs/coreiface" + iface "github.com/ipfs/go-libipfs/coreiface" "github.com/ipfs/go-libipfs/files" - iface "github.com/ipfs/interface-go-ipfs-core" - "github.com/ipfs/interface-go-ipfs-core/path" ) // Config is the configuration used when creating a new gateway handler. @@ -43,7 +43,7 @@ type API interface { // ResolvePath resolves the path using UnixFS resolver. If the path does not // exist due to a missing link, it should return an error of type: - // https://pkg.go.dev/github.com/ipfs/go-path@v0.3.0/resolver#ErrNoLink + // https://pkg.go.dev/github.com/ipfs/go-libipfs/path@v0.3.0/resolver#ErrNoLink ResolvePath(context.Context, path.Path) (path.Resolved, error) } diff --git a/gateway/gateway_test.go b/gateway/gateway_test.go index 2b9ef0c011..8a3b923163 100644 --- a/gateway/gateway_test.go +++ b/gateway/gateway_test.go @@ -13,26 +13,24 @@ import ( "strings" "testing" - "github.com/ipfs/go-blockservice" "github.com/ipfs/go-cid" - bsfetcher "github.com/ipfs/go-fetcher/impl/blockservice" - blockstore "github.com/ipfs/go-ipfs-blockstore" - offline "github.com/ipfs/go-ipfs-exchange-offline" format "github.com/ipfs/go-ipld-format" "github.com/ipfs/go-libipfs/blocks" + "github.com/ipfs/go-libipfs/blockservice" + blockstore "github.com/ipfs/go-libipfs/blockstore" + iface "github.com/ipfs/go-libipfs/coreiface" + ipath "github.com/ipfs/go-libipfs/coreiface" + nsopts "github.com/ipfs/go-libipfs/coreiface" + offline "github.com/ipfs/go-libipfs/exchange/offline" + bsfetcher "github.com/ipfs/go-libipfs/fetcher" "github.com/ipfs/go-libipfs/files" + "github.com/ipfs/go-libipfs/namesys" + "github.com/ipfs/go-libipfs/path" + path "github.com/ipfs/go-libipfs/path" + "github.com/ipfs/go-libipfs/unixfs" + ufile "github.com/ipfs/go-libipfs/unixfs" + uio "github.com/ipfs/go-libipfs/unixfs" "github.com/ipfs/go-merkledag" - "github.com/ipfs/go-namesys" - "github.com/ipfs/go-namesys/resolve" - path "github.com/ipfs/go-path" - "github.com/ipfs/go-path/resolver" - "github.com/ipfs/go-unixfs" - ufile "github.com/ipfs/go-unixfs/file" - uio "github.com/ipfs/go-unixfs/io" - "github.com/ipfs/go-unixfsnode" - iface "github.com/ipfs/interface-go-ipfs-core" - nsopts "github.com/ipfs/interface-go-ipfs-core/options/namesys" - ipath "github.com/ipfs/interface-go-ipfs-core/path" carblockstore "github.com/ipld/go-car/v2/blockstore" dagpb "github.com/ipld/go-codec-dagpb" "github.com/ipld/go-ipld-prime" diff --git a/gateway/handler.go b/gateway/handler.go index 92f88df6b6..6623cac3c8 100644 --- a/gateway/handler.go +++ b/gateway/handler.go @@ -16,9 +16,9 @@ import ( "time" cid "github.com/ipfs/go-cid" + coreiface "github.com/ipfs/go-libipfs/coreiface" + ipath "github.com/ipfs/go-libipfs/coreiface" logging "github.com/ipfs/go-log" - coreiface "github.com/ipfs/interface-go-ipfs-core" - ipath "github.com/ipfs/interface-go-ipfs-core/path" mc "github.com/multiformats/go-multicodec" prometheus "github.com/prometheus/client_golang/prometheus" "go.opentelemetry.io/otel" diff --git a/gateway/handler_block.go b/gateway/handler_block.go index a4f00ff9df..1192cbed3c 100644 --- a/gateway/handler_block.go +++ b/gateway/handler_block.go @@ -7,7 +7,7 @@ import ( "net/http" "time" - ipath "github.com/ipfs/interface-go-ipfs-core/path" + ipath "github.com/ipfs/go-libipfs/coreiface" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" ) diff --git a/gateway/handler_car.go b/gateway/handler_car.go index 0ccb602006..33f898f9a5 100644 --- a/gateway/handler_car.go +++ b/gateway/handler_car.go @@ -8,7 +8,7 @@ import ( cid "github.com/ipfs/go-cid" blocks "github.com/ipfs/go-libipfs/blocks" - ipath "github.com/ipfs/interface-go-ipfs-core/path" + ipath "github.com/ipfs/go-libipfs/coreiface" gocar "github.com/ipld/go-car" selectorparse "github.com/ipld/go-ipld-prime/traversal/selector/parse" "go.opentelemetry.io/otel/attribute" diff --git a/gateway/handler_codec.go b/gateway/handler_codec.go index ed02ab1071..aa27a2af9d 100644 --- a/gateway/handler_codec.go +++ b/gateway/handler_codec.go @@ -9,8 +9,8 @@ import ( "time" cid "github.com/ipfs/go-cid" + ipath "github.com/ipfs/go-libipfs/coreiface" "github.com/ipfs/go-libipfs/gateway/assets" - ipath "github.com/ipfs/interface-go-ipfs-core/path" "github.com/ipld/go-ipld-prime/multicodec" "github.com/ipld/go-ipld-prime/node/basicnode" mc "github.com/multiformats/go-multicodec" diff --git a/gateway/handler_ipns_record.go b/gateway/handler_ipns_record.go index e9548f7775..8941fdb7ee 100644 --- a/gateway/handler_ipns_record.go +++ b/gateway/handler_ipns_record.go @@ -10,8 +10,8 @@ import ( "github.com/gogo/protobuf/proto" "github.com/ipfs/go-cid" - ipns_pb "github.com/ipfs/go-ipns/pb" - ipath "github.com/ipfs/interface-go-ipfs-core/path" + ipath "github.com/ipfs/go-libipfs/coreiface" + ipns_pb "github.com/ipfs/go-libipfs/ipns" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" "go.uber.org/zap" diff --git a/gateway/handler_tar.go b/gateway/handler_tar.go index 569c031d45..7c5c2558c5 100644 --- a/gateway/handler_tar.go +++ b/gateway/handler_tar.go @@ -7,8 +7,8 @@ import ( "net/http" "time" + ipath "github.com/ipfs/go-libipfs/coreiface" "github.com/ipfs/go-libipfs/files" - ipath "github.com/ipfs/interface-go-ipfs-core/path" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" "go.uber.org/zap" diff --git a/gateway/handler_test.go b/gateway/handler_test.go index c3dcf9606a..ca7b244f64 100644 --- a/gateway/handler_test.go +++ b/gateway/handler_test.go @@ -11,10 +11,10 @@ import ( cid "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" "github.com/ipfs/go-libipfs/blocks" + iface "github.com/ipfs/go-libipfs/coreiface" + ipath "github.com/ipfs/go-libipfs/coreiface" "github.com/ipfs/go-libipfs/files" - "github.com/ipfs/go-path/resolver" - iface "github.com/ipfs/interface-go-ipfs-core" - ipath "github.com/ipfs/interface-go-ipfs-core/path" + "github.com/ipfs/go-libipfs/path" "github.com/stretchr/testify/assert" ) diff --git a/gateway/handler_unixfs.go b/gateway/handler_unixfs.go index 28a0677d3d..eda9c25139 100644 --- a/gateway/handler_unixfs.go +++ b/gateway/handler_unixfs.go @@ -6,8 +6,8 @@ import ( "net/http" "time" + ipath "github.com/ipfs/go-libipfs/coreiface" "github.com/ipfs/go-libipfs/files" - ipath "github.com/ipfs/interface-go-ipfs-core/path" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" "go.uber.org/zap" diff --git a/gateway/handler_unixfs__redirects.go b/gateway/handler_unixfs__redirects.go index b9658e4e52..600da60243 100644 --- a/gateway/handler_unixfs__redirects.go +++ b/gateway/handler_unixfs__redirects.go @@ -9,8 +9,8 @@ import ( "strings" redirects "github.com/ipfs/go-ipfs-redirects-file" + ipath "github.com/ipfs/go-libipfs/coreiface" "github.com/ipfs/go-libipfs/files" - ipath "github.com/ipfs/interface-go-ipfs-core/path" "go.uber.org/zap" ) diff --git a/gateway/handler_unixfs_dir.go b/gateway/handler_unixfs_dir.go index c03a4b81b3..1d61291bca 100644 --- a/gateway/handler_unixfs_dir.go +++ b/gateway/handler_unixfs_dir.go @@ -11,11 +11,11 @@ import ( "github.com/dustin/go-humanize" cid "github.com/ipfs/go-cid" + ipath "github.com/ipfs/go-libipfs/coreiface" "github.com/ipfs/go-libipfs/files" "github.com/ipfs/go-libipfs/gateway/assets" - path "github.com/ipfs/go-path" - "github.com/ipfs/go-path/resolver" - ipath "github.com/ipfs/interface-go-ipfs-core/path" + "github.com/ipfs/go-libipfs/path" + path "github.com/ipfs/go-libipfs/path" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" "go.uber.org/zap" diff --git a/gateway/handler_unixfs_file.go b/gateway/handler_unixfs_file.go index 55a61ee8ca..90c2da44cf 100644 --- a/gateway/handler_unixfs_file.go +++ b/gateway/handler_unixfs_file.go @@ -11,8 +11,8 @@ import ( "time" "github.com/gabriel-vasile/mimetype" + ipath "github.com/ipfs/go-libipfs/coreiface" "github.com/ipfs/go-libipfs/files" - ipath "github.com/ipfs/interface-go-ipfs-core/path" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" ) diff --git a/gateway/hostname_test.go b/gateway/hostname_test.go index 65970f5aa9..db6d08025b 100644 --- a/gateway/hostname_test.go +++ b/gateway/hostname_test.go @@ -8,7 +8,7 @@ import ( "testing" cid "github.com/ipfs/go-cid" - path "github.com/ipfs/go-path" + path "github.com/ipfs/go-libipfs/path" "github.com/stretchr/testify/assert" ) diff --git a/go.mod b/go.mod index a48d98dfd0..bf431c2276 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,9 @@ module github.com/ipfs/go-libipfs go 1.19 require ( + github.com/alecthomas/units v0.0.0-20210927113745-59d0afb8317a github.com/benbjohnson/clock v1.3.0 + github.com/cenkalti/backoff v2.2.1+incompatible github.com/cespare/xxhash v1.1.0 github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 github.com/cskr/pubsub v1.0.2 @@ -12,123 +14,164 @@ require ( github.com/gogo/protobuf v1.3.2 github.com/google/uuid v1.3.0 github.com/gorilla/mux v1.8.0 - github.com/ipfs/go-blockservice v0.5.0 + github.com/hashicorp/golang-lru v0.5.4 + github.com/ipfs/bbloom v0.0.4 + github.com/ipfs/go-bitfield v1.1.0 github.com/ipfs/go-cid v0.3.2 + github.com/ipfs/go-cidutil v0.1.0 github.com/ipfs/go-datastore v0.6.0 github.com/ipfs/go-detect-race v0.0.1 - github.com/ipfs/go-fetcher v1.6.1 - github.com/ipfs/go-ipfs-blockstore v1.2.0 + github.com/ipfs/go-ds-badger v0.3.0 + github.com/ipfs/go-ds-leveldb v0.5.0 github.com/ipfs/go-ipfs-blocksutil v0.0.1 github.com/ipfs/go-ipfs-delay v0.0.1 - github.com/ipfs/go-ipfs-exchange-interface v0.2.0 - github.com/ipfs/go-ipfs-exchange-offline v0.3.0 github.com/ipfs/go-ipfs-redirects-file v0.1.1 - github.com/ipfs/go-ipfs-routing v0.3.0 - github.com/ipfs/go-ipfs-util v0.0.2 + github.com/ipfs/go-ipld-cbor v0.0.6 github.com/ipfs/go-ipld-format v0.4.0 - github.com/ipfs/go-ipns v0.3.0 github.com/ipfs/go-log v1.0.5 github.com/ipfs/go-log/v2 v2.5.1 github.com/ipfs/go-merkledag v0.9.0 github.com/ipfs/go-metrics-interface v0.0.1 - github.com/ipfs/go-namesys v0.7.0 - github.com/ipfs/go-path v0.3.1 github.com/ipfs/go-peertaskqueue v0.8.1 - github.com/ipfs/go-unixfs v0.3.1 - github.com/ipfs/go-unixfsnode v1.5.1 - github.com/ipfs/interface-go-ipfs-core v0.10.0 + github.com/ipfs/go-unixfsnode v1.5.2 github.com/ipld/go-car v0.5.0 github.com/ipld/go-car/v2 v2.5.1 github.com/ipld/go-codec-dagpb v1.5.0 github.com/ipld/go-ipld-prime v0.19.0 github.com/jbenet/goprocess v0.1.4 github.com/libp2p/go-buffer-pool v0.1.0 - github.com/libp2p/go-libp2p v0.25.1 + github.com/libp2p/go-libp2p v0.26.2 + github.com/libp2p/go-libp2p-kad-dht v0.21.1 github.com/libp2p/go-libp2p-record v0.2.0 github.com/libp2p/go-libp2p-testing v0.12.0 github.com/libp2p/go-msgio v0.3.0 github.com/miekg/dns v1.1.50 + github.com/mr-tron/base58 v1.2.0 + github.com/multiformats/go-base32 v0.1.0 github.com/multiformats/go-multiaddr v0.8.0 + github.com/multiformats/go-multiaddr-dns v0.3.1 github.com/multiformats/go-multibase v0.1.1 github.com/multiformats/go-multicodec v0.7.0 github.com/multiformats/go-multihash v0.2.1 github.com/multiformats/go-multistream v0.4.1 + github.com/pkg/errors v0.9.1 + github.com/polydawn/refmt v0.89.0 github.com/prometheus/client_golang v1.14.0 github.com/samber/lo v1.36.0 + github.com/spaolacci/murmur3 v1.1.0 github.com/stretchr/testify v1.8.1 + github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc + github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f go.opencensus.io v0.24.0 go.opentelemetry.io/otel v1.7.0 go.opentelemetry.io/otel/trace v1.7.0 + go.uber.org/atomic v1.10.0 go.uber.org/multierr v1.9.0 go.uber.org/zap v1.24.0 - golang.org/x/sys v0.4.0 + golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b + golang.org/x/sync v0.1.0 + golang.org/x/sys v0.5.0 ) require ( - github.com/alecthomas/units v0.0.0-20210927113745-59d0afb8317a // indirect + github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/containerd/cgroups v1.0.4 // indirect + github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect + github.com/dgraph-io/badger v1.6.2 // indirect + github.com/dgraph-io/ristretto v0.0.2 // indirect + github.com/docker/go-units v0.5.0 // indirect + github.com/elastic/gosigar v0.14.2 // indirect + github.com/flynn/noise v1.0.0 // indirect + github.com/francoispqt/gojay v1.2.13 // indirect + github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/go-logr/logr v1.2.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect + github.com/godbus/dbus/v5 v5.1.0 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/mock v1.6.0 // indirect github.com/golang/protobuf v1.5.2 // indirect + github.com/golang/snappy v0.0.4 // indirect github.com/google/gopacket v1.1.19 // indirect + github.com/google/pprof v0.0.0-20221203041831-ce31453925ec // indirect + github.com/gopherjs/gopherjs v0.0.0-20190430165422-3e4dfb77656c // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect - github.com/hashicorp/golang-lru v0.5.4 // indirect github.com/huin/goupnp v1.0.3 // indirect - github.com/ipfs/bbloom v0.0.4 // indirect - github.com/ipfs/go-bitfield v1.0.0 // indirect github.com/ipfs/go-block-format v0.1.1 // indirect + github.com/ipfs/go-blockservice v0.5.0 // indirect + github.com/ipfs/go-ipfs-blockstore v1.2.0 // indirect + github.com/ipfs/go-ipfs-chunker v0.0.5 // indirect github.com/ipfs/go-ipfs-ds-help v1.1.0 // indirect - github.com/ipfs/go-ipfs-files v0.3.0 // indirect + github.com/ipfs/go-ipfs-exchange-interface v0.2.0 // indirect + github.com/ipfs/go-ipfs-exchange-offline v0.3.0 // indirect github.com/ipfs/go-ipfs-pq v0.0.3 // indirect - github.com/ipfs/go-ipld-cbor v0.0.6 // indirect + github.com/ipfs/go-ipfs-util v0.0.2 // indirect github.com/ipfs/go-ipld-legacy v0.1.1 // indirect + github.com/ipfs/go-ipns v0.3.0 // indirect + github.com/ipfs/go-unixfs v0.4.4 // indirect github.com/ipfs/go-verifcid v0.0.2 // indirect github.com/jackpal/go-nat-pmp v1.0.2 // indirect + github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect + github.com/klauspost/compress v1.15.12 // indirect github.com/klauspost/cpuid/v2 v2.2.3 // indirect github.com/koron/go-ssdp v0.0.3 // indirect github.com/libp2p/go-cidranger v1.1.0 // indirect + github.com/libp2p/go-flow-metrics v0.1.0 // indirect github.com/libp2p/go-libp2p-asn-util v0.2.0 // indirect - github.com/libp2p/go-libp2p-kad-dht v0.21.0 // indirect github.com/libp2p/go-libp2p-kbucket v0.5.0 // indirect github.com/libp2p/go-nat v0.1.0 // indirect github.com/libp2p/go-netroute v0.2.1 // indirect + github.com/libp2p/go-reuseport v0.2.0 // indirect + github.com/libp2p/go-yamux/v4 v4.0.0 // indirect + github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd // indirect github.com/mattn/go-isatty v0.0.17 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect + github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect + github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect github.com/minio/sha256-simd v1.0.0 // indirect - github.com/mr-tron/base58 v1.2.0 // indirect - github.com/multiformats/go-base32 v0.1.0 // indirect github.com/multiformats/go-base36 v0.2.0 // indirect - github.com/multiformats/go-multiaddr-dns v0.3.1 // indirect github.com/multiformats/go-multiaddr-fmt v0.1.0 // indirect github.com/multiformats/go-varint v0.0.7 // indirect + github.com/nxadm/tail v1.4.8 // indirect + github.com/onsi/ginkgo/v2 v2.5.1 // indirect + github.com/opencontainers/runtime-spec v1.0.2 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect + github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 // indirect - github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/polydawn/refmt v0.89.0 // indirect github.com/prometheus/client_model v0.3.0 // indirect github.com/prometheus/common v0.37.0 // indirect github.com/prometheus/procfs v0.8.0 // indirect - github.com/spaolacci/murmur3 v1.1.0 // indirect + github.com/quic-go/qpack v0.4.0 // indirect + github.com/quic-go/qtls-go1-19 v0.2.1 // indirect + github.com/quic-go/qtls-go1-20 v0.1.1 // indirect + github.com/quic-go/quic-go v0.33.0 // indirect + github.com/quic-go/webtransport-go v0.5.2 // indirect + github.com/raulk/go-watchdog v1.3.0 // indirect github.com/stretchr/objx v0.5.0 // indirect + github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect github.com/ucarion/urlpath v0.0.0-20200424170820-7ccc79b76bbb // indirect - github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc // indirect github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11 // indirect github.com/whyrusleeping/cbor-gen v0.0.0-20230126041949-52956bd4c9aa // indirect github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 // indirect - go.uber.org/atomic v1.10.0 // indirect + go.uber.org/dig v1.15.0 // indirect + go.uber.org/fx v1.18.2 // indirect golang.org/x/crypto v0.5.0 // indirect golang.org/x/exp v0.0.0-20230129154200-a960b3787bd2 // indirect golang.org/x/mod v0.7.0 // indirect - golang.org/x/net v0.5.0 // indirect - golang.org/x/sync v0.1.0 // indirect + golang.org/x/net v0.7.0 // indirect + golang.org/x/text v0.7.0 // indirect golang.org/x/tools v0.3.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect + google.golang.org/appengine v1.6.6 // indirect google.golang.org/protobuf v1.28.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect lukechampine.com/blake3 v1.1.7 // indirect + nhooyr.io/websocket v1.8.7 // indirect ) diff --git a/go.sum b/go.sum index ede1a802c3..5c21747a13 100644 --- a/go.sum +++ b/go.sum @@ -38,19 +38,12 @@ dmitri.shuralyov.com/html/belt v0.0.0-20180602232347-f7d459c86be0/go.mod h1:JLBr dmitri.shuralyov.com/service/change v0.0.0-20181023043359-a85b471d5412/go.mod h1:a1inKt/atXimZ4Mv927x+r7UpyzRUf4emIoiiSC2TN4= dmitri.shuralyov.com/state v0.0.0-20180228185332-28bcc343414c/go.mod h1:0PRwlb0D6DFvNNtx+9ybjezNCa8XF0xaYcETyp6rHWU= git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= -github.com/AndreasBriese/bbloom v0.0.0-20180913140656-343706a395b7/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= -github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= +github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 h1:cTp8I5+VIoKjsnZuH8vjyaysT/ses3EvZeaV/1UkF2M= +github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= -github.com/Kubuxu/go-os-helper v0.0.1/go.mod h1:N8B+I7vPCT80IcP58r50u4+gEEcsZETFUpAzWW2ep1Y= github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= -github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= -github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= -github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= -github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -59,16 +52,7 @@ github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk5 github.com/alecthomas/units v0.0.0-20210927113745-59d0afb8317a h1:E/8AP5dFtMhl5KPJz66Kt9G0n+7Sn41Fy1wv9/jHOrc= github.com/alecthomas/units v0.0.0-20210927113745-59d0afb8317a/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= -github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= -github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= -github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= -github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= -github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= -github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A= -github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= -github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= -github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= @@ -76,26 +60,9 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g= -github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32/go.mod h1:DrZx5ec/dmnfpw9KyYoQyYo7d0KEvTkk/5M/vbZjAr8= -github.com/btcsuite/btcd v0.0.0-20190523000118-16327141da8c/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI= -github.com/btcsuite/btcd v0.0.0-20190824003749-130ea5bddde3/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI= -github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= -github.com/btcsuite/btcd v0.21.0-beta/go.mod h1:ZSWyehm27aAuS9bvkATT+Xte3hjHZ+MRgMY/8NJ7K94= -github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= -github.com/btcsuite/btcutil v0.0.0-20190207003914-4c204d697803/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= -github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= -github.com/btcsuite/btcutil v1.0.2/go.mod h1:j9HUFwoQRsZL3V4n+qG+CUnEGHOarIxfC3Le2Yhbcts= -github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg= -github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY= -github.com/btcsuite/goleveldb v1.0.0/go.mod h1:QiK9vBlgftBg6rWQIj6wFzbPfRjiykIEhBH4obrXJ/I= -github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= -github.com/btcsuite/snappy-go v1.0.0/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= -github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= -github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= -github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= +github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= @@ -104,89 +71,80 @@ github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= +github.com/cilium/ebpf v0.2.0/go.mod h1:To2CFviqOWL/M0gIMsvSMlqe7em/l1ALkX1PyjrX2Qs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= -github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= +github.com/containerd/cgroups v0.0.0-20201119153540-4cbc285b3327/go.mod h1:ZJeTFisyysqgcCdecO57Dj79RfL0LNeGiFUqLYQRYLE= github.com/containerd/cgroups v1.0.4 h1:jN/mbWBEaz+T1pi5OFtnkQ+8qnmEbAr1Oo1FRm5B0dA= +github.com/containerd/cgroups v1.0.4/go.mod h1:nLNQtsF7Sl2HxNebu77i1R0oDlhiTG+kO4JTrUzo6IA= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d h1:t5Wuyh53qYyg9eqn4BbnlIT+vmhyww0TatL+zT3uWgI= github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd/v22 v22.1.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= -github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 h1:HVTnpeuvF6Owjd5mniCL8DEXo7uYXdQEmOP4FJbV5tg= github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3/go.mod h1:p1d6YEZWvFzEh4KLyvBcVSnrfNDDvK2zfK/4x2v/4pE= -github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/cskr/pubsub v1.0.2 h1:vlOzMhl6PFn60gRlTQQsIfVwaPB/B/8MziK8FhEPt/0= github.com/cskr/pubsub v1.0.2/go.mod h1:/8MzYXk/NJAz782G8RPkFzXTZVu63VotefPnR9TIRis= -github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davidlazar/go-crypto v0.0.0-20170701192655-dcfb0a7ac018/go.mod h1:rQYf4tfk5sSwFsnDg3qYaBxSjsD9S8+59vW0dKUgme4= github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c h1:pFUpOrbxDR6AkioZ1ySsx5yxlDQZ8stG2b88gTPxgJU= github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c/go.mod h1:6UhI8N9EjYm1c2odKpFpAYeR8dsBeM7PtzQhRgxRr9U= github.com/decred/dcrd/crypto/blake256 v1.0.0 h1:/8DMNYp9SGi5f0w7uCm6d6M4OU2rGFK09Y2A4Xv7EE0= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 h1:HbphB4TFFXpv7MNrT52FGrrgVXF1owhMVTHFZIlnvd4= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0/go.mod h1:DZGJHZMqrU4JJqFAWUS2UO1+lbSKsdiOoYi9Zzey7Fc= -github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218= -github.com/dgraph-io/badger v1.5.5-0.20190226225317-8115aed38f8f/go.mod h1:VZxzAIRPHRVNRKRo6AXrX9BJegn6il06VMTZVJYCIjQ= -github.com/dgraph-io/badger v1.6.0-rc1/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= -github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= -github.com/dgraph-io/badger v1.6.1/go.mod h1:FRmFw3uxvcpa8zG3Rxs0th+hCLIuaQg8HlNV5bjgnuU= +github.com/dgraph-io/badger v1.6.2 h1:mNw0qs90GVgGGWylh0umH5iag1j6n/PeJtNvL6KY/x8= +github.com/dgraph-io/badger v1.6.2/go.mod h1:JW2yswe3V058sS0kZ2h/AXeDSqFjxnZcRrVH//y2UQE= +github.com/dgraph-io/ristretto v0.0.2 h1:a5WaUrDa0qm0YrAAS1tUykT5El3kt62KNZZeMxQn3po= github.com/dgraph-io/ristretto v0.0.2/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= -github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= -github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= -github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= -github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= -github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= -github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= +github.com/elastic/gosigar v0.12.0/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs= github.com/elastic/gosigar v0.14.2 h1:Dg80n8cr90OZ7x+bAax/QjoW/XqTI11RmA79ZwIm9/4= -github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g= +github.com/elastic/gosigar v0.14.2/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/flynn/noise v1.0.0 h1:DlTHqmzmvcEiKj+4RYo/imoswx/4r6iBlCMfVtrMXpQ= github.com/flynn/noise v1.0.0/go.mod h1:xbMo+0i6+IGbYdJhF31t2eR1BIU0CYc12+BNAKwUTag= github.com/francoispqt/gojay v1.2.13 h1:d2m3sFjloqoIUQU3TsHBgj6qg/BVGlTBeHDUmyJnXKk= github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY= -github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= -github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= +github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/gabriel-vasile/mimetype v1.4.1 h1:TRWk7se+TOjCYgRth7+1/OYLNiRNIotknkFtf/dnN7Q= github.com/gabriel-vasile/mimetype v1.4.1/go.mod h1:05Vi0w3Y9c/lNvJOdmIwvrrAhX3rYhfQQCaf9VJcv7M= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= +github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= +github.com/gin-gonic/gin v1.6.3 h1:ahKqKTFpO5KTPHxWZjEdPScmYaGtLo8Y4DMHoEsnp14= +github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M= github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= -github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= @@ -198,26 +156,38 @@ github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q= +github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= +github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no= +github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= +github.com/go-playground/validator/v10 v10.2.0 h1:KgJ0snyC2R9VXYN2rneOtQcw5aHQB1Vv0sFl1UcHBOY= +github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0= +github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee h1:s+21KNqlpePfkah2I+gwHF8xmJWRjooY+5248k6m4A0= +github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= +github.com/gobwas/pool v0.2.0 h1:QEmUOlnSjWtnpRGHF3SauEiOsy82Cup83Vf2LcMlnc8= +github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= +github.com/gobwas/ws v1.0.2 h1:CoAavW/wd/kulfZmSIBt6p24n4j7tHgNVCjsfHVNUbo= +github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= +github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk= -github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= +github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= -github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20191027212112-611e8accdfc9/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -227,8 +197,8 @@ github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= +github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.0/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= @@ -246,6 +216,8 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -255,6 +227,7 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -276,81 +249,53 @@ github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20221203041831-ce31453925ec h1:fR20TYVVwhK4O7r7y+McjRYyaTH6/vjwJOajE+XhlzM= +github.com/google/pprof v0.0.0-20221203041831-ce31453925ec/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= github.com/googleapis/gax-go/v2 v2.0.3/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE08qbEPm1M08qg= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= -github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= -github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gopherjs/gopherjs v0.0.0-20190430165422-3e4dfb77656c h1:7lF+Vz0LqiRidnzC1Oq86fpX1q/iEv2KJdrCtttYjT4= +github.com/gopherjs/gopherjs v0.0.0-20190430165422-3e4dfb77656c/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= -github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= -github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU= github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48= -github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= -github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= -github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= -github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= -github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= -github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= -github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= -github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= -github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= -github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= -github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= github.com/huin/goupnp v1.0.0/go.mod h1:n9v9KO1tAxYH82qOn+UTIFQDmx5n1Zxd/ClZDMX7Bnc= github.com/huin/goupnp v1.0.3 h1:N8No57ls+MnjlB+JPiCVSOyy/ot7MJTqlo7rn+NYSqQ= github.com/huin/goupnp v1.0.3/go.mod h1:ZxNlw5WqJj6wSsRK5+YfflQGXYfccj5VgQsMNixHM7Y= github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/go-bitfield v1.0.0 h1:y/XHm2GEmD9wKngheWNNCNL0pzrWXZwCdQGv1ikXknQ= -github.com/ipfs/go-bitfield v1.0.0/go.mod h1:N/UiujQy+K+ceU1EF5EkVd1TNqevLrCQMIcAEPrdtus= -github.com/ipfs/go-bitswap v0.5.1/go.mod h1:P+ckC87ri1xFLvk74NlXdP0Kj9RmWAh4+H78sC6Qopo= +github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= +github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-bitswap v0.11.0 h1:j1WVvhDX1yhG32NTC9xfxnqycqYIlhzEzLXG/cU1HyQ= github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY= github.com/ipfs/go-block-format v0.0.3/go.mod h1:4LmD4ZUw0mhO+JSKdpWwrzATiEfM7WWgQ8H5l6P8MVk= github.com/ipfs/go-block-format v0.1.1 h1:129vSO3zwbsYADcyQWcOYiuCpAqt462SFfqFHdFJhhI= github.com/ipfs/go-block-format v0.1.1/go.mod h1:+McEIT+g52p+zz5xGAABGSOKrzmrdX97bc0USBdWPUs= -github.com/ipfs/go-blockservice v0.2.1/go.mod h1:k6SiwmgyYgs4M/qt+ww6amPeUH9EISLRBnvUurKJhi8= github.com/ipfs/go-blockservice v0.5.0 h1:B2mwhhhVQl2ntW2EIpaWPwSCxSuqr5fFA93Ms4bYLEY= github.com/ipfs/go-blockservice v0.5.0/go.mod h1:W6brZ5k20AehbmERplmERn8o2Ni3ZZubvAxaIUeaT6w= github.com/ipfs/go-cid v0.0.1/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= @@ -362,62 +307,40 @@ github.com/ipfs/go-cid v0.0.6/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqg github.com/ipfs/go-cid v0.0.7/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I= github.com/ipfs/go-cid v0.3.2 h1:OGgOd+JCFM+y1DjWPmVH+2/4POtpDzwcr7VgnB7mZXc= github.com/ipfs/go-cid v0.3.2/go.mod h1:gQ8pKqT/sUxGY+tIwy1RPpAojYu7jAyCp5Tz1svoupw= -github.com/ipfs/go-datastore v0.0.1/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE= -github.com/ipfs/go-datastore v0.1.1/go.mod h1:w38XXW9kVFNp57Zj5knbKWM2T+KOZCGDRVNdgPHtbHw= -github.com/ipfs/go-datastore v0.4.0/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA= -github.com/ipfs/go-datastore v0.4.1/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA= -github.com/ipfs/go-datastore v0.4.4/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA= -github.com/ipfs/go-datastore v0.4.5/go.mod h1:eXTcaaiN6uOlVCLS9GjJUJtlvJfM3xk23w3fyfrmmJs= +github.com/ipfs/go-cidutil v0.1.0 h1:RW5hO7Vcf16dplUU60Hs0AKDkQAVPVplr7lk97CFL+Q= +github.com/ipfs/go-cidutil v0.1.0/go.mod h1:e7OEVBMIv9JaOxt9zaGEmAoSlXW9jdFZ5lP/0PwcfpA= github.com/ipfs/go-datastore v0.5.0/go.mod h1:9zhEApYMTl17C8YDp7JmU7sQZi2/wqiYh73hakZ90Bk= github.com/ipfs/go-datastore v0.6.0 h1:JKyz+Gvz1QEZw0LsX1IBn+JFCJQH4SJVFtM4uWU0Myk= github.com/ipfs/go-datastore v0.6.0/go.mod h1:rt5M3nNbSO/8q1t4LNkLyUwRs8HupMeN/8O4Vn9YAT8= github.com/ipfs/go-detect-race v0.0.1 h1:qX/xay2W3E4Q1U7d9lNs1sU9nvguX0a7319XbyQ6cOk= github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46UU0LZ723meps= -github.com/ipfs/go-ds-badger v0.0.2/go.mod h1:Y3QpeSFWQf6MopLTiZD+VT6IC1yZqaGmjvRcKeSGij8= -github.com/ipfs/go-ds-badger v0.0.5/go.mod h1:g5AuuCGmr7efyzQhLL8MzwqcauPojGPUaHzfGTzuE3s= -github.com/ipfs/go-ds-badger v0.2.1/go.mod h1:Tx7l3aTph3FMFrRS838dcSJh+jjA7cX9DrGVwx/NOwE= -github.com/ipfs/go-ds-badger v0.2.3/go.mod h1:pEYw0rgg3FIrywKKnL+Snr+w/LjJZVMTBRn4FS6UHUk= -github.com/ipfs/go-ds-leveldb v0.0.1/go.mod h1:feO8V3kubwsEF22n0YRQCffeb79OOYIykR4L04tMOYc= -github.com/ipfs/go-ds-leveldb v0.4.1/go.mod h1:jpbku/YqBSsBc1qgME8BkWS4AxzF2cEu1Ii2r79Hh9s= -github.com/ipfs/go-ds-leveldb v0.4.2/go.mod h1:jpbku/YqBSsBc1qgME8BkWS4AxzF2cEu1Ii2r79Hh9s= -github.com/ipfs/go-fetcher v1.6.1 h1:UFuRVYX5AIllTiRhi5uK/iZkfhSpBCGX7L70nSZEmK8= -github.com/ipfs/go-fetcher v1.6.1/go.mod h1:27d/xMV8bodjVs9pugh/RCjjK2OZ68UgAMspMdingNo= -github.com/ipfs/go-ipfs-blockstore v0.2.1/go.mod h1:jGesd8EtCM3/zPgx+qr0/feTXGUeRai6adgwC+Q+JvE= +github.com/ipfs/go-ds-badger v0.3.0 h1:xREL3V0EH9S219kFFueOYJJTcjgNSZ2HY1iSvN7U1Ro= +github.com/ipfs/go-ds-badger v0.3.0/go.mod h1:1ke6mXNqeV8K3y5Ak2bAA0osoTfmxUdupVCGm4QUIek= +github.com/ipfs/go-ds-leveldb v0.5.0 h1:s++MEBbD3ZKc9/8/njrn4flZLnCuY9I79v94gBUNumo= +github.com/ipfs/go-ds-leveldb v0.5.0/go.mod h1:d3XG9RUDzQ6V4SHi8+Xgj9j1XuEk1z82lquxrVbml/Q= github.com/ipfs/go-ipfs-blockstore v1.2.0 h1:n3WTeJ4LdICWs/0VSfjHrlqpPpl6MZ+ySd3j8qz0ykw= github.com/ipfs/go-ipfs-blockstore v1.2.0/go.mod h1:eh8eTFLiINYNSNawfZOC7HOxNTxpB1PFuA5E1m/7exE= github.com/ipfs/go-ipfs-blocksutil v0.0.1 h1:Eh/H4pc1hsvhzsQoMEP3Bke/aW5P5rVM1IWFJMcGIPQ= github.com/ipfs/go-ipfs-blocksutil v0.0.1/go.mod h1:Yq4M86uIOmxmGPUHv/uI7uKqZNtLb449gwKqXjIsnRk= -github.com/ipfs/go-ipfs-chunker v0.0.1 h1:cHUUxKFQ99pozdahi+uSC/3Y6HeRpi9oTeUHbE27SEw= -github.com/ipfs/go-ipfs-chunker v0.0.1/go.mod h1:tWewYK0we3+rMbOh7pPFGDyypCtvGcBFymgY4rSDLAw= +github.com/ipfs/go-ipfs-chunker v0.0.5 h1:ojCf7HV/m+uS2vhUGWcogIIxiO5ubl5O57Q7NapWLY8= +github.com/ipfs/go-ipfs-chunker v0.0.5/go.mod h1:jhgdF8vxRHycr00k13FM8Y0E+6BoalYeobXmUyTreP8= github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= github.com/ipfs/go-ipfs-delay v0.0.1 h1:r/UXYyRcddO6thwOnhiznIAiSvxMECGgtv35Xs1IeRQ= github.com/ipfs/go-ipfs-delay v0.0.1/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= -github.com/ipfs/go-ipfs-ds-help v0.1.1/go.mod h1:SbBafGJuGsPI/QL3j9Fc5YPLeAu+SzOkI0gFwAg+mOs= github.com/ipfs/go-ipfs-ds-help v1.1.0 h1:yLE2w9RAsl31LtfMt91tRZcrx+e61O5mDxFRR994w4Q= github.com/ipfs/go-ipfs-ds-help v1.1.0/go.mod h1:YR5+6EaebOhfcqVCyqemItCLthrpVNot+rsOU/5IatU= -github.com/ipfs/go-ipfs-exchange-interface v0.1.0/go.mod h1:ych7WPlyHqFvCi/uQI48zLZuAWVP5iTQPXEfVaw5WEI= github.com/ipfs/go-ipfs-exchange-interface v0.2.0 h1:8lMSJmKogZYNo2jjhUs0izT+dck05pqUw4mWNW9Pw6Y= github.com/ipfs/go-ipfs-exchange-interface v0.2.0/go.mod h1:z6+RhJuDQbqKguVyslSOuVDhqF9JtTrO3eptSAiW2/Y= -github.com/ipfs/go-ipfs-exchange-offline v0.1.1/go.mod h1:vTiBRIbzSwDD0OWm+i3xeT0mO7jG2cbJYatp3HPk5XY= github.com/ipfs/go-ipfs-exchange-offline v0.3.0 h1:c/Dg8GDPzixGd0MC8Jh6mjOwU57uYokgWRFidfvEkuA= github.com/ipfs/go-ipfs-exchange-offline v0.3.0/go.mod h1:MOdJ9DChbb5u37M1IcbrRB02e++Z7521fMxqCNRrz9s= -github.com/ipfs/go-ipfs-files v0.0.3/go.mod h1:INEFm0LL2LWXBhNJ2PMIIb2w45hpXgPjNoE7yA8Y1d4= -github.com/ipfs/go-ipfs-files v0.3.0 h1:fallckyc5PYjuMEitPNrjRfpwl7YFt69heCOUhsbGxQ= -github.com/ipfs/go-ipfs-files v0.3.0/go.mod h1:xAUtYMwB+iu/dtf6+muHNSFQCJG2dSiStR2P6sn9tIM= -github.com/ipfs/go-ipfs-posinfo v0.0.1 h1:Esoxj+1JgSjX0+ylc0hUmJCOv6V2vFoZiETLR6OtpRs= -github.com/ipfs/go-ipfs-posinfo v0.0.1/go.mod h1:SwyeVP+jCwiDu0C313l/8jg6ZxM0qqtlt2a0vILTc1A= -github.com/ipfs/go-ipfs-pq v0.0.2/go.mod h1:LWIqQpqfRG3fNc5XsnIhz/wQ2XXGyugQwls7BgUmUfY= github.com/ipfs/go-ipfs-pq v0.0.3 h1:YpoHVJB+jzK15mr/xsWC574tyDLkezVrDNeaalQBsTE= github.com/ipfs/go-ipfs-pq v0.0.3/go.mod h1:btNw5hsHBpRcSSgZtiNm/SLj5gYIZ18AKtv3kERkRb4= github.com/ipfs/go-ipfs-redirects-file v0.1.1 h1:Io++k0Vf/wK+tfnhEh63Yte1oQK5VGT2hIEYpD0Rzx8= github.com/ipfs/go-ipfs-redirects-file v0.1.1/go.mod h1:tAwRjCV0RjLTjH8DR/AU7VYvfQECg+lpUy2Mdzv7gyk= -github.com/ipfs/go-ipfs-routing v0.2.1/go.mod h1:xiNNiwgjmLqPS1cimvAw6EyB9rkVDbiocA4yY+wRNLM= github.com/ipfs/go-ipfs-routing v0.3.0 h1:9W/W3N+g+y4ZDeffSgqhgo7BsBSJwPMcyssET9OWevc= -github.com/ipfs/go-ipfs-routing v0.3.0/go.mod h1:dKqtTFIql7e1zYsEuWLyuOU+E0WJWW8JjbTPLParDWo= github.com/ipfs/go-ipfs-util v0.0.1/go.mod h1:spsl5z8KUnrve+73pOhSVZND1SIxPW5RyBCNzQxlJBc= github.com/ipfs/go-ipfs-util v0.0.2 h1:59Sswnk1MFaiq+VcaknX7aYEyGyGDAA73ilhEK2POp8= github.com/ipfs/go-ipfs-util v0.0.2/go.mod h1:CbPtkWJzjLdEcezDns2XYaehFVNXG9zrdrtMecczcsQ= -github.com/ipfs/go-ipld-cbor v0.0.5/go.mod h1:BkCduEx3XBCO6t2Sfo5BaHzuok7hbhdMm9Oh8B2Ftq4= github.com/ipfs/go-ipld-cbor v0.0.6 h1:pYuWHyvSpIsOOLw4Jy7NbBkCyzLDcl64Bf/LZW7eBQ0= github.com/ipfs/go-ipld-cbor v0.0.6/go.mod h1:ssdxxaLJPXH7OjF5V4NSjBbcfh+evoR4ukuru0oPXMA= github.com/ipfs/go-ipld-format v0.0.1/go.mod h1:kyJtbkDALmFHv3QR6et67i35QzO3S0dCDnkOJhcZkms= @@ -425,83 +348,53 @@ github.com/ipfs/go-ipld-format v0.2.0/go.mod h1:3l3C1uKoadTPbeNfrDi+xMInYKlx2Cvg github.com/ipfs/go-ipld-format v0.3.0/go.mod h1:co/SdBE8h99968X0hViiw1MNlh6fvxxnHpvVLnH7jSM= github.com/ipfs/go-ipld-format v0.4.0 h1:yqJSaJftjmjc9jEOFYlpkwOLVKv68OD27jFLlSghBlQ= github.com/ipfs/go-ipld-format v0.4.0/go.mod h1:co/SdBE8h99968X0hViiw1MNlh6fvxxnHpvVLnH7jSM= -github.com/ipfs/go-ipld-legacy v0.1.0/go.mod h1:86f5P/srAmh9GcIcWQR9lfFLZPrIyyXQeVlOWeeWEuI= github.com/ipfs/go-ipld-legacy v0.1.1 h1:BvD8PEuqwBHLTKqlGFTHSwrwFOMkVESEvwIYwR2cdcc= github.com/ipfs/go-ipld-legacy v0.1.1/go.mod h1:8AyKFCjgRPsQFf15ZQgDB8Din4DML/fOmKZkkFkrIEg= github.com/ipfs/go-ipns v0.3.0 h1:ai791nTgVo+zTuq2bLvEGmWP1M0A6kGTXUsgv/Yq67A= github.com/ipfs/go-ipns v0.3.0/go.mod h1:3cLT2rbvgPZGkHJoPO1YMJeh6LtkxopCkKFcio/wE24= github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM= -github.com/ipfs/go-log v1.0.2/go.mod h1:1MNjMxe0u6xvJZgeqbJ8vdo2TKaGwZ1a0Bpza+sr2Sk= -github.com/ipfs/go-log v1.0.3/go.mod h1:OsLySYkwIbiSUR/yBTdv1qPtcE4FW3WPWk/ewz9Ru+A= -github.com/ipfs/go-log v1.0.4/go.mod h1:oDCg2FkjogeFOhqqb+N39l2RpTNPL6F/StPkB3kPgcs= github.com/ipfs/go-log v1.0.5 h1:2dOuUCB1Z7uoczMWgAyDck5JLb72zHzrMnGnCNNbvY8= github.com/ipfs/go-log v1.0.5/go.mod h1:j0b8ZoR+7+R99LD9jZ6+AJsrzkPbSXbZfGakb5JPtIo= -github.com/ipfs/go-log/v2 v2.0.2/go.mod h1:O7P1lJt27vWHhOwQmcFEvlmo49ry2VY2+JfBWFaa9+0= -github.com/ipfs/go-log/v2 v2.0.3/go.mod h1:O7P1lJt27vWHhOwQmcFEvlmo49ry2VY2+JfBWFaa9+0= github.com/ipfs/go-log/v2 v2.0.5/go.mod h1:eZs4Xt4ZUJQFM3DlanGhy7TkwwawCZcSByscwkWG+dw= -github.com/ipfs/go-log/v2 v2.1.1/go.mod h1:2v2nsGfZsvvAJz13SyFzf9ObaqwHiHxsPLEHntrv9KM= github.com/ipfs/go-log/v2 v2.1.3/go.mod h1:/8d0SH3Su5Ooc31QlL1WysJhvyOTDCjcCZ9Axpmri6g= -github.com/ipfs/go-log/v2 v2.3.0/go.mod h1:QqGoj30OTpnKaG/LKTGTxoP2mmQtjVMEnK72gynbe/g= github.com/ipfs/go-log/v2 v2.5.1 h1:1XdUzF7048prq4aBjDQQ4SL5RxftpRGdXhNRwKSAlcY= github.com/ipfs/go-log/v2 v2.5.1/go.mod h1:prSpmC1Gpllc9UYWxDiZDreBYw7zp4Iqp1kOLU9U5UI= -github.com/ipfs/go-merkledag v0.5.1/go.mod h1:cLMZXx8J08idkp5+id62iVftUQV+HlYJ3PIhDfZsjA4= github.com/ipfs/go-merkledag v0.9.0 h1:DFC8qZ96Dz1hMT7dtIpcY524eFFDiEWAF8hNJHWW2pk= github.com/ipfs/go-merkledag v0.9.0/go.mod h1:bPHqkHt5OZ0p1n3iqPeDiw2jIBkjAytRjS3WSBwjq90= github.com/ipfs/go-metrics-interface v0.0.1 h1:j+cpbjYvu4R8zbleSs36gvB7jR+wsL2fGD6n0jO4kdg= github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j/b/tL7HTWtJ4VPgWY= -github.com/ipfs/go-namesys v0.7.0 h1:xqosk71GIVRkFDtF2UNRcXn4LdNeo7tzuy8feHD6NbU= -github.com/ipfs/go-namesys v0.7.0/go.mod h1:KYSZBVZG3VJC34EfqqJPG7T48aWgxseoMPAPA5gLyyQ= -github.com/ipfs/go-path v0.3.1 h1:wkeaCWE/NTuuPGlEkLTsED5UkzfKYZpxaFFPgk8ZVLE= -github.com/ipfs/go-path v0.3.1/go.mod h1:eNLsxJEEMxn/CDzUJ6wuNl+6No6tEUhOZcPKsZsYX0E= -github.com/ipfs/go-peertaskqueue v0.7.0/go.mod h1:M/akTIE/z1jGNXMU7kFB4TeSEFvj68ow0Rrb04donIU= github.com/ipfs/go-peertaskqueue v0.8.1 h1:YhxAs1+wxb5jk7RvS0LHdyiILpNmRIRnZVztekOF0pg= github.com/ipfs/go-peertaskqueue v0.8.1/go.mod h1:Oxxd3eaK279FxeydSPPVGHzbwVeHjatZ2GA8XD+KbPU= -github.com/ipfs/go-unixfs v0.3.1 h1:LrfED0OGfG98ZEegO4/xiprx2O+yS+krCMQSp7zLVv8= -github.com/ipfs/go-unixfs v0.3.1/go.mod h1:h4qfQYzghiIc8ZNFKiLMFWOTzrWIAtzYQ59W/pCFf1o= -github.com/ipfs/go-unixfsnode v1.5.1 h1:JcR3t5C2nM1V7PMzhJ/Qmo19NkoFIKweDSZyDx+CjkI= -github.com/ipfs/go-unixfsnode v1.5.1/go.mod h1:ed79DaG9IEuZITJVQn4U6MZDftv6I3ygUBLPfhEbHvk= -github.com/ipfs/go-verifcid v0.0.1/go.mod h1:5Hrva5KBeIog4A+UpqlaIU+DEstipcJYQQZc0g37pY0= +github.com/ipfs/go-unixfs v0.4.4 h1:D/dLBOJgny5ZLIur2vIXVQVW0EyDHdOMBDEhgHrt6rY= +github.com/ipfs/go-unixfs v0.4.4/go.mod h1:TSG7G1UuT+l4pNj91raXAPkX0BhJi3jST1FDTfQ5QyM= +github.com/ipfs/go-unixfsnode v1.5.2 h1:CvsiTt58W2uR5dD8bqQv+aAY0c1qolmXmSyNbPHYiew= +github.com/ipfs/go-unixfsnode v1.5.2/go.mod h1:NlOebRwYx8lMCNMdhAhEspYPBD3obp7TE0LvBqHY+ks= github.com/ipfs/go-verifcid v0.0.2 h1:XPnUv0XmdH+ZIhLGKg6U2vaPaRDXb9urMyNVCE7uvTs= github.com/ipfs/go-verifcid v0.0.2/go.mod h1:40cD9x1y4OWnFXbLNJYRe7MpNvWlMn3LZAG5Wb4xnPU= -github.com/ipfs/interface-go-ipfs-core v0.10.0 h1:b/psL1oqJcySdQAsIBfW5ZJJkOAsYlhWtC0/Qvr4WiM= -github.com/ipfs/interface-go-ipfs-core v0.10.0/go.mod h1:F3EcmDy53GFkF0H3iEJpfJC320fZ/4G60eftnItrrJ0= github.com/ipld/go-car v0.5.0 h1:kcCEa3CvYMs0iE5BzD5sV7O2EwMiCIp3uF8tA6APQT8= github.com/ipld/go-car v0.5.0/go.mod h1:ppiN5GWpjOZU9PgpAZ9HbZd9ZgSpwPMr48fGRJOWmvE= github.com/ipld/go-car/v2 v2.5.1 h1:U2ux9JS23upEgrJScW8VQuxmE94560kYxj9CQUpcfmk= github.com/ipld/go-car/v2 v2.5.1/go.mod h1:jKjGOqoCj5zn6KjnabD6JbnCsMntqU2hLiU6baZVO3E= -github.com/ipld/go-codec-dagpb v1.3.0/go.mod h1:ga4JTU3abYApDC3pZ00BC2RSvC3qfBb9MSJkMLSwnhA= github.com/ipld/go-codec-dagpb v1.5.0 h1:RspDRdsJpLfgCI0ONhTAnbHdySGD4t+LHSPK4X1+R0k= github.com/ipld/go-codec-dagpb v1.5.0/go.mod h1:0yRIutEFD8o1DGVqw4RSHh+BUTlJA9XWldxaaWR/o4g= github.com/ipld/go-ipld-prime v0.9.1-0.20210324083106-dc342a9917db/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8= -github.com/ipld/go-ipld-prime v0.11.0/go.mod h1:+WIAkokurHmZ/KwzDOMUuoeJgaRQktHtEaLglS3ZeV8= github.com/ipld/go-ipld-prime v0.19.0 h1:5axC7rJmPc17Emw6TelxGwnzALk0PdupZ2oj2roDj04= github.com/ipld/go-ipld-prime v0.19.0/go.mod h1:Q9j3BaVXwaA3o5JUDNvptDDr/x8+F7FG6XJ8WI3ILg4= github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20211210234204-ce2a1c70cd73 h1:TsyATB2ZRRQGTwafJdgEUQkmjOExRV0DNokcihZxbnQ= -github.com/jackpal/gateway v1.0.5/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA= -github.com/jackpal/go-nat-pmp v1.0.1/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= -github.com/jbenet/go-cienv v0.0.0-20150120210510-1bb1476777ec/go.mod h1:rGaEvXB4uRSZMmzKNLoXvTu1sfx+1kv/DojUlPrSZGs= github.com/jbenet/go-cienv v0.1.0/go.mod h1:TqNnHUmJgXau0nCzC7kXWeotg3J9W34CUv5Djy1+FlA= -github.com/jbenet/go-temp-err-catcher v0.0.0-20150120210811-aac704a3f4f2/go.mod h1:8GXXJV31xl8whumTzdZsTt3RnUIiPqzkyf7mxToRCMs= github.com/jbenet/go-temp-err-catcher v0.1.0 h1:zpb3ZH6wIE8Shj2sKS+khgRvf7T7RABoLk/+KKHggpk= github.com/jbenet/go-temp-err-catcher v0.1.0/go.mod h1:0kJRvmDZXNMIiJirNPEYfhpPwbGVtZVWC34vc5WLsDk= -github.com/jbenet/goprocess v0.0.0-20160826012719-b497e2f366b8/go.mod h1:Ly/wlsjFq/qrU3Rar62tu1gASgGw6chQbSh/XgIIXCY= -github.com/jbenet/goprocess v0.1.3/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4= github.com/jbenet/goprocess v0.1.4 h1:DRGOFReOMqqDNXwW70QkacFW0YN9QnwLV0Vqk+3oU0o= github.com/jbenet/goprocess v0.1.4/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4= github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU= -github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= -github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= -github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= -github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= -github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= @@ -510,13 +403,13 @@ github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= -github.com/kami-zh/go-capturer v0.0.0-20171211120116-e492ea43421d/go.mod h1:P2viExyCEfeWGU259JnaQ34Inuec4R38JCyBx2edgD0= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= +github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.15.12 h1:YClS/PImqYbn+UILDnqxQCZ3RehC9N318SU3kElDUEM= +github.com/klauspost/compress v1.15.12/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.3 h1:sxCkb+qR91z4vsqw4vGGZlDgPz3G7gjaLyK3V8y70BU= @@ -535,220 +428,58 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/libp2p/go-addr-util v0.0.1/go.mod h1:4ac6O7n9rIAKB1dnd+s8IbbMXkt+oBpzX4/+RACcnlQ= -github.com/libp2p/go-addr-util v0.0.2/go.mod h1:Ecd6Fb3yIuLzq4bD7VcywcVSBtefcAwnUISBM3WG15E= -github.com/libp2p/go-buffer-pool v0.0.1/go.mod h1:xtyIz9PMobb13WaxR6Zo1Pd1zXJKYg0a8KiIvDp3TzQ= +github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= +github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= github.com/libp2p/go-buffer-pool v0.0.2/go.mod h1:MvaB6xw5vOrDl8rYZGLFdKAuk/hRoRZd1Vi32+RXyFM= github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6cdF0Y8= github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg= github.com/libp2p/go-cidranger v1.1.0 h1:ewPN8EZ0dd1LSnrtuwd4709PXVcITVeuwbag38yPW7c= github.com/libp2p/go-cidranger v1.1.0/go.mod h1:KWZTfSr+r9qEo9OkI9/SIEeAtw+NNoU0dXIXt15Okic= -github.com/libp2p/go-conn-security-multistream v0.1.0/go.mod h1:aw6eD7LOsHEX7+2hJkDxw1MteijaVcI+/eP2/x3J1xc= -github.com/libp2p/go-conn-security-multistream v0.2.0/go.mod h1:hZN4MjlNetKD3Rq5Jb/P5ohUnFLNzEAR4DLSzpn2QLU= -github.com/libp2p/go-conn-security-multistream v0.2.1/go.mod h1:cR1d8gA0Hr59Fj6NhaTpFhJZrjSYuNmhpT2r25zYR70= -github.com/libp2p/go-eventbus v0.1.0/go.mod h1:vROgu5cs5T7cv7POWlWxBaVLxfSegC5UGQf8A2eEmx4= -github.com/libp2p/go-eventbus v0.2.1/go.mod h1:jc2S4SoEVPP48H9Wpzm5aiGwUCBMfGhVhhBjyhhCJs8= -github.com/libp2p/go-flow-metrics v0.0.1/go.mod h1:Iv1GH0sG8DtYN3SVJ2eG221wMiNpZxBdp967ls1g+k8= -github.com/libp2p/go-flow-metrics v0.0.3/go.mod h1:HeoSNUrOJVK1jEpDqVEiUOIXqhbnS27omG0uWU5slZs= github.com/libp2p/go-flow-metrics v0.1.0 h1:0iPhMI8PskQwzh57jB9WxIuIOQ0r+15PChFGkx3Q3WM= -github.com/libp2p/go-libp2p v0.6.1/go.mod h1:CTFnWXogryAHjXAKEbOf1OWY+VeAP3lDMZkfEI5sT54= -github.com/libp2p/go-libp2p v0.7.0/go.mod h1:hZJf8txWeCduQRDC/WSqBGMxaTHCOYHt2xSU1ivxn0k= -github.com/libp2p/go-libp2p v0.7.4/go.mod h1:oXsBlTLF1q7pxr+9w6lqzS1ILpyHsaBPniVO7zIHGMw= -github.com/libp2p/go-libp2p v0.8.1/go.mod h1:QRNH9pwdbEBpx5DTJYg+qxcVaDMAz3Ee/qDKwXujH5o= -github.com/libp2p/go-libp2p v0.14.3/go.mod h1:d12V4PdKbpL0T1/gsUNN8DfgMuRPDX8bS2QxCZlwRH0= -github.com/libp2p/go-libp2p v0.25.1 h1:YK+YDCHpYyTvitKWVxa5PfElgIpOONU01X5UcLEwJGA= -github.com/libp2p/go-libp2p v0.25.1/go.mod h1:xnK9/1d9+jeQCVvi/f1g12KqtVi/jP/SijtKV1hML3g= +github.com/libp2p/go-flow-metrics v0.1.0/go.mod h1:4Xi8MX8wj5aWNDAZttg6UPmc0ZrnFNsMtpsYUClFtro= +github.com/libp2p/go-libp2p v0.26.2 h1:eHEoW/696FP7/6DxOvcrKfTD6Bi0DExxiMSZUJxswA0= +github.com/libp2p/go-libp2p v0.26.2/go.mod h1:x75BN32YbwuY0Awm2Uix4d4KOz+/4piInkp4Wr3yOo8= github.com/libp2p/go-libp2p-asn-util v0.2.0 h1:rg3+Os8jbnO5DxkC7K/Utdi+DkY3q/d1/1q+8WeNAsw= github.com/libp2p/go-libp2p-asn-util v0.2.0/go.mod h1:WoaWxbHKBymSN41hWSq/lGKJEca7TNm58+gGJi2WsLI= -github.com/libp2p/go-libp2p-autonat v0.1.1/go.mod h1:OXqkeGOY2xJVWKAGV2inNF5aKN/djNA3fdpCWloIudE= -github.com/libp2p/go-libp2p-autonat v0.2.0/go.mod h1:DX+9teU4pEEoZUqR1PiMlqliONQdNbfzE1C718tcViI= -github.com/libp2p/go-libp2p-autonat v0.2.1/go.mod h1:MWtAhV5Ko1l6QBsHQNSuM6b1sRkXrpk0/LqCr+vCVxI= -github.com/libp2p/go-libp2p-autonat v0.2.2/go.mod h1:HsM62HkqZmHR2k1xgX34WuWDzk/nBwNHoeyyT4IWV6A= -github.com/libp2p/go-libp2p-autonat v0.4.2/go.mod h1:YxaJlpr81FhdOv3W3BTconZPfhaYivRdf53g+S2wobk= -github.com/libp2p/go-libp2p-blankhost v0.1.1/go.mod h1:pf2fvdLJPsC1FsVrNP3DUUvMzUts2dsLLBEpo1vW1ro= -github.com/libp2p/go-libp2p-blankhost v0.1.4/go.mod h1:oJF0saYsAXQCSfDq254GMNmLNz6ZTHTOvtF4ZydUvwU= -github.com/libp2p/go-libp2p-blankhost v0.2.0/go.mod h1:eduNKXGTioTuQAUcZ5epXi9vMl+t4d8ugUBRQ4SqaNQ= -github.com/libp2p/go-libp2p-circuit v0.1.4/go.mod h1:CY67BrEjKNDhdTk8UgBX1Y/H5c3xkAcs3gnksxY7osU= -github.com/libp2p/go-libp2p-circuit v0.2.1/go.mod h1:BXPwYDN5A8z4OEY9sOfr2DUQMLQvKt/6oku45YUmjIo= -github.com/libp2p/go-libp2p-circuit v0.4.0/go.mod h1:t/ktoFIUzM6uLQ+o1G6NuBl2ANhBKN9Bc8jRIk31MoA= -github.com/libp2p/go-libp2p-core v0.0.1/go.mod h1:g/VxnTZ/1ygHxH3dKok7Vno1VfpvGcGip57wjTU4fco= -github.com/libp2p/go-libp2p-core v0.0.2/go.mod h1:9dAcntw/n46XycV4RnlBq3BpgrmyUi9LuoTNdPrbUco= -github.com/libp2p/go-libp2p-core v0.0.4/go.mod h1:jyuCQP356gzfCFtRKyvAbNkyeuxb7OlyhWZ3nls5d2I= -github.com/libp2p/go-libp2p-core v0.2.0/go.mod h1:X0eyB0Gy93v0DZtSYbEM7RnMChm9Uv3j7yRXjO77xSI= -github.com/libp2p/go-libp2p-core v0.2.2/go.mod h1:8fcwTbsG2B+lTgRJ1ICZtiM5GWCWZVoVrLaDRvIRng0= -github.com/libp2p/go-libp2p-core v0.2.4/go.mod h1:STh4fdfa5vDYr0/SzYYeqnt+E6KfEV5VxfIrm0bcI0g= -github.com/libp2p/go-libp2p-core v0.3.0/go.mod h1:ACp3DmS3/N64c2jDzcV429ukDpicbL6+TrrxANBjPGw= -github.com/libp2p/go-libp2p-core v0.3.1/go.mod h1:thvWy0hvaSBhnVBaW37BvzgVV68OUhgJJLAa6almrII= -github.com/libp2p/go-libp2p-core v0.4.0/go.mod h1:49XGI+kc38oGVwqSBhDEwytaAxgZasHhFfQKibzTls0= -github.com/libp2p/go-libp2p-core v0.5.0/go.mod h1:49XGI+kc38oGVwqSBhDEwytaAxgZasHhFfQKibzTls0= -github.com/libp2p/go-libp2p-core v0.5.1/go.mod h1:uN7L2D4EvPCvzSH5SrhR72UWbnSGpt5/a35Sm4upn4Y= -github.com/libp2p/go-libp2p-core v0.5.4/go.mod h1:uN7L2D4EvPCvzSH5SrhR72UWbnSGpt5/a35Sm4upn4Y= -github.com/libp2p/go-libp2p-core v0.5.5/go.mod h1:vj3awlOr9+GMZJFH9s4mpt9RHHgGqeHCopzbYKZdRjM= -github.com/libp2p/go-libp2p-core v0.5.6/go.mod h1:txwbVEhHEXikXn9gfC7/UDDw7rkxuX0bJvM49Ykaswo= -github.com/libp2p/go-libp2p-core v0.5.7/go.mod h1:txwbVEhHEXikXn9gfC7/UDDw7rkxuX0bJvM49Ykaswo= -github.com/libp2p/go-libp2p-core v0.6.0/go.mod h1:txwbVEhHEXikXn9gfC7/UDDw7rkxuX0bJvM49Ykaswo= -github.com/libp2p/go-libp2p-core v0.7.0/go.mod h1:FfewUH/YpvWbEB+ZY9AQRQ4TAD8sJBt/G1rVvhz5XT8= -github.com/libp2p/go-libp2p-core v0.8.0/go.mod h1:FfewUH/YpvWbEB+ZY9AQRQ4TAD8sJBt/G1rVvhz5XT8= -github.com/libp2p/go-libp2p-core v0.8.1/go.mod h1:FfewUH/YpvWbEB+ZY9AQRQ4TAD8sJBt/G1rVvhz5XT8= -github.com/libp2p/go-libp2p-core v0.8.2/go.mod h1:FfewUH/YpvWbEB+ZY9AQRQ4TAD8sJBt/G1rVvhz5XT8= -github.com/libp2p/go-libp2p-core v0.8.5/go.mod h1:FfewUH/YpvWbEB+ZY9AQRQ4TAD8sJBt/G1rVvhz5XT8= -github.com/libp2p/go-libp2p-crypto v0.1.0/go.mod h1:sPUokVISZiy+nNuTTH/TY+leRSxnFj/2GLjtOTW90hI= -github.com/libp2p/go-libp2p-discovery v0.2.0/go.mod h1:s4VGaxYMbw4+4+tsoQTqh7wfxg97AEdo4GYBt6BadWg= -github.com/libp2p/go-libp2p-discovery v0.3.0/go.mod h1:o03drFnz9BVAZdzC/QUQ+NeQOu38Fu7LJGEOK2gQltw= -github.com/libp2p/go-libp2p-discovery v0.5.0/go.mod h1:+srtPIU9gDaBNu//UHvcdliKBIcr4SfDcm0/PfPJLug= -github.com/libp2p/go-libp2p-kad-dht v0.21.0 h1:J0Yd22VA+sk0CJRGMgtfHvLVIkZDyJ3AJGiljywIw5U= -github.com/libp2p/go-libp2p-kad-dht v0.21.0/go.mod h1:Bhm9diAFmc6qcWAr084bHNL159srVZRKADdp96Qqd1I= +github.com/libp2p/go-libp2p-kad-dht v0.21.1 h1:xpfp8/t9+X2ip1l8Umap1/UGNnJ3RHJgKGAEsnRAlTo= +github.com/libp2p/go-libp2p-kad-dht v0.21.1/go.mod h1:Oy8wvbdjpB70eS5AaFaI68tOtrdo3KylTvXDjikxqFo= github.com/libp2p/go-libp2p-kbucket v0.5.0 h1:g/7tVm8ACHDxH29BGrpsQlnNeu+6OF1A9bno/4/U1oA= github.com/libp2p/go-libp2p-kbucket v0.5.0/go.mod h1:zGzGCpQd78b5BNTDGHNDLaTt9aDK/A02xeZp9QeFC4U= -github.com/libp2p/go-libp2p-loggables v0.1.0/go.mod h1:EyumB2Y6PrYjr55Q3/tiJ/o3xoDasoRYM7nOzEpoa90= -github.com/libp2p/go-libp2p-mplex v0.2.0/go.mod h1:Ejl9IyjvXJ0T9iqUTE1jpYATQ9NM3g+OtR+EMMODbKo= -github.com/libp2p/go-libp2p-mplex v0.2.1/go.mod h1:SC99Rxs8Vuzrf/6WhmH41kNn13TiYdAWNYHrwImKLnE= -github.com/libp2p/go-libp2p-mplex v0.2.2/go.mod h1:74S9eum0tVQdAfFiKxAyKzNdSuLqw5oadDq7+L/FELo= -github.com/libp2p/go-libp2p-mplex v0.2.3/go.mod h1:CK3p2+9qH9x+7ER/gWWDYJ3QW5ZxWDkm+dVvjfuG3ek= -github.com/libp2p/go-libp2p-mplex v0.4.0/go.mod h1:yCyWJE2sc6TBTnFpjvLuEJgTSw/u+MamvzILKdX7asw= -github.com/libp2p/go-libp2p-mplex v0.4.1/go.mod h1:cmy+3GfqfM1PceHTLL7zQzAAYaryDu6iPSC+CIb094g= -github.com/libp2p/go-libp2p-nat v0.0.5/go.mod h1:1qubaE5bTZMJE+E/uu2URroMbzdubFz1ChgiN79yKPE= -github.com/libp2p/go-libp2p-nat v0.0.6/go.mod h1:iV59LVhB3IkFvS6S6sauVTSOrNEANnINbI/fkaLimiw= -github.com/libp2p/go-libp2p-netutil v0.1.0/go.mod h1:3Qv/aDqtMLTUyQeundkKsA+YCThNdbQD54k3TqjpbFU= -github.com/libp2p/go-libp2p-noise v0.2.0/go.mod h1:IEbYhBBzGyvdLBoxxULL/SGbJARhUeqlO8lVSREYu2Q= -github.com/libp2p/go-libp2p-peer v0.2.0/go.mod h1:RCffaCvUyW2CJmG2gAWVqwePwW7JMgxjsHm7+J5kjWY= -github.com/libp2p/go-libp2p-peerstore v0.1.0/go.mod h1:2CeHkQsr8svp4fZ+Oi9ykN1HBb6u0MOvdJ7YIsmcwtY= -github.com/libp2p/go-libp2p-peerstore v0.1.3/go.mod h1:BJ9sHlm59/80oSkpWgr1MyY1ciXAXV397W6h1GH/uKI= -github.com/libp2p/go-libp2p-peerstore v0.2.0/go.mod h1:N2l3eVIeAitSg3Pi2ipSrJYnqhVnMNQZo9nkSCuAbnQ= -github.com/libp2p/go-libp2p-peerstore v0.2.1/go.mod h1:NQxhNjWxf1d4w6PihR8btWIRjwRLBr4TYKfNgrUkOPA= -github.com/libp2p/go-libp2p-peerstore v0.2.2/go.mod h1:NQxhNjWxf1d4w6PihR8btWIRjwRLBr4TYKfNgrUkOPA= -github.com/libp2p/go-libp2p-peerstore v0.2.6/go.mod h1:ss/TWTgHZTMpsU/oKVVPQCGuDHItOpf2W8RxAi50P2s= -github.com/libp2p/go-libp2p-peerstore v0.2.7/go.mod h1:ss/TWTgHZTMpsU/oKVVPQCGuDHItOpf2W8RxAi50P2s= -github.com/libp2p/go-libp2p-pnet v0.2.0/go.mod h1:Qqvq6JH/oMZGwqs3N1Fqhv8NVhrdYcO0BW4wssv21LA= -github.com/libp2p/go-libp2p-quic-transport v0.10.0/go.mod h1:RfJbZ8IqXIhxBRm5hqUEJqjiiY8xmEuq3HUDS993MkA= -github.com/libp2p/go-libp2p-record v0.1.0/go.mod h1:ujNc8iuE5dlKWVy6wuL6dd58t0n7xI4hAIl8pE6wu5Q= github.com/libp2p/go-libp2p-record v0.2.0 h1:oiNUOCWno2BFuxt3my4i1frNrt7PerzB3queqa1NkQ0= github.com/libp2p/go-libp2p-record v0.2.0/go.mod h1:I+3zMkvvg5m2OcSdoL0KPljyJyvNDFGKX7QdlpYUcwk= -github.com/libp2p/go-libp2p-secio v0.1.0/go.mod h1:tMJo2w7h3+wN4pgU2LSYeiKPrfqBgkOsdiKK77hE7c8= -github.com/libp2p/go-libp2p-secio v0.2.0/go.mod h1:2JdZepB8J5V9mBp79BmwsaPQhRPNN2NrnB2lKQcdy6g= -github.com/libp2p/go-libp2p-secio v0.2.1/go.mod h1:cWtZpILJqkqrSkiYcDBh5lA3wbT2Q+hz3rJQq3iftD8= -github.com/libp2p/go-libp2p-secio v0.2.2/go.mod h1:wP3bS+m5AUnFA+OFO7Er03uO1mncHG0uVwGrwvjYlNY= -github.com/libp2p/go-libp2p-swarm v0.1.0/go.mod h1:wQVsCdjsuZoc730CgOvh5ox6K8evllckjebkdiY5ta4= -github.com/libp2p/go-libp2p-swarm v0.2.2/go.mod h1:fvmtQ0T1nErXym1/aa1uJEyN7JzaTNyBcHImCxRpPKU= -github.com/libp2p/go-libp2p-swarm v0.2.3/go.mod h1:P2VO/EpxRyDxtChXz/VPVXyTnszHvokHKRhfkEgFKNM= -github.com/libp2p/go-libp2p-swarm v0.2.8/go.mod h1:JQKMGSth4SMqonruY0a8yjlPVIkb0mdNSwckW7OYziM= -github.com/libp2p/go-libp2p-swarm v0.3.0/go.mod h1:hdv95GWCTmzkgeJpP+GK/9D9puJegb7H57B5hWQR5Kk= -github.com/libp2p/go-libp2p-swarm v0.5.0/go.mod h1:sU9i6BoHE0Ve5SKz3y9WfKrh8dUat6JknzUehFx8xW4= -github.com/libp2p/go-libp2p-testing v0.0.2/go.mod h1:gvchhf3FQOtBdr+eFUABet5a4MBLK8jM3V4Zghvmi+E= -github.com/libp2p/go-libp2p-testing v0.0.3/go.mod h1:gvchhf3FQOtBdr+eFUABet5a4MBLK8jM3V4Zghvmi+E= -github.com/libp2p/go-libp2p-testing v0.0.4/go.mod h1:gvchhf3FQOtBdr+eFUABet5a4MBLK8jM3V4Zghvmi+E= -github.com/libp2p/go-libp2p-testing v0.1.0/go.mod h1:xaZWMJrPUM5GlDBxCeGUi7kI4eqnjVyavGroI2nxEM0= -github.com/libp2p/go-libp2p-testing v0.1.1/go.mod h1:xaZWMJrPUM5GlDBxCeGUi7kI4eqnjVyavGroI2nxEM0= -github.com/libp2p/go-libp2p-testing v0.1.2-0.20200422005655-8775583591d8/go.mod h1:Qy8sAncLKpwXtS2dSnDOP8ktexIAHKu+J+pnZOFZLTc= -github.com/libp2p/go-libp2p-testing v0.3.0/go.mod h1:efZkql4UZ7OVsEfaxNHZPzIehtsBXMrXnCfJIgDti5g= -github.com/libp2p/go-libp2p-testing v0.4.0/go.mod h1:Q+PFXYoiYFN5CAEG2w3gLPEzotlKsNSbKQ/lImlOWF0= github.com/libp2p/go-libp2p-testing v0.12.0 h1:EPvBb4kKMWO29qP4mZGyhVzUyR25dvfUIK5WDu6iPUA= github.com/libp2p/go-libp2p-testing v0.12.0/go.mod h1:KcGDRXyN7sQCllucn1cOOS+Dmm7ujhfEyXQL5lvkcPg= -github.com/libp2p/go-libp2p-tls v0.1.3/go.mod h1:wZfuewxOndz5RTnCAxFliGjvYSDA40sKitV4c50uI1M= -github.com/libp2p/go-libp2p-transport-upgrader v0.1.1/go.mod h1:IEtA6or8JUbsV07qPW4r01GnTenLW4oi3lOPbUMGJJA= -github.com/libp2p/go-libp2p-transport-upgrader v0.2.0/go.mod h1:mQcrHj4asu6ArfSoMuyojOdjx73Q47cYD7s5+gZOlns= -github.com/libp2p/go-libp2p-transport-upgrader v0.3.0/go.mod h1:i+SKzbRnvXdVbU3D1dwydnTmKRPXiAR/fyvi1dXuL4o= -github.com/libp2p/go-libp2p-transport-upgrader v0.4.2/go.mod h1:NR8ne1VwfreD5VIWIU62Agt/J18ekORFU/j1i2y8zvk= -github.com/libp2p/go-libp2p-yamux v0.2.0/go.mod h1:Db2gU+XfLpm6E4rG5uGCFX6uXA8MEXOxFcRoXUODaK8= -github.com/libp2p/go-libp2p-yamux v0.2.2/go.mod h1:lIohaR0pT6mOt0AZ0L2dFze9hds9Req3OfS+B+dv4qw= -github.com/libp2p/go-libp2p-yamux v0.2.5/go.mod h1:Zpgj6arbyQrmZ3wxSZxfBmbdnWtbZ48OpsfmQVTErwA= -github.com/libp2p/go-libp2p-yamux v0.2.7/go.mod h1:X28ENrBMU/nm4I3Nx4sZ4dgjZ6VhLEn0XhIoZ5viCwU= -github.com/libp2p/go-libp2p-yamux v0.2.8/go.mod h1:/t6tDqeuZf0INZMTgd0WxIRbtK2EzI2h7HbFm9eAKI4= -github.com/libp2p/go-libp2p-yamux v0.4.0/go.mod h1:+DWDjtFMzoAwYLVkNZftoucn7PelNoy5nm3tZ3/Zw30= -github.com/libp2p/go-libp2p-yamux v0.5.0/go.mod h1:AyR8k5EzyM2QN9Bbdg6X1SkVVuqLwTGf0L4DFq9g6po= -github.com/libp2p/go-libp2p-yamux v0.5.4/go.mod h1:tfrXbyaTqqSU654GTvK3ocnSZL3BuHoeTSqhcel1wsE= -github.com/libp2p/go-maddr-filter v0.0.4/go.mod h1:6eT12kSQMA9x2pvFQa+xesMKUBlj9VImZbj3B9FBH/Q= -github.com/libp2p/go-maddr-filter v0.0.5/go.mod h1:Jk+36PMfIqCJhAnaASRH83bdAvfDRp/w6ENFaC9bG+M= -github.com/libp2p/go-maddr-filter v0.1.0/go.mod h1:VzZhTXkMucEGGEOSKddrwGiOv0tUhgnKqNEmIAz/bPU= -github.com/libp2p/go-mplex v0.0.3/go.mod h1:pK5yMLmOoBR1pNCqDlA2GQrdAVTMkqFalaTWe7l4Yd0= -github.com/libp2p/go-mplex v0.1.0/go.mod h1:SXgmdki2kwCUlCCbfGLEgHjC4pFqhTp0ZoV6aiKgxDU= -github.com/libp2p/go-mplex v0.1.1/go.mod h1:Xgz2RDCi3co0LeZfgjm4OgUF15+sVR8SRcu3SFXI1lk= -github.com/libp2p/go-mplex v0.1.2/go.mod h1:Xgz2RDCi3co0LeZfgjm4OgUF15+sVR8SRcu3SFXI1lk= -github.com/libp2p/go-mplex v0.2.0/go.mod h1:0Oy/A9PQlwBytDRp4wSkFnzHYDKcpLot35JQ6msjvYQ= -github.com/libp2p/go-mplex v0.3.0/go.mod h1:0Oy/A9PQlwBytDRp4wSkFnzHYDKcpLot35JQ6msjvYQ= -github.com/libp2p/go-msgio v0.0.2/go.mod h1:63lBBgOTDKQL6EWazRMCwXsEeEeK9O2Cd+0+6OOuipQ= -github.com/libp2p/go-msgio v0.0.4/go.mod h1:63lBBgOTDKQL6EWazRMCwXsEeEeK9O2Cd+0+6OOuipQ= -github.com/libp2p/go-msgio v0.0.6/go.mod h1:4ecVB6d9f4BDSL5fqvPiC4A3KivjWn+Venn/1ALLMWA= github.com/libp2p/go-msgio v0.3.0 h1:mf3Z8B1xcFN314sWX+2vOTShIE0Mmn2TXn3YCUQGNj0= github.com/libp2p/go-msgio v0.3.0/go.mod h1:nyRM819GmVaF9LX3l03RMh10QdOroF++NBbxAb0mmDM= -github.com/libp2p/go-nat v0.0.4/go.mod h1:Nmw50VAvKuk38jUBcmNh6p9lUJLoODbJRvYAa/+KSDo= -github.com/libp2p/go-nat v0.0.5/go.mod h1:B7NxsVNPZmRLvMOwiEO1scOSyjA56zxYAGv1yQgRkEU= github.com/libp2p/go-nat v0.1.0 h1:MfVsH6DLcpa04Xr+p8hmVRG4juse0s3J8HyNWYHffXg= github.com/libp2p/go-nat v0.1.0/go.mod h1:X7teVkwRHNInVNWQiO/tAiAVRwSr5zoRz4YSTC3uRBM= github.com/libp2p/go-netroute v0.1.2/go.mod h1:jZLDV+1PE8y5XxBySEBgbuVAXbhtuHSdmLPL2n9MKbk= -github.com/libp2p/go-netroute v0.1.3/go.mod h1:jZLDV+1PE8y5XxBySEBgbuVAXbhtuHSdmLPL2n9MKbk= -github.com/libp2p/go-netroute v0.1.5/go.mod h1:V1SR3AaECRkEQCoFFzYwVYWvYIEtlxx89+O3qcpCl4A= -github.com/libp2p/go-netroute v0.1.6/go.mod h1:AqhkMh0VuWmfgtxKPp3Oc1LdU5QSWS7wl0QLhSZqXxQ= github.com/libp2p/go-netroute v0.2.1 h1:V8kVrpD8GK0Riv15/7VN6RbUQ3URNZVosw7H2v9tksU= github.com/libp2p/go-netroute v0.2.1/go.mod h1:hraioZr0fhBjG0ZRXJJ6Zj2IVEVNx6tDTFQfSmcq7mQ= -github.com/libp2p/go-openssl v0.0.2/go.mod h1:v8Zw2ijCSWBQi8Pq5GAixw6DbFfa9u6VIYDXnvOXkc0= -github.com/libp2p/go-openssl v0.0.3/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO1HmaZjggc= -github.com/libp2p/go-openssl v0.0.4/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO1HmaZjggc= -github.com/libp2p/go-openssl v0.0.5/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO1HmaZjggc= -github.com/libp2p/go-openssl v0.0.7/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO1HmaZjggc= -github.com/libp2p/go-reuseport v0.0.1/go.mod h1:jn6RmB1ufnQwl0Q1f+YxAj8isJgDCQzaaxIFYDhcYEA= -github.com/libp2p/go-reuseport v0.0.2/go.mod h1:SPD+5RwGC7rcnzngoYC86GjPzjSywuQyMVAheVBD9nQ= github.com/libp2p/go-reuseport v0.2.0 h1:18PRvIMlpY6ZK85nIAicSBuXXvrYoSw3dsBAR7zc560= -github.com/libp2p/go-reuseport-transport v0.0.2/go.mod h1:YkbSDrvjUVDL6b8XqriyA20obEtsW9BLkuOUyQAOCbs= -github.com/libp2p/go-reuseport-transport v0.0.3/go.mod h1:Spv+MPft1exxARzP2Sruj2Wb5JSyHNncjf1Oi2dEbzM= -github.com/libp2p/go-reuseport-transport v0.0.4/go.mod h1:trPa7r/7TJK/d+0hdBLOCGvpQQVOU74OXbNCIMkufGw= +github.com/libp2p/go-reuseport v0.2.0/go.mod h1:bvVho6eLMm6Bz5hmU0LYN3ixd3nPPvtIlaURZZgOY4k= github.com/libp2p/go-sockaddr v0.0.2/go.mod h1:syPvOmNs24S3dFVGJA1/mrqdeijPxLV2Le3BRLKd68k= -github.com/libp2p/go-sockaddr v0.1.0/go.mod h1:syPvOmNs24S3dFVGJA1/mrqdeijPxLV2Le3BRLKd68k= -github.com/libp2p/go-sockaddr v0.1.1/go.mod h1:syPvOmNs24S3dFVGJA1/mrqdeijPxLV2Le3BRLKd68k= -github.com/libp2p/go-stream-muxer v0.0.1/go.mod h1:bAo8x7YkSpadMTbtTaxGVHWUQsR/l5MEaHbKaliuT14= -github.com/libp2p/go-stream-muxer-multistream v0.2.0/go.mod h1:j9eyPol/LLRqT+GPLSxvimPhNph4sfYfMoDPd7HkzIc= -github.com/libp2p/go-stream-muxer-multistream v0.3.0/go.mod h1:yDh8abSIzmZtqtOt64gFJUXEryejzNb0lisTt+fAMJA= -github.com/libp2p/go-tcp-transport v0.1.0/go.mod h1:oJ8I5VXryj493DEJ7OsBieu8fcg2nHGctwtInJVpipc= -github.com/libp2p/go-tcp-transport v0.1.1/go.mod h1:3HzGvLbx6etZjnFlERyakbaYPdfjg2pWP97dFZworkY= -github.com/libp2p/go-tcp-transport v0.2.0/go.mod h1:vX2U0CnWimU4h0SGSEsg++AzvBcroCGYw28kh94oLe0= -github.com/libp2p/go-tcp-transport v0.2.3/go.mod h1:9dvr03yqrPyYGIEN6Dy5UvdJZjyPFvl1S/igQ5QD1SU= -github.com/libp2p/go-ws-transport v0.2.0/go.mod h1:9BHJz/4Q5A9ludYWKoGCFC5gUElzlHoKzu0yY9p/klM= -github.com/libp2p/go-ws-transport v0.3.0/go.mod h1:bpgTJmRZAvVHrgHybCVyqoBmyLQ1fiZuEaBYusP5zsk= -github.com/libp2p/go-ws-transport v0.4.0/go.mod h1:EcIEKqf/7GDjth6ksuS/6p7R49V4CBY6/E7R/iyhYUA= -github.com/libp2p/go-yamux v1.2.2/go.mod h1:FGTiPvoV/3DVdgWpX+tM0OW3tsM+W5bSE3gZwqQTcow= -github.com/libp2p/go-yamux v1.3.0/go.mod h1:FGTiPvoV/3DVdgWpX+tM0OW3tsM+W5bSE3gZwqQTcow= -github.com/libp2p/go-yamux v1.3.3/go.mod h1:FGTiPvoV/3DVdgWpX+tM0OW3tsM+W5bSE3gZwqQTcow= -github.com/libp2p/go-yamux v1.3.5/go.mod h1:FGTiPvoV/3DVdgWpX+tM0OW3tsM+W5bSE3gZwqQTcow= -github.com/libp2p/go-yamux v1.3.7/go.mod h1:fr7aVgmdNGJK+N1g+b6DW6VxzbRCjCOejR/hkmpooHE= -github.com/libp2p/go-yamux v1.4.0/go.mod h1:fr7aVgmdNGJK+N1g+b6DW6VxzbRCjCOejR/hkmpooHE= -github.com/libp2p/go-yamux v1.4.1 h1:P1Fe9vF4th5JOxxgQvfbOHkrGqIZniTLf+ddhZp8YTI= -github.com/libp2p/go-yamux v1.4.1/go.mod h1:fr7aVgmdNGJK+N1g+b6DW6VxzbRCjCOejR/hkmpooHE= -github.com/libp2p/go-yamux/v2 v2.2.0/go.mod h1:3So6P6TV6r75R9jiBpiIKgU/66lOarCZjqROGxzPpPQ= github.com/libp2p/go-yamux/v4 v4.0.0 h1:+Y80dV2Yx/kv7Y7JKu0LECyVdMXm1VUoko+VQ9rBfZQ= -github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= -github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= -github.com/lucas-clemente/quic-go v0.19.3/go.mod h1:ADXpNbTQjq1hIzCpB+y/k5iz4n4z4IwqoLb94Kh5Hu8= +github.com/libp2p/go-yamux/v4 v4.0.0/go.mod h1:NWjl8ZTLOGlozrXSOZ/HlfG++39iKNnM5wwmtQP1YB4= github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI= -github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/marten-seemann/qpack v0.2.1/go.mod h1:F7Gl5L1jIgN1D11ucXefiuJS9UMVP2opoCp2jDKb7wc= -github.com/marten-seemann/qtls v0.10.0/go.mod h1:UvMd1oaYDACI99/oZUYLzMCkBXQVT0aGm99sJhbT8hs= -github.com/marten-seemann/qtls-go1-15 v0.1.1/go.mod h1:GyFwywLKkRt+6mfU99csTEY1joMZz5vmB1WNZH3P81I= github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd h1:br0buuQ854V8u83wA0rVZ8ttrq5CpaPZdvrK0LP2lOk= github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd/go.mod h1:QuCEs1Nt24+FYQEqAAncTDPJIuGs+LxK1MCiFL25pMU= -github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= -github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.13/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= -github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4= -github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= -github.com/miekg/dns v1.1.12/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= -github.com/miekg/dns v1.1.28/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= github.com/miekg/dns v1.1.50 h1:DQUfb9uc6smULcREF09Uc+/Gd46YWqJd5DbpPE9xkcA= github.com/miekg/dns v1.1.50/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= +github.com/mikioh/tcp v0.0.0-20190314235350-803a9b46060c h1:bzE/A84HN25pxAuk9Eej1Kz9OUelF97nAc82bDquQI8= github.com/mikioh/tcp v0.0.0-20190314235350-803a9b46060c/go.mod h1:0SQS9kMwD2VsyFEB++InYyBJroV/FRmBgcydeSUcJms= github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b h1:z78hV3sbSMAUoyUMM0I83AUIT6Hu17AWfgjzIbtrYFc= github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b/go.mod h1:lxPUiZwKoFL8DUUmalo2yJJUCxbPKtm8OKfqr2/FTNU= @@ -756,27 +487,19 @@ github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc h1:PTfri+PuQmWDqERdn github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc/go.mod h1:cGKTAVKx4SxOuR/czcZ/E2RSJ3sfHs8FpHhQ5CWMf9s= github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ= github.com/minio/sha256-simd v0.0.0-20190131020904-2d45a736cd16/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= -github.com/minio/sha256-simd v0.0.0-20190328051042-05b4dd3047e5/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= -github.com/minio/sha256-simd v0.1.0/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= -github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= -github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= -github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= -github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= -github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= -github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= -github.com/mr-tron/base58 v1.1.1/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= github.com/mr-tron/base58 v1.1.2/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= @@ -787,35 +510,14 @@ github.com/multiformats/go-base32 v0.1.0/go.mod h1:Kj3tFY6zNr+ABYMqeUNeGvkIC/UYg github.com/multiformats/go-base36 v0.1.0/go.mod h1:kFGE83c6s80PklsHO9sRn2NCoffoRdUUOENyW/Vv6sM= github.com/multiformats/go-base36 v0.2.0 h1:lFsAbNOGeKtuKozrtBsAkSVhv1p9D0/qedU9rQyccr0= github.com/multiformats/go-base36 v0.2.0/go.mod h1:qvnKE++v+2MWCfePClUEjE78Z7P2a1UV0xHgWc0hkp4= -github.com/multiformats/go-multiaddr v0.0.1/go.mod h1:xKVEak1K9cS1VdmPZW3LSIb6lgmoS58qz/pzqmAxV44= -github.com/multiformats/go-multiaddr v0.0.2/go.mod h1:xKVEak1K9cS1VdmPZW3LSIb6lgmoS58qz/pzqmAxV44= -github.com/multiformats/go-multiaddr v0.0.4/go.mod h1:xKVEak1K9cS1VdmPZW3LSIb6lgmoS58qz/pzqmAxV44= -github.com/multiformats/go-multiaddr v0.1.0/go.mod h1:xKVEak1K9cS1VdmPZW3LSIb6lgmoS58qz/pzqmAxV44= github.com/multiformats/go-multiaddr v0.1.1/go.mod h1:aMKBKNEYmzmDmxfX88/vz+J5IU55txyt0p4aiWVohjo= github.com/multiformats/go-multiaddr v0.2.0/go.mod h1:0nO36NvPpyV4QzvTLi/lafl2y95ncPj0vFwVF6k6wJ4= -github.com/multiformats/go-multiaddr v0.2.1/go.mod h1:s/Apk6IyxfvMjDafnhJgJ3/46z7tZ04iMk5wP4QMGGE= -github.com/multiformats/go-multiaddr v0.2.2/go.mod h1:NtfXiOtHvghW9KojvtySjH5y0u0xW5UouOmQQrn6a3Y= -github.com/multiformats/go-multiaddr v0.3.0/go.mod h1:dF9kph9wfJ+3VLAaeBqo9Of8x4fJxp6ggJGteB8HQTI= -github.com/multiformats/go-multiaddr v0.3.1/go.mod h1:uPbspcUPd5AfaP6ql3ujFY+QWzmBD8uLLL4bXW0XfGc= -github.com/multiformats/go-multiaddr v0.3.3/go.mod h1:lCKNGP1EQ1eZ35Za2wlqnabm9xQkib3fyB+nZXHLag0= github.com/multiformats/go-multiaddr v0.8.0 h1:aqjksEcqK+iD/Foe1RRFsGZh8+XFiGo7FgUCZlpv3LU= github.com/multiformats/go-multiaddr v0.8.0/go.mod h1:Fs50eBDWvZu+l3/9S6xAE7ZYj6yhxlvaVZjakWN7xRs= -github.com/multiformats/go-multiaddr-dns v0.0.1/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q= -github.com/multiformats/go-multiaddr-dns v0.0.2/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q= -github.com/multiformats/go-multiaddr-dns v0.2.0/go.mod h1:TJ5pr5bBO7Y1B18djPuRsVkduhQH2YqYSbxWJzYGdK0= github.com/multiformats/go-multiaddr-dns v0.3.1 h1:QgQgR+LQVt3NPTjbrLLpsaT2ufAA2y0Mkk+QRVJbW3A= github.com/multiformats/go-multiaddr-dns v0.3.1/go.mod h1:G/245BRQ6FJGmryJCrOuTdB37AMA5AMOVuO6NY3JwTk= -github.com/multiformats/go-multiaddr-fmt v0.0.1/go.mod h1:aBYjqL4T/7j4Qx+R73XSv/8JsgnRFlf0w2KGLCmXl3Q= github.com/multiformats/go-multiaddr-fmt v0.1.0 h1:WLEFClPycPkp4fnIzoFoV9FVd49/eQsuaL3/CWe167E= github.com/multiformats/go-multiaddr-fmt v0.1.0/go.mod h1:hGtDIW4PU4BqJ50gW2quDuPVjyWNZxToGUh/HwTZYJo= -github.com/multiformats/go-multiaddr-net v0.0.1/go.mod h1:nw6HSxNmCIQH27XPGBuX+d1tnvM7ihcFwHMSstNAVUU= -github.com/multiformats/go-multiaddr-net v0.1.0/go.mod h1:5JNbcfBOP4dnhoZOv10JJVkJO0pCCEf8mTnipAo2UZQ= -github.com/multiformats/go-multiaddr-net v0.1.1/go.mod h1:5JNbcfBOP4dnhoZOv10JJVkJO0pCCEf8mTnipAo2UZQ= -github.com/multiformats/go-multiaddr-net v0.1.2/go.mod h1:QsWt3XK/3hwvNxZJp92iMQKME1qHfpYmyIjFVsSOY6Y= -github.com/multiformats/go-multiaddr-net v0.1.3/go.mod h1:ilNnaM9HbmVFqsb/qcNysjCu4PVONlrBZpHIrw/qQuA= -github.com/multiformats/go-multiaddr-net v0.1.4/go.mod h1:ilNnaM9HbmVFqsb/qcNysjCu4PVONlrBZpHIrw/qQuA= -github.com/multiformats/go-multiaddr-net v0.1.5/go.mod h1:ilNnaM9HbmVFqsb/qcNysjCu4PVONlrBZpHIrw/qQuA= -github.com/multiformats/go-multiaddr-net v0.2.0/go.mod h1:gGdH3UXny6U3cKKYCvpXI5rnK7YaOIEOPVDI9tsJbEA= github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs= github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPwIdYQD509ZjSb5y9Oc= github.com/multiformats/go-multibase v0.1.1 h1:3ASCDsuLX8+j4kx58qnJ4YFq/JWTJpCyDW27ztsVTOI= @@ -823,7 +525,6 @@ github.com/multiformats/go-multibase v0.1.1/go.mod h1:ZEjHE+IsUrgp5mhlEAYjMtZwK1 github.com/multiformats/go-multicodec v0.7.0 h1:rTUjGOwjlhGHbEMbPoSUJowG1spZTVsITRANCjKTUAQ= github.com/multiformats/go-multicodec v0.7.0/go.mod h1:GUC8upxSBE4oG+q3kWZRw/+6yC1BqO550bjhWsJbZlw= github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U= -github.com/multiformats/go-multihash v0.0.5/go.mod h1:lt/HCbqlQwlPBz7lv0sQCdtfcMtlJvakRUn/0Ual8po= github.com/multiformats/go-multihash v0.0.8/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= github.com/multiformats/go-multihash v0.0.10/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc= @@ -831,140 +532,99 @@ github.com/multiformats/go-multihash v0.0.14/go.mod h1:VdAWLKTwram9oKAatUcLxBNUj github.com/multiformats/go-multihash v0.0.15/go.mod h1:D6aZrWNLFTV/ynMpKsNtB40mJzmCl4jb1alC0OvHiHg= github.com/multiformats/go-multihash v0.2.1 h1:aem8ZT0VA2nCHHk7bPJ1BjUbHNciqZC/d16Vve9l108= github.com/multiformats/go-multihash v0.2.1/go.mod h1:WxoMcYG85AZVQUyRyo9s4wULvW5qrI9vb2Lt6evduFc= -github.com/multiformats/go-multistream v0.1.0/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg= -github.com/multiformats/go-multistream v0.1.1/go.mod h1:KmHZ40hzVxiaiwlj3MEbYgK9JFk2/9UktWZAF54Du38= -github.com/multiformats/go-multistream v0.2.1/go.mod h1:5GZPQZbkWOLOn3J2y4Y99vVW7vOfsAflxARk3x14o6k= -github.com/multiformats/go-multistream v0.2.2/go.mod h1:UIcnm7Zuo8HKG+HkWgfQsGL+/MIEhyTqbODbIUwSXKs= github.com/multiformats/go-multistream v0.4.1 h1:rFy0Iiyn3YT0asivDUIR05leAdwZq3de4741sbiSdfo= github.com/multiformats/go-multistream v0.4.1/go.mod h1:Mz5eykRVAjJWckE2U78c6xqdtyNUEhKSM0Lwar2p77Q= github.com/multiformats/go-varint v0.0.1/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= -github.com/multiformats/go-varint v0.0.2/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/multiformats/go-varint v0.0.6/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/nEGOHFS8= github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOELpZAu9eioSos/OU= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= -github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU= -github.com/nats-io/nats-server/v2 v2.1.2/go.mod h1:Afk+wRZqkMQs/p45uXdrVLuab3gwv3Z8C4HTBu8GD/k= -github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w= -github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= -github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= -github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo= github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= -github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= -github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= -github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo/v2 v2.5.1 h1:auzK7OI497k6x4OvWq+TKAcpcSAlod0doAH72oIN0Jw= -github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= +github.com/onsi/ginkgo/v2 v2.5.1/go.mod h1:63DOGlLAH8+REH8jUGdL3YpCpu7JODesutUjdENfUAc= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= -github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= +github.com/onsi/gomega v1.24.0 h1:+0glovB9Jd6z3VR+ScSwQqXVTIfJcGA9UBM8yzQxhqg= github.com/opencontainers/runtime-spec v1.0.2 h1:UfAcuLBJB9Coz72x1hgl8O5RVzTdNiaglX6v2DM6FI0= -github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= -github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= +github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= -github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= -github.com/openzipkin-contrib/zipkin-go-opentracing v0.4.5/go.mod h1:/wsWhb9smxSfWAKL3wpBW7V8scJMt8N8gnaMCS9E/cA= github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8= -github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= -github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= -github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= -github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM= -github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0= -github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= +github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 h1:1/WtZae0yGtPq+TI6+Tv1WTxkukpXeMlviSxvL7SRgk= github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9/go.mod h1:x3N5drFsm2uilKKuuYo6LdyD8vZAW55sH/9w+pbo1sw= -github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= -github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/polydawn/refmt v0.0.0-20190221155625-df39d6c2d992/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= github.com/polydawn/refmt v0.0.0-20190807091052-3d65705ee9f1/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= -github.com/polydawn/refmt v0.0.0-20201211092308-30ac6d18308e/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= github.com/polydawn/refmt v0.89.0 h1:ADJTApkvkeBZsN0tBTx8QjpD9JkmxbKp0cxfr9qszm4= github.com/polydawn/refmt v0.89.0/go.mod h1:/zvteZs/GwLtCgZ4BL6CBsk9IKIlexP43ObX9AxTqTw= -github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= -github.com/prometheus/client_golang v1.10.0/go.mod h1:WJM3cc3yu7XKBKa/I8WeZm+V3eltZnBwfENSU7mdogU= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= github.com/prometheus/client_golang v1.14.0 h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw= github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= -github.com/prometheus/common v0.18.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE= github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= github.com/quic-go/qpack v0.4.0 h1:Cr9BXA1sQS2SmDUWjSofMPNKmvF6IiIfDRmgU0w1ZCo= -github.com/quic-go/qtls-go1-18 v0.2.0 h1:5ViXqBZ90wpUcZS0ge79rf029yx0dYB0McyPJwqqj7U= -github.com/quic-go/qtls-go1-19 v0.2.0 h1:Cvn2WdhyViFUHoOqK52i51k4nDX8EwIh5VJiVM4nttk= -github.com/quic-go/qtls-go1-20 v0.1.0 h1:d1PK3ErFy9t7zxKsG3NXBJXZjp/kMLoIb3y/kV54oAI= -github.com/quic-go/quic-go v0.32.0 h1:lY02md31s1JgPiiyfqJijpu/UX/Iun304FI3yUqX7tA= -github.com/quic-go/webtransport-go v0.5.1 h1:1eVb7WDWCRoaeTtFHpFBJ6WDN1bSrPrRoW6tZgSw0Ow= +github.com/quic-go/qpack v0.4.0/go.mod h1:UZVnYIfi5GRk+zI9UMaCPsmZ2xKJP7XBUvVyT1Knj9A= +github.com/quic-go/qtls-go1-19 v0.2.1 h1:aJcKNMkH5ASEJB9FXNeZCyTEIHU1J7MmHyz1Q1TSG1A= +github.com/quic-go/qtls-go1-19 v0.2.1/go.mod h1:ySOI96ew8lnoKPtSqx2BlI5wCpUVPT05RMAlajtnyOI= +github.com/quic-go/qtls-go1-20 v0.1.1 h1:KbChDlg82d3IHqaj2bn6GfKRj84Per2VGf5XV3wSwQk= +github.com/quic-go/qtls-go1-20 v0.1.1/go.mod h1:JKtK6mjbAVcUTN/9jZpvLbGxvdWIKS8uT7EiStoU1SM= +github.com/quic-go/quic-go v0.33.0 h1:ItNoTDN/Fm/zBlq769lLJc8ECe9gYaW40veHCCco7y0= +github.com/quic-go/quic-go v0.33.0/go.mod h1:YMuhaAV9/jIu0XclDXwZPAsP/2Kgr5yMYhe9oxhhOFA= +github.com/quic-go/webtransport-go v0.5.2 h1:GA6Bl6oZY+g/flt00Pnu0XtivSD8vukOu3lYhJjnGEk= +github.com/quic-go/webtransport-go v0.5.2/go.mod h1:OhmmgJIzTTqXK5xvtuX0oBpLV2GkLWNDA+UeTGJXErU= github.com/raulk/go-watchdog v1.3.0 h1:oUmdlHxdkXRJlwfG0O9omj8ukerm8MEQavSiDTEtBsk= -github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= -github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/raulk/go-watchdog v1.3.0/go.mod h1:fIvOnLbF0b0ZwkB9YU4mOW9Did//4vPZtDqv66NfsMU= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/samber/lo v1.36.0 h1:4LaOxH1mHnbDGhTVE0i1z8v/lWaQW8AIfOD3HU4mSaw= github.com/samber/lo v1.36.0/go.mod h1:HLeWcJRRyLKp3+/XBJvOrerCQn9mhdKMHyd7IRlgeQ8= -github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= -github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shurcooL/component v0.0.0-20170202220835-f88ec8f54cc4/go.mod h1:XhFIlyj5a1fBNx5aJTbKoIq0mNaPvOagO+HjB3EtxrY= github.com/shurcooL/events v0.0.0-20181021180414-410e4ca65f48/go.mod h1:5u70Mqkb5O5cxEA8nxTsgrgLehJeAw6Oc4Ab1c/P1HM= @@ -992,6 +652,7 @@ github.com/shurcooL/webdavfs v0.0.0-20170829043945-18c3829fa133/go.mod h1:hKmq5k github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.2.0 h1:42S6lae5dvLc7BrLu/0ugRtcFVjoJNMC/N3yZFZkDFs= github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= @@ -999,28 +660,17 @@ github.com/smartystreets/goconvey v0.0.0-20190222223459-a17d461953aa/go.mod h1:2 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smartystreets/goconvey v1.7.2 h1:9RBaZCeXEQ3UselpuwUQHltGVXvdwm6cv1hgR6gDIPg= github.com/smartystreets/goconvey v1.7.2/go.mod h1:Vw0tHAZW6lzCRk3xgdin6fKYcG+G3Pg9vgXWeJpQFMM= -github.com/smola/gocompat v0.2.0/go.mod h1:1B0MlxbmoZNo3h8guHp8HztB3BSYR5itql9qtVc0ypY= -github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= -github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:UdhH50NIW0fCiwBSr0co2m7BnFLdv4fQTgdqdJTHFeE= github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod h1:HuIsMU8RRBOtsCgI77wP899iHVBQpCmg4ErYMZB+2IA= -github.com/spacemonkeygo/openssl v0.0.0-20181017203307-c2dcc5cca94a/go.mod h1:7AyxJNCJ7SBZ1MfVQCWD6Uqo2oubI2Eq2y2eqf+A5r0= -github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572/go.mod h1:w0SWMsp6j9O/dk4/ZpIhL+3CkG8ofA2vuv7k+ltqUMc= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= -github.com/src-d/envconfig v1.0.0/go.mod h1:Q9YQZ7BKITldTBnoxsE5gOeB5y66RyPXeue/R4aaNBc= -github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= -github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= -github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= @@ -1030,22 +680,25 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= +github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= +github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= github.com/thoas/go-funk v0.9.1 h1:O549iLZqPpTUQ10ykd26sZhzD+rmR5pWhuElrhbC20M= github.com/tj/assert v0.0.3 h1:Df/BlaZ20mq6kuai7f5z2TvPFiwC3xaWJSDQNiIS3Rk= -github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/ucarion/urlpath v0.0.0-20200424170820-7ccc79b76bbb h1:Ywfo8sUltxogBpFuMOFRrrSifO788kAFxmvVw31PtQQ= github.com/ucarion/urlpath v0.0.0-20200424170820-7ccc79b76bbb/go.mod h1:ikPs9bRWicNw3S7XpJ8sK/smGwU9WcSVU3dy9qahYBM= +github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo= +github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= -github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= -github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs= +github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= +github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli v1.22.10/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU= github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM= @@ -1066,88 +719,58 @@ github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f/go.mod h1:p9 github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 h1:EKhdznlJHPMoKr0XTrX+IlJs1LH3lyx2nfr1dOlZ79k= github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1/go.mod h1:8UvriyWtv5Q5EOgjHaSseUEdkQfvwFv1I/In/O2M9gc= github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM= -github.com/whyrusleeping/go-logging v0.0.1/go.mod h1:lDPYj54zutzG1XYfHAhcc7oNXEburHQBn+Iqd4yS4vE= -github.com/whyrusleeping/mafmt v1.2.8/go.mod h1:faQJFPbLSxzD9xpA02ttW/tS9vZykNvXwGvqIpk20FA= -github.com/whyrusleeping/mdns v0.0.0-20190826153040-b9b60ed33aa9/go.mod h1:j4l84WPFclQPj320J9gp0XwNKBb3U0zt5CBqjPp22G4= -github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7/go.mod h1:X2c0RVCI1eSUFI8eLcY3c0423ykwiUdxLJtkDvruhjI= -github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE= -github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA= -go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= -go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.opencensus.io v0.22.1/go.mod h1:Ap50jQcDJrx6rB6VgeeFPtuPIf3wMRvRfrfYDO6+BmA= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/otel v1.7.0 h1:Z2lA3Tdch0iDcrhJXDIlC94XE+bxok1F9B+4Lz/lGsM= go.opentelemetry.io/otel v1.7.0/go.mod h1:5BdUoMIz5WEs0vt0CUEMtSSaTSHBBVwrhnz7+nrD5xk= go.opentelemetry.io/otel/trace v1.7.0 h1:O37Iogk1lEkMRXewVtZ1BBTVn5JEp8GrJvP92bJqC6o= go.opentelemetry.io/otel/trace v1.7.0/go.mod h1:fzLSB9nqR2eXzxPXb2JW9IKE+ScyXA48yyE4TNvoHqU= -go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/dig v1.15.0 h1:vq3YWr8zRj1eFGC7Gvf907hE0eRjPTZ1d3xHadD6liE= +go.uber.org/dig v1.15.0/go.mod h1:pKHs0wMynzL6brANhB2hLMro+zalv1osARTviTcqHLM= go.uber.org/fx v1.18.2 h1:bUNI6oShr+OVFQeU8cDNbnN7VFsu+SsjHzUF51V/GAU= -go.uber.org/goleak v1.0.0/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= +go.uber.org/fx v1.18.2/go.mod h1:g0V1KMQ66zIRk8bLu3Ea5Jt2w/cHlOIp4wdRsgh0JaY= go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= -go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= -go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= -go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= go.uber.org/zap v1.14.1/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= -go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d/go.mod h1:OWs+y06UdEOHN4y+MfF/py+xQ/tYqIWW03b70/CG9Rw= -golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190225124518-7f87c0fbb88b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190618222545-ea8f1a30c443/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200602180216-279210d13fed/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.5.0 h1:U/0M97KRkSFvyD/3FSmdP5W5swImpNgle/EHFhOsQPE= @@ -1188,19 +811,14 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.7.0 h1:LapD9S96VoQRhi/GrNTqeBJFrUjs5UHCAtTlgwA5oZA= golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181029044818-c44066c5c816/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181106065722-10aee1819953/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190227160552-c95aed5357e7/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -1213,8 +831,6 @@ golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -1229,21 +845,20 @@ golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20210423184538-5f58ad60dda6/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.5.0 h1:GyT4nK/YDHSqa1c4753ouYCDajOYKTja9Xb/OHtgvSw= -golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= +golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1252,6 +867,7 @@ golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4Iltr golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b h1:clP8eMhB30EHdc0bd2Twtq6kgU7yl5ub2cQLSdrv1Dg= golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1267,21 +883,17 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180810173357-98c5dad5d1a0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181029174526-d69651ed3497/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190219092855-153ac476189d/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190228124157-a34e9553db1e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190302025703-b6889370fb10/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190316082340-a2f829d7f35f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190405154228-4b34438f7a67/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1289,25 +901,22 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190526052359-791d8a0f4d09/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1323,16 +932,14 @@ golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210317225723-c4fcb01b228e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210426080607-c94f62235c83/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1342,8 +949,9 @@ golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.4.0 h1:Zr2JFtRQNX3BCZ8YtxRE9hNJYC8J6I1MVbMg6owUp18= -golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -1354,7 +962,8 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.6.0 h1:3XmdazWV+ubf7QgHSTWeykHOci5oeekaGJBLkrkaw4k= +golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1364,7 +973,6 @@ golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030000716-a0a13e073c7b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181130052023-1c3d964395ce/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= @@ -1382,16 +990,13 @@ golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= @@ -1412,6 +1017,7 @@ golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.3.0 h1:SrNbZl6ECOS1qFzgTdQfWXZM9XBkiA6tkFrH9YSTPHM= @@ -1425,7 +1031,6 @@ golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNq google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= google.golang.org/api v0.0.0-20181030000543-1d582fd0359e/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= google.golang.org/api v0.1.0/go.mod h1:UGEZY7KEX120AnNLIHFMKIo4obdJhkp2tPbaPlQx13Y= -google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -1449,6 +1054,7 @@ google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6 h1:lMO5rYAqUxkmaj76jAkRUvt5JZgFymx/+Q5Mzfivuhc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= @@ -1459,7 +1065,6 @@ google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRn google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= @@ -1488,23 +1093,17 @@ google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmE google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.22.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= -google.golang.org/grpc v1.28.1/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= @@ -1518,7 +1117,6 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= @@ -1527,17 +1125,11 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= -gopkg.in/src-d/go-cli.v0 v0.0.0-20181105080154-d492247bbc0d/go.mod h1:z+K8VcOYVYcSwSjGebuDL6176A1XskgbtNl64NSg+n8= -gopkg.in/src-d/go-log.v1 v1.0.1/go.mod h1:GN34hKP0g305ysm2/hctJ0Y8nWP3zxXXJ8GFabTyABE= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= -gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -1562,10 +1154,9 @@ honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9 lukechampine.com/blake3 v1.1.7 h1:GgRMhmdsuK8+ii6UZFDL8Nb+VyMwadAgcJyfYHxG6n0= lukechampine.com/blake3 v1.1.7/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA= nhooyr.io/websocket v1.8.7 h1:usjR2uOr/zjjkVMy0lW+PPohFok7PCow5sDjLgX4P4g= +nhooyr.io/websocket v1.8.7/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= -sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU= sourcegraph.com/sourcegraph/go-diff v0.5.0/go.mod h1:kuch7UrkMzY0X+p9CRK03kfuPQ2zzQcaEFbx8wA8rck= sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0= diff --git a/ipns/README.md b/ipns/README.md new file mode 100644 index 0000000000..b526674dd1 --- /dev/null +++ b/ipns/README.md @@ -0,0 +1,31 @@ +## Usage + +To create a new IPNS record: + +```go +import ( + "time" + + ipns "github.com/ipfs/go-libipfs/ipns" + crypto "github.com/libp2p/go-libp2p/core/crypto" +) + +// Generate a private key to sign the IPNS record with. Most of the time, +// however, you'll want to retrieve an already-existing key from IPFS using the +// go-ipfs/core/coreapi CoreAPI.KeyAPI() interface. +privateKey, publicKey, err := crypto.GenerateKeyPair(crypto.RSA, 2048) +if err != nil { + panic(err) +} + +// Create an IPNS record that expires in one hour and points to the IPFS address +// /ipfs/Qme1knMqwt1hKZbc1BmQFmnm9f36nyQGwXxPGVpVJ9rMK5 +ipnsRecord, err := ipns.Create(privateKey, []byte("/ipfs/Qme1knMqwt1hKZbc1BmQFmnm9f36nyQGwXxPGVpVJ9rMK5"), 0, time.Now().Add(1*time.Hour)) +if err != nil { + panic(err) +} +``` + +Once you have the record, you’ll need to use IPFS to *publish* it. + +There are several other major operations you can do with `go-ipns`. Check out the [API docs](https://pkg.go.dev/github.com/ipfs/go-libipfs/ipns) or look at the tests in this repo for examples. diff --git a/ipns/errors.go b/ipns/errors.go new file mode 100644 index 0000000000..d78aafffa0 --- /dev/null +++ b/ipns/errors.go @@ -0,0 +1,44 @@ +package ipns + +import ( + "errors" +) + +// ErrExpiredRecord should be returned when an ipns record is +// invalid due to being too old +var ErrExpiredRecord = errors.New("expired record") + +// ErrUnrecognizedValidity is returned when an IpnsRecord has an +// unknown validity type. +var ErrUnrecognizedValidity = errors.New("unrecognized validity type") + +// ErrInvalidPath should be returned when an ipns record path +// is not in a valid format +var ErrInvalidPath = errors.New("record path invalid") + +// ErrSignature should be returned when an ipns record fails +// signature verification +var ErrSignature = errors.New("record signature verification failed") + +// ErrKeyFormat should be returned when an ipns record key is +// incorrectly formatted (not a peer ID) +var ErrKeyFormat = errors.New("record key could not be parsed into peer ID") + +// ErrPublicKeyNotFound should be returned when the public key +// corresponding to the ipns record path cannot be retrieved +// from the peer store +var ErrPublicKeyNotFound = errors.New("public key not found in peer store") + +// ErrPublicKeyMismatch should be returned when the public key embedded in the +// record doesn't match the expected public key. +var ErrPublicKeyMismatch = errors.New("public key in record did not match expected pubkey") + +// ErrBadRecord should be returned when an ipns record cannot be unmarshalled +var ErrBadRecord = errors.New("record could not be unmarshalled") + +// 10 KiB limit defined in https://github.com/ipfs/specs/pull/319 +const MaxRecordSize int = 10 << (10 * 1) + +// ErrRecordSize should be returned when an ipns record is +// invalid due to being too big +var ErrRecordSize = errors.New("record exceeds allowed size limit") diff --git a/ipns/examples/embed.go b/ipns/examples/embed.go new file mode 100644 index 0000000000..8e3c64addc --- /dev/null +++ b/ipns/examples/embed.go @@ -0,0 +1,27 @@ +package examples + +import ( + "time" + + pb "github.com/ipfs/go-libipfs/ipns/pb" + + "github.com/ipfs/go-libipfs/ipns" + "github.com/libp2p/go-libp2p/core/crypto" +) + +// CreateEntryWithEmbed shows how you can create an IPNS entry +// and embed it with a public key. For ed25519 keys this is not needed +// so attempting to embed with an ed25519 key, will not actually embed the key +func CreateEntryWithEmbed(ipfsPath string, publicKey crypto.PubKey, privateKey crypto.PrivKey) (*pb.IpnsEntry, error) { + ipfsPathByte := []byte(ipfsPath) + eol := time.Now().Add(time.Hour * 48) + entry, err := ipns.Create(privateKey, ipfsPathByte, 1, eol, 0) + if err != nil { + return nil, err + } + err = ipns.EmbedPublicKey(publicKey, entry) + if err != nil { + return nil, err + } + return entry, nil +} diff --git a/ipns/examples/examples_test.go b/ipns/examples/examples_test.go new file mode 100644 index 0000000000..c60ffebcbe --- /dev/null +++ b/ipns/examples/examples_test.go @@ -0,0 +1,60 @@ +package examples_test + +import ( + "testing" + + "github.com/ipfs/go-libipfs/ipns/examples" + "github.com/libp2p/go-libp2p/core/crypto" +) + +var testPath = "/ipfs/Qme1knMqwt1hKZbc1BmQFmnm9f36nyQGwXxPGVpVJ9rMK5" + +func TestKeyGeneration(t *testing.T) { + _, err := generateRSAKey() + if err != nil { + t.Error(err) + } + + _, err = generateEDKey() + if err != nil { + t.Error(err) + } +} + +func TestEmbeddedEntryCreation(t *testing.T) { + rk, err := generateRSAKey() + if err != nil { + t.Fatal(err) + } + + ek, err := generateEDKey() + if err != nil { + t.Fatal(err) + } + _, err = examples.CreateEntryWithEmbed(testPath, rk.GetPublic(), rk) + if err != nil { + t.Error(err) + } + + _, err = examples.CreateEntryWithEmbed(testPath, ek.GetPublic(), ek) + if err != nil { + t.Error(err) + } + +} +func generateRSAKey() (crypto.PrivKey, error) { + k, err := examples.GenerateRSAKeyPair(2048) + if err != nil { + return nil, err + } + return k, nil +} + +func generateEDKey() (crypto.PrivKey, error) { + // ED25519 uses 256bit keys, and ignore the bit param + k, err := examples.GenerateEDKeyPair() + if err != nil { + return nil, err + } + return k, nil +} diff --git a/ipns/examples/key.go b/ipns/examples/key.go new file mode 100644 index 0000000000..94f219b8d2 --- /dev/null +++ b/ipns/examples/key.go @@ -0,0 +1,24 @@ +package examples + +import ( + "github.com/libp2p/go-libp2p/core/crypto" +) + +// GenerateRSAKeyPair is used to generate an RSA key pair +func GenerateRSAKeyPair(bits int) (crypto.PrivKey, error) { + priv, _, err := crypto.GenerateKeyPair(crypto.RSA, bits) + if err != nil { + return nil, err + } + return priv, nil +} + +// GenerateEDKeyPair is used to generate an ED25519 keypair +func GenerateEDKeyPair() (crypto.PrivKey, error) { + // ED25519 ignores the bit param and uses 256bit keys + priv, _, err := crypto.GenerateKeyPair(crypto.Ed25519, 256) + if err != nil { + return nil, err + } + return priv, nil +} diff --git a/ipns/ipns.go b/ipns/ipns.go new file mode 100644 index 0000000000..8eff906290 --- /dev/null +++ b/ipns/ipns.go @@ -0,0 +1,398 @@ +package ipns + +import ( + "bytes" + "fmt" + "sort" + "time" + + "github.com/multiformats/go-multicodec" + "github.com/pkg/errors" + + "github.com/ipld/go-ipld-prime" + _ "github.com/ipld/go-ipld-prime/codec/dagcbor" // used to import the DagCbor encoder/decoder + ipldcodec "github.com/ipld/go-ipld-prime/multicodec" + basicnode "github.com/ipld/go-ipld-prime/node/basic" + + "github.com/gogo/protobuf/proto" + + pb "github.com/ipfs/go-libipfs/ipns/pb" + + u "github.com/ipfs/go-libipfs/util" + ic "github.com/libp2p/go-libp2p/core/crypto" + "github.com/libp2p/go-libp2p/core/peer" +) + +const ( + validity = "Validity" + validityType = "ValidityType" + value = "Value" + sequence = "Sequence" + ttl = "TTL" +) + +// Create creates a new IPNS entry and signs it with the given private key. +// +// This function does not embed the public key. If you want to do that, use +// `EmbedPublicKey`. +func Create(sk ic.PrivKey, val []byte, seq uint64, eol time.Time, ttl time.Duration) (*pb.IpnsEntry, error) { + entry := new(pb.IpnsEntry) + + entry.Value = val + typ := pb.IpnsEntry_EOL + entry.ValidityType = &typ + entry.Sequence = &seq + entry.Validity = []byte(u.FormatRFC3339(eol)) + + ttlNs := uint64(ttl.Nanoseconds()) + entry.Ttl = proto.Uint64(ttlNs) + + cborData, err := createCborDataForIpnsEntry(entry) + if err != nil { + return nil, err + } + entry.Data = cborData + + // For now we still create V1 signatures. These are deprecated, and not + // used during verification anymore (Validate func requires SignatureV2), + // but setting it here allows legacy nodes (e.g., go-ipfs < v0.9.0) to + // still resolve IPNS published by modern nodes. + sig1, err := sk.Sign(ipnsEntryDataForSigV1(entry)) + if err != nil { + return nil, errors.Wrap(err, "could not compute signature data") + } + entry.SignatureV1 = sig1 + + sig2Data, err := ipnsEntryDataForSigV2(entry) + if err != nil { + return nil, err + } + sig2, err := sk.Sign(sig2Data) + if err != nil { + return nil, err + } + entry.SignatureV2 = sig2 + + return entry, nil +} + +func createCborDataForIpnsEntry(e *pb.IpnsEntry) ([]byte, error) { + m := make(map[string]ipld.Node) + var keys []string + m[value] = basicnode.NewBytes(e.GetValue()) + keys = append(keys, value) + + m[validity] = basicnode.NewBytes(e.GetValidity()) + keys = append(keys, validity) + + m[validityType] = basicnode.NewInt(int64(e.GetValidityType())) + keys = append(keys, validityType) + + m[sequence] = basicnode.NewInt(int64(e.GetSequence())) + keys = append(keys, sequence) + + m[ttl] = basicnode.NewInt(int64(e.GetTtl())) + keys = append(keys, ttl) + + sort.Sort(cborMapKeyString_RFC7049(keys)) + + newNd := basicnode.Prototype__Map{}.NewBuilder() + ma, err := newNd.BeginMap(int64(len(keys))) + if err != nil { + return nil, err + } + + for _, k := range keys { + if err := ma.AssembleKey().AssignString(k); err != nil { + return nil, err + } + if err := ma.AssembleValue().AssignNode(m[k]); err != nil { + return nil, err + } + } + + if err := ma.Finish(); err != nil { + return nil, err + } + + nd := newNd.Build() + + enc, err := ipldcodec.LookupEncoder(uint64(multicodec.DagCbor)) + if err != nil { + return nil, err + } + + buf := new(bytes.Buffer) + if err := enc(nd, buf); err != nil { + return nil, err + } + + return buf.Bytes(), nil +} + +// Validates validates the given IPNS entry against the given public key. +func Validate(pk ic.PubKey, entry *pb.IpnsEntry) error { + // Make sure max size is respected + if entry.Size() > MaxRecordSize { + return ErrRecordSize + } + + // Check the ipns record signature with the public key + if entry.GetSignatureV2() == nil { + // always error if no valid signature could be found + return ErrSignature + } + + sig2Data, err := ipnsEntryDataForSigV2(entry) + if err != nil { + return fmt.Errorf("could not compute signature data: %w", err) + } + if ok, err := pk.Verify(sig2Data, entry.GetSignatureV2()); err != nil || !ok { + return ErrSignature + } + + // TODO: If we switch from pb.IpnsEntry to a more generic IpnsRecord type then perhaps we should only check + // this if there is no v1 signature. In the meanwhile this helps avoid some potential rough edges around people + // checking the entry fields instead of doing CBOR decoding everywhere. + // See https://github.com/ipfs/go-libipfs/ipns/pull/42 for next steps here + if err := validateCborDataMatchesPbData(entry); err != nil { + return err + } + + eol, err := GetEOL(entry) + if err != nil { + return err + } + if time.Now().After(eol) { + return ErrExpiredRecord + } + return nil +} + +// TODO: Most of this function could probably be replaced with codegen +func validateCborDataMatchesPbData(entry *pb.IpnsEntry) error { + if len(entry.GetData()) == 0 { + return fmt.Errorf("record data is missing") + } + + dec, err := ipldcodec.LookupDecoder(uint64(multicodec.DagCbor)) + if err != nil { + return err + } + + ndbuilder := basicnode.Prototype__Map{}.NewBuilder() + if err := dec(ndbuilder, bytes.NewReader(entry.GetData())); err != nil { + return err + } + + fullNd := ndbuilder.Build() + nd, err := fullNd.LookupByString(value) + if err != nil { + return err + } + ndBytes, err := nd.AsBytes() + if err != nil { + return err + } + if !bytes.Equal(entry.GetValue(), ndBytes) { + return fmt.Errorf("field \"%v\" did not match between protobuf and CBOR", value) + } + + nd, err = fullNd.LookupByString(validity) + if err != nil { + return err + } + ndBytes, err = nd.AsBytes() + if err != nil { + return err + } + if !bytes.Equal(entry.GetValidity(), ndBytes) { + return fmt.Errorf("field \"%v\" did not match between protobuf and CBOR", validity) + } + + nd, err = fullNd.LookupByString(validityType) + if err != nil { + return err + } + ndInt, err := nd.AsInt() + if err != nil { + return err + } + if int64(entry.GetValidityType()) != ndInt { + return fmt.Errorf("field \"%v\" did not match between protobuf and CBOR", validityType) + } + + nd, err = fullNd.LookupByString(sequence) + if err != nil { + return err + } + ndInt, err = nd.AsInt() + if err != nil { + return err + } + + if entry.GetSequence() != uint64(ndInt) { + return fmt.Errorf("field \"%v\" did not match between protobuf and CBOR", sequence) + } + + nd, err = fullNd.LookupByString("TTL") + if err != nil { + return err + } + ndInt, err = nd.AsInt() + if err != nil { + return err + } + if entry.GetTtl() != uint64(ndInt) { + return fmt.Errorf("field \"%v\" did not match between protobuf and CBOR", ttl) + } + + return nil +} + +// GetEOL returns the EOL of this IPNS entry +// +// This function returns ErrUnrecognizedValidity if the validity type of the +// record isn't EOL. Otherwise, it returns an error if it can't parse the EOL. +func GetEOL(entry *pb.IpnsEntry) (time.Time, error) { + if entry.GetValidityType() != pb.IpnsEntry_EOL { + return time.Time{}, ErrUnrecognizedValidity + } + return u.ParseRFC3339(string(entry.GetValidity())) +} + +// EmbedPublicKey embeds the given public key in the given ipns entry. While not +// strictly required, some nodes (e.g., DHT servers) may reject IPNS entries +// that don't embed their public keys as they may not be able to validate them +// efficiently. +func EmbedPublicKey(pk ic.PubKey, entry *pb.IpnsEntry) error { + // Try extracting the public key from the ID. If we can, *don't* embed + // it. + id, err := peer.IDFromPublicKey(pk) + if err != nil { + return err + } + if _, err := id.ExtractPublicKey(); err != peer.ErrNoPublicKey { + // Either a *real* error or nil. + return err + } + + // We failed to extract the public key from the peer ID, embed it in the + // record. + pkBytes, err := ic.MarshalPublicKey(pk) + if err != nil { + return err + } + entry.PubKey = pkBytes + return nil +} + +// ExtractPublicKey extracts a public key matching `pid` from the IPNS record, +// if possible. +// +// This function returns (nil, nil) when no public key can be extracted and +// nothing is malformed. +func ExtractPublicKey(pid peer.ID, entry *pb.IpnsEntry) (ic.PubKey, error) { + if entry.PubKey != nil { + pk, err := ic.UnmarshalPublicKey(entry.PubKey) + if err != nil { + return nil, fmt.Errorf("unmarshaling pubkey in record: %s", err) + } + + expPid, err := peer.IDFromPublicKey(pk) + if err != nil { + return nil, fmt.Errorf("could not regenerate peerID from pubkey: %s", err) + } + + if pid != expPid { + return nil, ErrPublicKeyMismatch + } + return pk, nil + } + + return pid.ExtractPublicKey() +} + +// Compare compares two IPNS entries. It returns: +// +// * -1 if a is older than b +// * 0 if a and b cannot be ordered (this doesn't mean that they are equal) +// * +1 if a is newer than b +// +// It returns an error when either a or b are malformed. +// +// NOTE: It *does not* validate the records, the caller is responsible for calling +// `Validate` first. +// +// NOTE: If a and b cannot be ordered by this function, you can determine their +// order by comparing their serialized byte representations (using +// `bytes.Compare`). You must do this if you are implementing a libp2p record +// validator (or you can just use the one provided for you by this package). +func Compare(a, b *pb.IpnsEntry) (int, error) { + aHasV2Sig := a.GetSignatureV2() != nil + bHasV2Sig := b.GetSignatureV2() != nil + + // Having a newer signature version is better than an older signature version + if aHasV2Sig && !bHasV2Sig { + return 1, nil + } else if !aHasV2Sig && bHasV2Sig { + return -1, nil + } + + as := a.GetSequence() + bs := b.GetSequence() + + if as > bs { + return 1, nil + } else if as < bs { + return -1, nil + } + + at, err := u.ParseRFC3339(string(a.GetValidity())) + if err != nil { + return 0, err + } + + bt, err := u.ParseRFC3339(string(b.GetValidity())) + if err != nil { + return 0, err + } + + if at.After(bt) { + return 1, nil + } else if bt.After(at) { + return -1, nil + } + + return 0, nil +} + +func ipnsEntryDataForSigV1(e *pb.IpnsEntry) []byte { + return bytes.Join([][]byte{ + e.Value, + e.Validity, + []byte(fmt.Sprint(e.GetValidityType())), + }, + []byte{}) +} + +func ipnsEntryDataForSigV2(e *pb.IpnsEntry) ([]byte, error) { + dataForSig := []byte("ipns-signature:") + dataForSig = append(dataForSig, e.Data...) + + return dataForSig, nil +} + +type cborMapKeyString_RFC7049 []string + +func (x cborMapKeyString_RFC7049) Len() int { return len(x) } +func (x cborMapKeyString_RFC7049) Swap(i, j int) { x[i], x[j] = x[j], x[i] } +func (x cborMapKeyString_RFC7049) Less(i, j int) bool { + li, lj := len(x[i]), len(x[j]) + if li == lj { + return x[i] < x[j] + } + return li < lj +} + +var _ sort.Interface = (cborMapKeyString_RFC7049)(nil) diff --git a/ipns/ipns_test.go b/ipns/ipns_test.go new file mode 100644 index 0000000000..a4b700744a --- /dev/null +++ b/ipns/ipns_test.go @@ -0,0 +1,63 @@ +package ipns + +import ( + "fmt" + "testing" + "time" + + u "github.com/ipfs/go-libipfs/util" + ci "github.com/libp2p/go-libp2p/core/crypto" + "github.com/libp2p/go-libp2p/core/peer" +) + +func TestEmbedPublicKey(t *testing.T) { + + sr := u.NewTimeSeededRand() + priv, pub, err := ci.GenerateKeyPairWithReader(ci.RSA, 2048, sr) + if err != nil { + t.Fatal(err) + } + + pid, err := peer.IDFromPublicKey(pub) + if err != nil { + t.Fatal(err) + } + + e, err := Create(priv, []byte("/a/b"), 0, time.Now().Add(1*time.Hour), 0) + if err != nil { + t.Fatal(err) + } + if err := EmbedPublicKey(pub, e); err != nil { + t.Fatal(err) + } + embeddedPk, err := ci.UnmarshalPublicKey(e.PubKey) + if err != nil { + t.Fatal(err) + } + embeddedPid, err := peer.IDFromPublicKey(embeddedPk) + if err != nil { + t.Fatal(err) + } + if embeddedPid != pid { + t.Fatalf("pid mismatch: %s != %s", pid, embeddedPid) + } +} + +func ExampleCreate() { + // Generate a private key to sign the IPNS record with. Most of the time, + // however, you'll want to retrieve an already-existing key from IPFS using + // go-ipfs/core/coreapi CoreAPI.KeyAPI() interface. + privateKey, _, err := ci.GenerateKeyPair(ci.RSA, 2048) + if err != nil { + panic(err) + } + + // Create an IPNS record that expires in one hour and points to the IPFS address + // /ipfs/Qme1knMqwt1hKZbc1BmQFmnm9f36nyQGwXxPGVpVJ9rMK5 + ipnsRecord, err := Create(privateKey, []byte("/ipfs/Qme1knMqwt1hKZbc1BmQFmnm9f36nyQGwXxPGVpVJ9rMK5"), 0, time.Now().Add(1*time.Hour), 0) + if err != nil { + panic(err) + } + + fmt.Println(ipnsRecord) +} diff --git a/ipns/pb/Makefile b/ipns/pb/Makefile new file mode 100644 index 0000000000..eb14b5768a --- /dev/null +++ b/ipns/pb/Makefile @@ -0,0 +1,11 @@ +PB = $(wildcard *.proto) +GO = $(PB:.proto=.pb.go) + +all: $(GO) + +%.pb.go: %.proto + protoc --proto_path=$(GOPATH)/src:. --gogofast_out=. $< + +clean: + rm -f *.pb.go + rm -f *.go diff --git a/ipns/pb/ipns.pb.go b/ipns/pb/ipns.pb.go new file mode 100644 index 0000000000..8bbb654d21 --- /dev/null +++ b/ipns/pb/ipns.pb.go @@ -0,0 +1,987 @@ +// Code generated by protoc-gen-gogo. DO NOT EDIT. +// source: ipns.proto + +package ipns_pb + +import ( + fmt "fmt" + io "io" + math "math" + math_bits "math/bits" + + proto "github.com/gogo/protobuf/proto" +) + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package + +type IpnsEntry_ValidityType int32 + +const ( + // setting an EOL says "this record is valid until..." + IpnsEntry_EOL IpnsEntry_ValidityType = 0 +) + +var IpnsEntry_ValidityType_name = map[int32]string{ + 0: "EOL", +} + +var IpnsEntry_ValidityType_value = map[string]int32{ + "EOL": 0, +} + +func (x IpnsEntry_ValidityType) Enum() *IpnsEntry_ValidityType { + p := new(IpnsEntry_ValidityType) + *p = x + return p +} + +func (x IpnsEntry_ValidityType) String() string { + return proto.EnumName(IpnsEntry_ValidityType_name, int32(x)) +} + +func (x *IpnsEntry_ValidityType) UnmarshalJSON(data []byte) error { + value, err := proto.UnmarshalJSONEnum(IpnsEntry_ValidityType_value, data, "IpnsEntry_ValidityType") + if err != nil { + return err + } + *x = IpnsEntry_ValidityType(value) + return nil +} + +func (IpnsEntry_ValidityType) EnumDescriptor() ([]byte, []int) { + return fileDescriptor_4d5b16fb32bfe8ea, []int{0, 0} +} + +type IpnsEntry struct { + Value []byte `protobuf:"bytes,1,opt,name=value" json:"value,omitempty"` + SignatureV1 []byte `protobuf:"bytes,2,opt,name=signatureV1" json:"signatureV1,omitempty"` + ValidityType *IpnsEntry_ValidityType `protobuf:"varint,3,opt,name=validityType,enum=ipns.pb.IpnsEntry_ValidityType" json:"validityType,omitempty"` + Validity []byte `protobuf:"bytes,4,opt,name=validity" json:"validity,omitempty"` + Sequence *uint64 `protobuf:"varint,5,opt,name=sequence" json:"sequence,omitempty"` + Ttl *uint64 `protobuf:"varint,6,opt,name=ttl" json:"ttl,omitempty"` + // in order for nodes to properly validate a record upon receipt, they need the public + // key associated with it. For old RSA keys, its easiest if we just send this as part of + // the record itself. For newer ed25519 keys, the public key can be embedded in the + // peerID, making this field unnecessary. + PubKey []byte `protobuf:"bytes,7,opt,name=pubKey" json:"pubKey,omitempty"` + SignatureV2 []byte `protobuf:"bytes,8,opt,name=signatureV2" json:"signatureV2,omitempty"` + Data []byte `protobuf:"bytes,9,opt,name=data" json:"data,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *IpnsEntry) Reset() { *m = IpnsEntry{} } +func (m *IpnsEntry) String() string { return proto.CompactTextString(m) } +func (*IpnsEntry) ProtoMessage() {} +func (*IpnsEntry) Descriptor() ([]byte, []int) { + return fileDescriptor_4d5b16fb32bfe8ea, []int{0} +} +func (m *IpnsEntry) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *IpnsEntry) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_IpnsEntry.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *IpnsEntry) XXX_Merge(src proto.Message) { + xxx_messageInfo_IpnsEntry.Merge(m, src) +} +func (m *IpnsEntry) XXX_Size() int { + return m.Size() +} +func (m *IpnsEntry) XXX_DiscardUnknown() { + xxx_messageInfo_IpnsEntry.DiscardUnknown(m) +} + +var xxx_messageInfo_IpnsEntry proto.InternalMessageInfo + +func (m *IpnsEntry) GetValue() []byte { + if m != nil { + return m.Value + } + return nil +} + +func (m *IpnsEntry) GetSignatureV1() []byte { + if m != nil { + return m.SignatureV1 + } + return nil +} + +func (m *IpnsEntry) GetValidityType() IpnsEntry_ValidityType { + if m != nil && m.ValidityType != nil { + return *m.ValidityType + } + return IpnsEntry_EOL +} + +func (m *IpnsEntry) GetValidity() []byte { + if m != nil { + return m.Validity + } + return nil +} + +func (m *IpnsEntry) GetSequence() uint64 { + if m != nil && m.Sequence != nil { + return *m.Sequence + } + return 0 +} + +func (m *IpnsEntry) GetTtl() uint64 { + if m != nil && m.Ttl != nil { + return *m.Ttl + } + return 0 +} + +func (m *IpnsEntry) GetPubKey() []byte { + if m != nil { + return m.PubKey + } + return nil +} + +func (m *IpnsEntry) GetSignatureV2() []byte { + if m != nil { + return m.SignatureV2 + } + return nil +} + +func (m *IpnsEntry) GetData() []byte { + if m != nil { + return m.Data + } + return nil +} + +type IpnsSignatureV2Checker struct { + PubKey []byte `protobuf:"bytes,7,opt,name=pubKey" json:"pubKey,omitempty"` + SignatureV2 []byte `protobuf:"bytes,8,opt,name=signatureV2" json:"signatureV2,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *IpnsSignatureV2Checker) Reset() { *m = IpnsSignatureV2Checker{} } +func (m *IpnsSignatureV2Checker) String() string { return proto.CompactTextString(m) } +func (*IpnsSignatureV2Checker) ProtoMessage() {} +func (*IpnsSignatureV2Checker) Descriptor() ([]byte, []int) { + return fileDescriptor_4d5b16fb32bfe8ea, []int{1} +} +func (m *IpnsSignatureV2Checker) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *IpnsSignatureV2Checker) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_IpnsSignatureV2Checker.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *IpnsSignatureV2Checker) XXX_Merge(src proto.Message) { + xxx_messageInfo_IpnsSignatureV2Checker.Merge(m, src) +} +func (m *IpnsSignatureV2Checker) XXX_Size() int { + return m.Size() +} +func (m *IpnsSignatureV2Checker) XXX_DiscardUnknown() { + xxx_messageInfo_IpnsSignatureV2Checker.DiscardUnknown(m) +} + +var xxx_messageInfo_IpnsSignatureV2Checker proto.InternalMessageInfo + +func (m *IpnsSignatureV2Checker) GetPubKey() []byte { + if m != nil { + return m.PubKey + } + return nil +} + +func (m *IpnsSignatureV2Checker) GetSignatureV2() []byte { + if m != nil { + return m.SignatureV2 + } + return nil +} + +func init() { + proto.RegisterEnum("ipns.pb.IpnsEntry_ValidityType", IpnsEntry_ValidityType_name, IpnsEntry_ValidityType_value) + proto.RegisterType((*IpnsEntry)(nil), "ipns.pb.IpnsEntry") + proto.RegisterType((*IpnsSignatureV2Checker)(nil), "ipns.pb.IpnsSignatureV2Checker") +} + +func init() { proto.RegisterFile("ipns.proto", fileDescriptor_4d5b16fb32bfe8ea) } + +var fileDescriptor_4d5b16fb32bfe8ea = []byte{ + // 272 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xe2, 0xca, 0x2c, 0xc8, 0x2b, + 0xd6, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0x62, 0x87, 0xb0, 0x93, 0x94, 0x76, 0x32, 0x71, 0x71, + 0x7a, 0x16, 0xe4, 0x15, 0xbb, 0xe6, 0x95, 0x14, 0x55, 0x0a, 0x89, 0x70, 0xb1, 0x96, 0x25, 0xe6, + 0x94, 0xa6, 0x4a, 0x30, 0x2a, 0x30, 0x6a, 0xf0, 0x04, 0x41, 0x38, 0x42, 0x0a, 0x5c, 0xdc, 0xc5, + 0x99, 0xe9, 0x79, 0x89, 0x25, 0xa5, 0x45, 0xa9, 0x61, 0x86, 0x12, 0x4c, 0x60, 0x39, 0x64, 0x21, + 0x21, 0x67, 0x2e, 0x9e, 0xb2, 0xc4, 0x9c, 0xcc, 0x94, 0xcc, 0x92, 0xca, 0x90, 0xca, 0x82, 0x54, + 0x09, 0x66, 0x05, 0x46, 0x0d, 0x3e, 0x23, 0x79, 0x3d, 0xa8, 0x2d, 0x7a, 0x70, 0x1b, 0xf4, 0xc2, + 0x90, 0x94, 0x05, 0xa1, 0x68, 0x12, 0x92, 0xe2, 0xe2, 0x80, 0xf1, 0x25, 0x58, 0xc0, 0x76, 0xc0, + 0xf9, 0x20, 0xb9, 0xe2, 0xd4, 0xc2, 0xd2, 0xd4, 0xbc, 0xe4, 0x54, 0x09, 0x56, 0x05, 0x46, 0x0d, + 0x96, 0x20, 0x38, 0x5f, 0x48, 0x80, 0x8b, 0xb9, 0xa4, 0x24, 0x47, 0x82, 0x0d, 0x2c, 0x0c, 0x62, + 0x0a, 0x89, 0x71, 0xb1, 0x15, 0x94, 0x26, 0x79, 0xa7, 0x56, 0x4a, 0xb0, 0x83, 0xcd, 0x81, 0xf2, + 0x50, 0x3d, 0x62, 0x24, 0xc1, 0x81, 0xee, 0x11, 0x23, 0x21, 0x21, 0x2e, 0x96, 0x94, 0xc4, 0x92, + 0x44, 0x09, 0x4e, 0xb0, 0x14, 0x98, 0xad, 0x24, 0xce, 0xc5, 0x83, 0xec, 0x6a, 0x21, 0x76, 0x2e, + 0x66, 0x57, 0x7f, 0x1f, 0x01, 0x06, 0xa5, 0x20, 0x2e, 0x31, 0x90, 0xc7, 0x82, 0x11, 0xfa, 0x9d, + 0x33, 0x52, 0x93, 0xb3, 0x53, 0x8b, 0xc8, 0x77, 0x80, 0x93, 0xc4, 0x89, 0x47, 0x72, 0x8c, 0x17, + 0x1e, 0xc9, 0x31, 0x3e, 0x78, 0x24, 0xc7, 0x18, 0xc5, 0xa5, 0xa7, 0x6f, 0x0d, 0x0a, 0xc3, 0xf8, + 0x82, 0x24, 0x40, 0x00, 0x00, 0x00, 0xff, 0xff, 0x32, 0x85, 0x5b, 0xed, 0xbf, 0x01, 0x00, 0x00, +} + +func (m *IpnsEntry) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *IpnsEntry) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *IpnsEntry) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if m.Data != nil { + i -= len(m.Data) + copy(dAtA[i:], m.Data) + i = encodeVarintIpns(dAtA, i, uint64(len(m.Data))) + i-- + dAtA[i] = 0x4a + } + if m.SignatureV2 != nil { + i -= len(m.SignatureV2) + copy(dAtA[i:], m.SignatureV2) + i = encodeVarintIpns(dAtA, i, uint64(len(m.SignatureV2))) + i-- + dAtA[i] = 0x42 + } + if m.PubKey != nil { + i -= len(m.PubKey) + copy(dAtA[i:], m.PubKey) + i = encodeVarintIpns(dAtA, i, uint64(len(m.PubKey))) + i-- + dAtA[i] = 0x3a + } + if m.Ttl != nil { + i = encodeVarintIpns(dAtA, i, uint64(*m.Ttl)) + i-- + dAtA[i] = 0x30 + } + if m.Sequence != nil { + i = encodeVarintIpns(dAtA, i, uint64(*m.Sequence)) + i-- + dAtA[i] = 0x28 + } + if m.Validity != nil { + i -= len(m.Validity) + copy(dAtA[i:], m.Validity) + i = encodeVarintIpns(dAtA, i, uint64(len(m.Validity))) + i-- + dAtA[i] = 0x22 + } + if m.ValidityType != nil { + i = encodeVarintIpns(dAtA, i, uint64(*m.ValidityType)) + i-- + dAtA[i] = 0x18 + } + if m.SignatureV1 != nil { + i -= len(m.SignatureV1) + copy(dAtA[i:], m.SignatureV1) + i = encodeVarintIpns(dAtA, i, uint64(len(m.SignatureV1))) + i-- + dAtA[i] = 0x12 + } + if m.Value != nil { + i -= len(m.Value) + copy(dAtA[i:], m.Value) + i = encodeVarintIpns(dAtA, i, uint64(len(m.Value))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *IpnsSignatureV2Checker) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *IpnsSignatureV2Checker) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *IpnsSignatureV2Checker) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if m.SignatureV2 != nil { + i -= len(m.SignatureV2) + copy(dAtA[i:], m.SignatureV2) + i = encodeVarintIpns(dAtA, i, uint64(len(m.SignatureV2))) + i-- + dAtA[i] = 0x42 + } + if m.PubKey != nil { + i -= len(m.PubKey) + copy(dAtA[i:], m.PubKey) + i = encodeVarintIpns(dAtA, i, uint64(len(m.PubKey))) + i-- + dAtA[i] = 0x3a + } + return len(dAtA) - i, nil +} + +func encodeVarintIpns(dAtA []byte, offset int, v uint64) int { + offset -= sovIpns(v) + base := offset + for v >= 1<<7 { + dAtA[offset] = uint8(v&0x7f | 0x80) + v >>= 7 + offset++ + } + dAtA[offset] = uint8(v) + return base +} +func (m *IpnsEntry) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Value != nil { + l = len(m.Value) + n += 1 + l + sovIpns(uint64(l)) + } + if m.SignatureV1 != nil { + l = len(m.SignatureV1) + n += 1 + l + sovIpns(uint64(l)) + } + if m.ValidityType != nil { + n += 1 + sovIpns(uint64(*m.ValidityType)) + } + if m.Validity != nil { + l = len(m.Validity) + n += 1 + l + sovIpns(uint64(l)) + } + if m.Sequence != nil { + n += 1 + sovIpns(uint64(*m.Sequence)) + } + if m.Ttl != nil { + n += 1 + sovIpns(uint64(*m.Ttl)) + } + if m.PubKey != nil { + l = len(m.PubKey) + n += 1 + l + sovIpns(uint64(l)) + } + if m.SignatureV2 != nil { + l = len(m.SignatureV2) + n += 1 + l + sovIpns(uint64(l)) + } + if m.Data != nil { + l = len(m.Data) + n += 1 + l + sovIpns(uint64(l)) + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func (m *IpnsSignatureV2Checker) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.PubKey != nil { + l = len(m.PubKey) + n += 1 + l + sovIpns(uint64(l)) + } + if m.SignatureV2 != nil { + l = len(m.SignatureV2) + n += 1 + l + sovIpns(uint64(l)) + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func sovIpns(x uint64) (n int) { + return (math_bits.Len64(x|1) + 6) / 7 +} +func sozIpns(x uint64) (n int) { + return sovIpns(uint64((x << 1) ^ uint64((int64(x) >> 63)))) +} +func (m *IpnsEntry) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowIpns + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: IpnsEntry: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: IpnsEntry: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Value", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowIpns + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthIpns + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthIpns + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Value = append(m.Value[:0], dAtA[iNdEx:postIndex]...) + if m.Value == nil { + m.Value = []byte{} + } + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field SignatureV1", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowIpns + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthIpns + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthIpns + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.SignatureV1 = append(m.SignatureV1[:0], dAtA[iNdEx:postIndex]...) + if m.SignatureV1 == nil { + m.SignatureV1 = []byte{} + } + iNdEx = postIndex + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field ValidityType", wireType) + } + var v IpnsEntry_ValidityType + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowIpns + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= IpnsEntry_ValidityType(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.ValidityType = &v + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Validity", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowIpns + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthIpns + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthIpns + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Validity = append(m.Validity[:0], dAtA[iNdEx:postIndex]...) + if m.Validity == nil { + m.Validity = []byte{} + } + iNdEx = postIndex + case 5: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Sequence", wireType) + } + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowIpns + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.Sequence = &v + case 6: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Ttl", wireType) + } + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowIpns + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.Ttl = &v + case 7: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field PubKey", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowIpns + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthIpns + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthIpns + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.PubKey = append(m.PubKey[:0], dAtA[iNdEx:postIndex]...) + if m.PubKey == nil { + m.PubKey = []byte{} + } + iNdEx = postIndex + case 8: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field SignatureV2", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowIpns + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthIpns + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthIpns + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.SignatureV2 = append(m.SignatureV2[:0], dAtA[iNdEx:postIndex]...) + if m.SignatureV2 == nil { + m.SignatureV2 = []byte{} + } + iNdEx = postIndex + case 9: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Data", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowIpns + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthIpns + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthIpns + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Data = append(m.Data[:0], dAtA[iNdEx:postIndex]...) + if m.Data == nil { + m.Data = []byte{} + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipIpns(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthIpns + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *IpnsSignatureV2Checker) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowIpns + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: IpnsSignatureV2Checker: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: IpnsSignatureV2Checker: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 7: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field PubKey", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowIpns + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthIpns + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthIpns + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.PubKey = append(m.PubKey[:0], dAtA[iNdEx:postIndex]...) + if m.PubKey == nil { + m.PubKey = []byte{} + } + iNdEx = postIndex + case 8: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field SignatureV2", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowIpns + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthIpns + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthIpns + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.SignatureV2 = append(m.SignatureV2[:0], dAtA[iNdEx:postIndex]...) + if m.SignatureV2 == nil { + m.SignatureV2 = []byte{} + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipIpns(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthIpns + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func skipIpns(dAtA []byte) (n int, err error) { + l := len(dAtA) + iNdEx := 0 + depth := 0 + for iNdEx < l { + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowIpns + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + wireType := int(wire & 0x7) + switch wireType { + case 0: + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowIpns + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + iNdEx++ + if dAtA[iNdEx-1] < 0x80 { + break + } + } + case 1: + iNdEx += 8 + case 2: + var length int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowIpns + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + length |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if length < 0 { + return 0, ErrInvalidLengthIpns + } + iNdEx += length + case 3: + depth++ + case 4: + if depth == 0 { + return 0, ErrUnexpectedEndOfGroupIpns + } + depth-- + case 5: + iNdEx += 4 + default: + return 0, fmt.Errorf("proto: illegal wireType %d", wireType) + } + if iNdEx < 0 { + return 0, ErrInvalidLengthIpns + } + if depth == 0 { + return iNdEx, nil + } + } + return 0, io.ErrUnexpectedEOF +} + +var ( + ErrInvalidLengthIpns = fmt.Errorf("proto: negative length found during unmarshaling") + ErrIntOverflowIpns = fmt.Errorf("proto: integer overflow") + ErrUnexpectedEndOfGroupIpns = fmt.Errorf("proto: unexpected end of group") +) diff --git a/ipns/pb/ipns.proto b/ipns/pb/ipns.proto new file mode 100644 index 0000000000..9fd8bda83b --- /dev/null +++ b/ipns/pb/ipns.proto @@ -0,0 +1,36 @@ +syntax = "proto2"; + +package ipns.pb; + +option go_package = "./;ipns_pb"; + +message IpnsEntry { + enum ValidityType { + // setting an EOL says "this record is valid until..." + EOL = 0; + } + optional bytes value = 1; + optional bytes signatureV1 = 2; + + optional ValidityType validityType = 3; + optional bytes validity = 4; + + optional uint64 sequence = 5; + + optional uint64 ttl = 6; + + // in order for nodes to properly validate a record upon receipt, they need the public + // key associated with it. For old RSA keys, its easiest if we just send this as part of + // the record itself. For newer ed25519 keys, the public key can be embedded in the + // peerID, making this field unnecessary. + optional bytes pubKey = 7; + + optional bytes signatureV2 = 8; + + optional bytes data = 9; +} + +message IpnsSignatureV2Checker { + optional bytes pubKey = 7; + optional bytes signatureV2 = 8; +} diff --git a/ipns/record.go b/ipns/record.go new file mode 100644 index 0000000000..079e34d07c --- /dev/null +++ b/ipns/record.go @@ -0,0 +1,126 @@ +package ipns + +import ( + "bytes" + "errors" + + pb "github.com/ipfs/go-libipfs/ipns/pb" + + "github.com/gogo/protobuf/proto" + logging "github.com/ipfs/go-log/v2" + record "github.com/libp2p/go-libp2p-record" + ic "github.com/libp2p/go-libp2p/core/crypto" + "github.com/libp2p/go-libp2p/core/peer" + pstore "github.com/libp2p/go-libp2p/core/peerstore" +) + +var log = logging.Logger("ipns") + +var _ record.Validator = Validator{} + +// RecordKey returns the libp2p record key for a given peer ID. +func RecordKey(pid peer.ID) string { + return "/ipns/" + string(pid) +} + +// Validator is an IPNS record validator that satisfies the libp2p record +// validator interface. +type Validator struct { + // KeyBook, if non-nil, will be used to lookup keys for validating IPNS + // records. + KeyBook pstore.KeyBook +} + +// Validate validates an IPNS record. +func (v Validator) Validate(key string, value []byte) error { + ns, pidString, err := record.SplitKey(key) + if err != nil || ns != "ipns" { + return ErrInvalidPath + } + + // Parse the value into an IpnsEntry + entry := new(pb.IpnsEntry) + err = proto.Unmarshal(value, entry) + if err != nil { + return ErrBadRecord + } + + // Get the public key defined by the ipns path + pid, err := peer.IDFromBytes([]byte(pidString)) + if err != nil { + log.Debugf("failed to parse ipns record key %s into peer ID", pidString) + return ErrKeyFormat + } + + pubk, err := v.getPublicKey(pid, entry) + if err != nil { + return err + } + + return Validate(pubk, entry) +} + +func (v Validator) getPublicKey(pid peer.ID, entry *pb.IpnsEntry) (ic.PubKey, error) { + switch pk, err := ExtractPublicKey(pid, entry); err { + case peer.ErrNoPublicKey: + case nil: + return pk, nil + default: + return nil, err + } + + if v.KeyBook == nil { + log.Debugf("public key with hash %s not found in IPNS record and no peer store provided", pid) + return nil, ErrPublicKeyNotFound + } + + pubk := v.KeyBook.PubKey(pid) + if pubk == nil { + log.Debugf("public key with hash %s not found in peer store", pid) + return nil, ErrPublicKeyNotFound + } + return pubk, nil +} + +// Select selects the best record by checking which has the highest sequence +// number and latest EOL. +// +// This function returns an error if any of the records fail to parse. Validate +// your records first! +func (v Validator) Select(k string, vals [][]byte) (int, error) { + var recs []*pb.IpnsEntry + for _, v := range vals { + e := new(pb.IpnsEntry) + if err := proto.Unmarshal(v, e); err != nil { + return -1, err + } + recs = append(recs, e) + } + + return selectRecord(recs, vals) +} + +func selectRecord(recs []*pb.IpnsEntry, vals [][]byte) (int, error) { + switch len(recs) { + case 0: + return -1, errors.New("no usable records in given set") + case 1: + return 0, nil + } + + var i int + for j := 1; j < len(recs); j++ { + cmp, err := Compare(recs[i], recs[j]) + if err != nil { + return -1, err + } + if cmp == 0 { + cmp = bytes.Compare(vals[i], vals[j]) + } + if cmp < 0 { + i = j + } + } + + return i, nil +} diff --git a/ipns/select_test.go b/ipns/select_test.go new file mode 100644 index 0000000000..30e8b19178 --- /dev/null +++ b/ipns/select_test.go @@ -0,0 +1,126 @@ +package ipns + +import ( + "fmt" + "math/rand" + "testing" + "time" + + pb "github.com/ipfs/go-libipfs/ipns/pb" + + "github.com/gogo/protobuf/proto" + u "github.com/ipfs/go-libipfs/util" + ci "github.com/libp2p/go-libp2p/core/crypto" +) + +func shuffle(a []*pb.IpnsEntry) { + for n := 0; n < 5; n++ { + for i := range a { + j := rand.Intn(len(a)) + a[i], a[j] = a[j], a[i] + } + } +} + +func AssertSelected(r *pb.IpnsEntry, from ...*pb.IpnsEntry) error { + shuffle(from) + var vals [][]byte + for _, r := range from { + data, err := proto.Marshal(r) + if err != nil { + return err + } + vals = append(vals, data) + } + + i, err := selectRecord(from, vals) + if err != nil { + return err + } + + if from[i] != r { + return fmt.Errorf("selected incorrect record %d", i) + } + + return nil +} + +func TestOrdering(t *testing.T) { + // select timestamp so selection is deterministic + ts := time.Unix(1000000, 0) + + // generate a key for signing the records + r := u.NewSeededRand(15) // generate deterministic keypair + priv, _, err := ci.GenerateKeyPairWithReader(ci.RSA, 2048, r) + if err != nil { + t.Fatal(err) + } + + e1, err := Create(priv, []byte("foo"), 1, ts.Add(time.Hour), 0) + if err != nil { + t.Fatal(err) + } + + e2, err := Create(priv, []byte("bar"), 2, ts.Add(time.Hour), 0) + if err != nil { + t.Fatal(err) + } + + e3, err := Create(priv, []byte("baz"), 3, ts.Add(time.Hour), 0) + if err != nil { + t.Fatal(err) + } + + e4, err := Create(priv, []byte("cat"), 3, ts.Add(time.Hour*2), 0) + if err != nil { + t.Fatal(err) + } + + e5, err := Create(priv, []byte("dog"), 4, ts.Add(time.Hour*3), 0) + if err != nil { + t.Fatal(err) + } + + e6, err := Create(priv, []byte("fish"), 4, ts.Add(time.Hour*3), 0) + if err != nil { + t.Fatal(err) + } + + // e1 is the only record, i hope it gets this right + err = AssertSelected(e1, e1) + if err != nil { + t.Fatal(err) + } + + // e2 has the highest sequence number + err = AssertSelected(e2, e1, e2) + if err != nil { + t.Fatal(err) + } + + // e3 has the highest sequence number + err = AssertSelected(e3, e1, e2, e3) + if err != nil { + t.Fatal(err) + } + + // e4 has a higher timeout + err = AssertSelected(e4, e1, e2, e3, e4) + if err != nil { + t.Fatal(err) + } + + // e5 has the highest sequence number + err = AssertSelected(e5, e1, e2, e3, e4, e5) + if err != nil { + t.Fatal(err) + } + + // e6 should be selected as its signauture will win in the comparison + err = AssertSelected(e6, e1, e2, e3, e4, e5, e6) + if err != nil { + t.Fatal(err) + } + + _ = []interface{}{e1, e2, e3, e4, e5, e6} +} diff --git a/ipns/validate_test.go b/ipns/validate_test.go new file mode 100644 index 0000000000..0db7229512 --- /dev/null +++ b/ipns/validate_test.go @@ -0,0 +1,415 @@ +package ipns + +import ( + "bytes" + "errors" + "fmt" + "math/rand" + "strings" + "testing" + "time" + + "github.com/gogo/protobuf/proto" + pb "github.com/ipfs/go-libipfs/ipns/pb" + u "github.com/ipfs/go-libipfs/util" + ipldcodec "github.com/ipld/go-ipld-prime/multicodec" + basicnode "github.com/ipld/go-ipld-prime/node/basic" + "github.com/libp2p/go-libp2p/core/crypto" + "github.com/libp2p/go-libp2p/core/peer" + pstore "github.com/libp2p/go-libp2p/core/peerstore" + "github.com/libp2p/go-libp2p/p2p/host/peerstore/pstoremem" + "github.com/multiformats/go-multicodec" +) + +func testValidatorCase(t *testing.T, priv crypto.PrivKey, kbook pstore.KeyBook, key string, val []byte, eol time.Time, exp error) { + t.Helper() + + match := func(t *testing.T, err error) { + t.Helper() + if err != exp { + params := fmt.Sprintf("key: %s\neol: %s\n", key, eol) + if exp == nil { + t.Fatalf("Unexpected error %s for params %s", err, params) + } else if err == nil { + t.Fatalf("Expected error %s but there was no error for params %s", exp, params) + } else { + t.Fatalf("Expected error %s but got %s for params %s", exp, err, params) + } + } + } + + testValidatorCaseMatchFunc(t, priv, kbook, key, val, eol, match) +} + +func testValidatorCaseMatchFunc(t *testing.T, priv crypto.PrivKey, kbook pstore.KeyBook, key string, val []byte, eol time.Time, matchf func(*testing.T, error)) { + t.Helper() + validator := Validator{kbook} + + data := val + if data == nil { + p := []byte("/ipfs/QmfM2r8seH2GiRaC4esTjeraXEachRt8ZsSeGaWTPLyMoG") + entry, err := Create(priv, p, 1, eol, 0) + if err != nil { + t.Fatal(err) + } + + data, err = proto.Marshal(entry) + if err != nil { + t.Fatal(err) + } + } + + matchf(t, validator.Validate(key, data)) +} + +func TestValidator(t *testing.T) { + ts := time.Now() + + priv, id, _ := genKeys(t) + priv2, id2, _ := genKeys(t) + kbook, err := pstoremem.NewPeerstore() + if err != nil { + t.Fatal(err) + } + if err := kbook.AddPubKey(id, priv.GetPublic()); err != nil { + t.Fatal(err) + } + emptyKbook, err := pstoremem.NewPeerstore() + if err != nil { + t.Fatal(err) + } + + testValidatorCase(t, priv, kbook, "/ipns/"+string(id), nil, ts.Add(time.Hour), nil) + testValidatorCase(t, priv, kbook, "/ipns/"+string(id), nil, ts.Add(time.Hour*-1), ErrExpiredRecord) + testValidatorCase(t, priv, kbook, "/ipns/"+string(id), []byte("bad data"), ts.Add(time.Hour), ErrBadRecord) + testValidatorCase(t, priv, kbook, "/ipns/"+"bad key", nil, ts.Add(time.Hour), ErrKeyFormat) + testValidatorCase(t, priv, emptyKbook, "/ipns/"+string(id), nil, ts.Add(time.Hour), ErrPublicKeyNotFound) + testValidatorCase(t, priv2, kbook, "/ipns/"+string(id2), nil, ts.Add(time.Hour), ErrPublicKeyNotFound) + testValidatorCase(t, priv2, kbook, "/ipns/"+string(id), nil, ts.Add(time.Hour), ErrSignature) + testValidatorCase(t, priv, kbook, "//"+string(id), nil, ts.Add(time.Hour), ErrInvalidPath) + testValidatorCase(t, priv, kbook, "/wrong/"+string(id), nil, ts.Add(time.Hour), ErrInvalidPath) +} + +func mustMarshal(t *testing.T, entry *pb.IpnsEntry) []byte { + t.Helper() + data, err := proto.Marshal(entry) + if err != nil { + t.Fatal(err) + } + return data +} + +func TestEmbeddedPubKeyValidate(t *testing.T) { + goodeol := time.Now().Add(time.Hour) + kbook, err := pstoremem.NewPeerstore() + if err != nil { + t.Fatal(err) + } + + pth := []byte("/ipfs/QmfM2r8seH2GiRaC4esTjeraXEachRt8ZsSeGaWTPLyMoG") + + priv, _, ipnsk := genKeys(t) + + entry, err := Create(priv, pth, 1, goodeol, 0) + if err != nil { + t.Fatal(err) + } + + testValidatorCase(t, priv, kbook, ipnsk, mustMarshal(t, entry), goodeol, ErrPublicKeyNotFound) + + pubkb, err := crypto.MarshalPublicKey(priv.GetPublic()) + if err != nil { + t.Fatal(err) + } + + entry.PubKey = pubkb + testValidatorCase(t, priv, kbook, ipnsk, mustMarshal(t, entry), goodeol, nil) + + entry.PubKey = []byte("probably not a public key") + testValidatorCaseMatchFunc(t, priv, kbook, ipnsk, mustMarshal(t, entry), goodeol, func(t *testing.T, err error) { + if !strings.Contains(err.Error(), "unmarshaling pubkey in record:") { + t.Fatal("expected pubkey unmarshaling to fail") + } + }) + + opriv, _, _ := genKeys(t) + wrongkeydata, err := crypto.MarshalPublicKey(opriv.GetPublic()) + if err != nil { + t.Fatal(err) + } + + entry.PubKey = wrongkeydata + testValidatorCase(t, priv, kbook, ipnsk, mustMarshal(t, entry), goodeol, ErrPublicKeyMismatch) +} + +func TestPeerIDPubKeyValidate(t *testing.T) { + t.Skip("disabled until libp2p/go-libp2p-crypto#51 is fixed") + + goodeol := time.Now().Add(time.Hour) + kbook, err := pstoremem.NewPeerstore() + if err != nil { + t.Fatal(err) + } + + pth := []byte("/ipfs/QmfM2r8seH2GiRaC4esTjeraXEachRt8ZsSeGaWTPLyMoG") + + sk, pk, err := crypto.GenerateEd25519Key(rand.New(rand.NewSource(42))) + if err != nil { + t.Fatal(err) + } + + pid, err := peer.IDFromPublicKey(pk) + if err != nil { + t.Fatal(err) + } + + ipnsk := "/ipns/" + string(pid) + + entry, err := Create(sk, pth, 1, goodeol, 0) + if err != nil { + t.Fatal(err) + } + + dataNoKey, err := proto.Marshal(entry) + if err != nil { + t.Fatal(err) + } + + testValidatorCase(t, sk, kbook, ipnsk, dataNoKey, goodeol, nil) +} + +func TestOnlySignatureV2Validate(t *testing.T) { + goodeol := time.Now().Add(time.Hour) + + sk, pk, err := crypto.GenerateEd25519Key(rand.New(rand.NewSource(42))) + if err != nil { + t.Fatal(err) + } + + path1 := []byte("/path/1") + entry, err := Create(sk, path1, 1, goodeol, 0) + if err != nil { + t.Fatal(err) + } + + if err := Validate(pk, entry); err != nil { + t.Fatal(err) + } + + entry.SignatureV2 = nil + if err := Validate(pk, entry); !errors.Is(err, ErrSignature) { + t.Fatal(err) + } +} + +func TestSignatureV1Ignored(t *testing.T) { + goodeol := time.Now().Add(time.Hour) + + sk, pk, err := crypto.GenerateEd25519Key(rand.New(rand.NewSource(42))) + if err != nil { + t.Fatal(err) + } + + pid, err := peer.IDFromPublicKey(pk) + if err != nil { + t.Fatal(err) + } + + ipnsk := "/ipns/" + string(pid) + + path1 := []byte("/path/1") + entry1, err := Create(sk, path1, 1, goodeol, 0) + if err != nil { + t.Fatal(err) + } + + path2 := []byte("/path/2") + entry2, err := Create(sk, path2, 2, goodeol, 0) + if err != nil { + t.Fatal(err) + } + + if err := Validate(pk, entry1); err != nil { + t.Fatal(err) + } + + if err := Validate(pk, entry2); err != nil { + t.Fatal(err) + } + + v := Validator{} + best, err := v.Select(ipnsk, [][]byte{mustMarshal(t, entry1), mustMarshal(t, entry2)}) + if err != nil { + t.Fatal(err) + } + if best != 1 { + t.Fatal("entry2 should be better than entry1") + } + + // Having only the v1 signature should be invalid + entry2.SignatureV2 = nil + if err := Validate(pk, entry2); !errors.Is(err, ErrSignature) { + t.Fatal(err) + } + + // Record with v2 signature should always be preferred + best, err = v.Select(ipnsk, [][]byte{mustMarshal(t, entry1), mustMarshal(t, entry2)}) + if err != nil { + t.Fatal(err) + } + if best != 0 { + t.Fatal("entry1 should be better than entry2") + } + + // Having a missing v1 signature is acceptable as long as there is a valid v2 signature + entry1.SignatureV1 = nil + if err := Validate(pk, entry1); err != nil { + t.Fatal(err) + } + + // Having an invalid v1 signature is acceptable as long as there is a valid v2 signature + entry1.SignatureV1 = []byte("garbage") + if err := Validate(pk, entry1); err != nil { + t.Fatal(err) + } +} + +func TestMaxSizeValidate(t *testing.T) { + goodeol := time.Now().Add(time.Hour) + + sk, pk, err := crypto.GenerateEd25519Key(rand.New(rand.NewSource(42))) + if err != nil { + t.Fatal(err) + } + + // Create record over the max size (value+other fields) + value := make([]byte, MaxRecordSize) + entry, err := Create(sk, value, 1, goodeol, 0) + if err != nil { + t.Fatal(err) + } + // Must fail with ErrRecordSize + if err := Validate(pk, entry); !errors.Is(err, ErrRecordSize) { + t.Fatal(err) + } +} + +func TestCborDataCanonicalization(t *testing.T) { + goodeol := time.Now().Add(time.Hour) + + sk, pk, err := crypto.GenerateEd25519Key(rand.New(rand.NewSource(42))) + if err != nil { + t.Fatal(err) + } + + path := append([]byte("/path/1"), 0x00) + seqnum := uint64(1) + entry, err := Create(sk, path, seqnum, goodeol, time.Hour) + if err != nil { + t.Fatal(err) + } + + if err := Validate(pk, entry); err != nil { + t.Fatal(err) + } + + dec, err := ipldcodec.LookupDecoder(uint64(multicodec.DagCbor)) + if err != nil { + t.Fatal(err) + } + + ndbuilder := basicnode.Prototype__Map{}.NewBuilder() + if err := dec(ndbuilder, bytes.NewReader(entry.GetData())); err != nil { + t.Fatal(err) + } + + nd := ndbuilder.Build() + iter := nd.MapIterator() + var fields []string + for !iter.Done() { + k, v, err := iter.Next() + if err != nil { + t.Fatal(err) + } + kstr, err := k.AsString() + if err != nil { + t.Fatal(err) + } + + switch kstr { + case value: + b, err := v.AsBytes() + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(path, b) { + t.Fatal("value did not match") + } + case sequence: + s, err := v.AsInt() + if err != nil { + t.Fatal(err) + } + if uint64(s) != seqnum { + t.Fatal("sequence numbers did not match") + } + case validity: + val, err := v.AsBytes() + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(val, []byte(u.FormatRFC3339(goodeol))) { + t.Fatal("validity did not match") + } + case validityType: + vt, err := v.AsInt() + if err != nil { + t.Fatal(err) + } + if uint64(vt) != 0 { + t.Fatal("validity types did not match") + } + case ttl: + ttlVal, err := v.AsInt() + if err != nil { + t.Fatal(err) + } + // TODO: test non-zero TTL + if uint64(ttlVal) != uint64(time.Hour.Nanoseconds()) { + t.Fatal("TTLs did not match") + } + } + + fields = append(fields, kstr) + } + + // Check for map sort order (i.e. by length then by value) + expectedOrder := []string{"TTL", "Value", "Sequence", "Validity", "ValidityType"} + if len(fields) != len(expectedOrder) { + t.Fatal("wrong number of fields") + } + + for i, f := range fields { + expected := expectedOrder[i] + if f != expected { + t.Fatalf("expected %s, got %s", expected, f) + } + } +} + +func genKeys(t *testing.T) (crypto.PrivKey, peer.ID, string) { + sr := u.NewTimeSeededRand() + priv, _, err := crypto.GenerateKeyPairWithReader(crypto.RSA, 2048, sr) + if err != nil { + t.Fatal(err) + } + + // Create entry with expiry in one hour + pid, err := peer.IDFromPrivateKey(priv) + if err != nil { + t.Fatal(err) + } + ipnsKey := RecordKey(pid) + + return priv, pid, ipnsKey +} diff --git a/keystore/keystore.go b/keystore/keystore.go new file mode 100644 index 0000000000..fc6793a1ee --- /dev/null +++ b/keystore/keystore.go @@ -0,0 +1,188 @@ +package keystore + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "encoding/base32" + + logging "github.com/ipfs/go-log" + ci "github.com/libp2p/go-libp2p/core/crypto" +) + +var log = logging.Logger("keystore") + +var codec = base32.StdEncoding.WithPadding(base32.NoPadding) + +// Keystore provides a key management interface +type Keystore interface { + // Has returns whether or not a key exists in the Keystore + Has(string) (bool, error) + // Put stores a key in the Keystore, if a key with the same name already exists, returns ErrKeyExists + Put(string, ci.PrivKey) error + // Get retrieves a key from the Keystore if it exists, and returns ErrNoSuchKey + // otherwise. + Get(string) (ci.PrivKey, error) + // Delete removes a key from the Keystore + Delete(string) error + // List returns a list of key identifier + List() ([]string, error) +} + +// ErrNoSuchKey is an error message returned when no key of a given name was found. +var ErrNoSuchKey = fmt.Errorf("no key by the given name was found") + +// ErrKeyExists is an error message returned when a key already exists +var ErrKeyExists = fmt.Errorf("key by that name already exists, refusing to overwrite") + +const keyFilenamePrefix = "key_" + +// FSKeystore is a keystore backed by files in a given directory stored on disk. +type FSKeystore struct { + dir string +} + +// NewFSKeystore returns a new filesystem-backed keystore. +func NewFSKeystore(dir string) (*FSKeystore, error) { + err := os.Mkdir(dir, 0700) + switch { + case os.IsExist(err): + case err == nil: + default: + return nil, err + } + return &FSKeystore{dir}, nil +} + +// Has returns whether or not a key exists in the Keystore +func (ks *FSKeystore) Has(name string) (bool, error) { + name, err := encode(name) + if err != nil { + return false, err + } + + kp := filepath.Join(ks.dir, name) + + _, err = os.Stat(kp) + + if os.IsNotExist(err) { + return false, nil + } + return err == nil, err +} + +// Put stores a key in the Keystore, if a key with the same name already exists, returns ErrKeyExists +func (ks *FSKeystore) Put(name string, k ci.PrivKey) error { + name, err := encode(name) + if err != nil { + return err + } + + b, err := ci.MarshalPrivateKey(k) + if err != nil { + return err + } + + kp := filepath.Join(ks.dir, name) + + fi, err := os.OpenFile(kp, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0400) + if err != nil { + if os.IsExist(err) { + err = ErrKeyExists + } + return err + } + defer fi.Close() + + _, err = fi.Write(b) + + return err +} + +// Get retrieves a key from the Keystore if it exists, and returns ErrNoSuchKey +// otherwise. +func (ks *FSKeystore) Get(name string) (ci.PrivKey, error) { + name, err := encode(name) + if err != nil { + return nil, err + } + + kp := filepath.Join(ks.dir, name) + + data, err := os.ReadFile(kp) + if err != nil { + if os.IsNotExist(err) { + return nil, ErrNoSuchKey + } + return nil, err + } + + return ci.UnmarshalPrivateKey(data) +} + +// Delete removes a key from the Keystore +func (ks *FSKeystore) Delete(name string) error { + name, err := encode(name) + if err != nil { + return err + } + + kp := filepath.Join(ks.dir, name) + + return os.Remove(kp) +} + +// List return a list of key identifier +func (ks *FSKeystore) List() ([]string, error) { + dir, err := os.Open(ks.dir) + if err != nil { + return nil, err + } + + dirs, err := dir.Readdirnames(0) + if err != nil { + return nil, err + } + + list := make([]string, 0, len(dirs)) + + for _, name := range dirs { + decodedName, err := decode(name) + if err == nil { + list = append(list, decodedName) + } else { + log.Errorf("Ignoring keyfile with invalid encoded filename: %s", name) + } + } + + return list, nil +} + +func encode(name string) (string, error) { + if name == "" { + return "", fmt.Errorf("key name must be at least one character") + } + + encodedName := codec.EncodeToString([]byte(name)) + log.Debugf("Encoded key name: %s to: %s", name, encodedName) + + return keyFilenamePrefix + strings.ToLower(encodedName), nil +} + +func decode(name string) (string, error) { + if !strings.HasPrefix(name, keyFilenamePrefix) { + return "", fmt.Errorf("key's filename has unexpected format") + } + + nameWithoutPrefix := strings.ToUpper(name[len(keyFilenamePrefix):]) + decodedName, err := codec.DecodeString(nameWithoutPrefix) + if err != nil { + return "", err + } + + log.Debugf("Decoded key name: %s to: %s", name, decodedName) + + return string(decodedName), nil +} diff --git a/keystore/keystore_test.go b/keystore/keystore_test.go new file mode 100644 index 0000000000..9a44062173 --- /dev/null +++ b/keystore/keystore_test.go @@ -0,0 +1,277 @@ +package keystore + +import ( + "fmt" + "math/rand" + "os" + "path/filepath" + "sort" + "testing" + + ci "github.com/libp2p/go-libp2p/core/crypto" +) + +type rr struct{} + +func (rr rr) Read(b []byte) (int, error) { + return rand.Read(b) +} + +func privKeyOrFatal(t *testing.T) ci.PrivKey { + priv, _, err := ci.GenerateEd25519Key(rr{}) + if err != nil { + t.Fatal(err) + } + return priv +} + +func TestKeystoreBasics(t *testing.T) { + tdir, err := os.MkdirTemp("", "keystore-test") + if err != nil { + t.Fatal(err) + } + + ks, err := NewFSKeystore(tdir) + if err != nil { + t.Fatal(err) + } + + l, err := ks.List() + if err != nil { + t.Fatal(err) + } + + if len(l) != 0 { + t.Fatal("expected no keys") + } + + k1 := privKeyOrFatal(t) + k2 := privKeyOrFatal(t) + k3 := privKeyOrFatal(t) + k4 := privKeyOrFatal(t) + + err = ks.Put("foo", k1) + if err != nil { + t.Fatal(err) + } + + err = ks.Put("bar", k2) + if err != nil { + t.Fatal(err) + } + + l, err = ks.List() + if err != nil { + t.Fatal(err) + } + + sort.Strings(l) + if l[0] != "bar" || l[1] != "foo" { + t.Fatal("wrong entries listed") + } + + if err := assertDirContents(tdir, []string{"foo", "bar"}); err != nil { + t.Fatal(err) + } + + err = ks.Put("foo", k3) + if err == nil { + t.Fatal("should not be able to overwrite key") + } + + if err := assertDirContents(tdir, []string{"foo", "bar"}); err != nil { + t.Fatal(err) + } + + exist, err := ks.Has("foo") + if !exist { + t.Fatal("should know it has a key named foo") + } + if err != nil { + t.Fatal(err) + } + + exist, err = ks.Has("nonexistingkey") + if exist { + t.Fatal("should know it doesn't have a key named nonexistingkey") + } + if err != nil { + t.Fatal(err) + } + + if err := ks.Delete("bar"); err != nil { + t.Fatal(err) + } + + if err := assertDirContents(tdir, []string{"foo"}); err != nil { + t.Fatal(err) + } + + if err := ks.Put("beep", k3); err != nil { + t.Fatal(err) + } + + if err := ks.Put("boop", k4); err != nil { + t.Fatal(err) + } + + if err := assertDirContents(tdir, []string{"foo", "beep", "boop"}); err != nil { + t.Fatal(err) + } + + if err := assertGetKey(ks, "foo", k1); err != nil { + t.Fatal(err) + } + + if err := assertGetKey(ks, "beep", k3); err != nil { + t.Fatal(err) + } + + if err := assertGetKey(ks, "boop", k4); err != nil { + t.Fatal(err) + } + + if err := ks.Put("..///foo/", k1); err != nil { + t.Fatal(err) + } + + if err := ks.Put("", k1); err == nil { + t.Fatal("shouldn't be able to put a key with no name") + } + + if err := ks.Put(".foo", k1); err != nil { + t.Fatal(err) + } +} + +func TestInvalidKeyFiles(t *testing.T) { + tdir, err := os.MkdirTemp("", "keystore-test") + + if err != nil { + t.Fatal(err) + } + + defer os.RemoveAll(tdir) + + ks, err := NewFSKeystore(tdir) + if err != nil { + t.Fatal(err) + } + + key := privKeyOrFatal(t) + + bytes, err := key.Raw() + if err != nil { + t.Fatal(err) + } + + encodedName, err := encode("valid") + if err != nil { + t.Fatal(err) + } + + err = os.WriteFile(filepath.Join(ks.dir, encodedName), bytes, 0644) + if err != nil { + t.Fatal(err) + } + + err = os.WriteFile(filepath.Join(ks.dir, "z.invalid"), bytes, 0644) + if err != nil { + t.Fatal(err) + } + + l, err := ks.List() + if err != nil { + t.Fatal(err) + } + + sort.Strings(l) + if len(l) != 1 { + t.Fatal("wrong entry count") + } + + if l[0] != "valid" { + t.Fatal("wrong entries listed") + } + + exist, err := ks.Has("valid") + if !exist { + t.Fatal("should know it has a key named valid") + } + if err != nil { + t.Fatal(err) + } +} + +func TestNonExistingKey(t *testing.T) { + tdir, err := os.MkdirTemp("", "keystore-test") + if err != nil { + t.Fatal(err) + } + + ks, err := NewFSKeystore(tdir) + if err != nil { + t.Fatal(err) + } + + k, err := ks.Get("does-it-exist") + if err != ErrNoSuchKey { + t.Fatalf("expected: %s, got %s", ErrNoSuchKey, err) + } + if k != nil { + t.Fatalf("Get on nonexistant key should give nil") + } +} + +func TestMakeKeystoreNoDir(t *testing.T) { + _, err := NewFSKeystore("/this/is/not/a/real/dir") + if err == nil { + t.Fatal("shouldnt be able to make a keystore in a nonexistant directory") + } +} + +func assertGetKey(ks Keystore, name string, exp ci.PrivKey) error { + outK, err := ks.Get(name) + if err != nil { + return err + } + + if !outK.Equals(exp) { + return fmt.Errorf("key we got out didn't match expectation") + } + + return nil +} + +func assertDirContents(dir string, exp []string) error { + finfos, err := os.ReadDir(dir) + if err != nil { + return err + } + + if len(finfos) != len(exp) { + return fmt.Errorf("expected %d directory entries", len(exp)) + } + + var names []string + for _, fi := range finfos { + decodedName, err := decode(fi.Name()) + if err != nil { + return err + } + names = append(names, decodedName) + } + + sort.Strings(names) + sort.Strings(exp) + if len(names) != len(exp) { + return fmt.Errorf("directory had wrong number of entries in it") + } + + for i, v := range names { + if v != exp[i] { + return fmt.Errorf("had wrong entry in directory") + } + } + return nil +} diff --git a/keystore/memkeystore.go b/keystore/memkeystore.go new file mode 100644 index 0000000000..0ea62f4e1a --- /dev/null +++ b/keystore/memkeystore.go @@ -0,0 +1,64 @@ +package keystore + +import ( + "errors" + + ci "github.com/libp2p/go-libp2p/core/crypto" +) + +// MemKeystore is an in memory keystore implementation that is not persisted to +// any backing storage. +type MemKeystore struct { + keys map[string]ci.PrivKey +} + +// NewMemKeystore creates a MemKeystore. +func NewMemKeystore() *MemKeystore { + return &MemKeystore{make(map[string]ci.PrivKey)} +} + +// Has return whether or not a key exists in the Keystore +func (mk *MemKeystore) Has(name string) (bool, error) { + _, ok := mk.keys[name] + return ok, nil +} + +// Put store a key in the Keystore +func (mk *MemKeystore) Put(name string, k ci.PrivKey) error { + if name == "" { + return errors.New("key name must be at least one character") + } + + _, ok := mk.keys[name] + if ok { + return ErrKeyExists + } + + mk.keys[name] = k + return nil +} + +// Get retrieve a key from the Keystore +func (mk *MemKeystore) Get(name string) (ci.PrivKey, error) { + k, ok := mk.keys[name] + if !ok { + return nil, ErrNoSuchKey + } + + return k, nil +} + +// Delete remove a key from the Keystore +func (mk *MemKeystore) Delete(name string) error { + delete(mk.keys, name) + return nil +} + +// List return a list of key identifier +func (mk *MemKeystore) List() ([]string, error) { + out := make([]string, 0, len(mk.keys)) + for k := range mk.keys { + out = append(out, k) + } + return out, nil +} diff --git a/keystore/memkeystore_test.go b/keystore/memkeystore_test.go new file mode 100644 index 0000000000..907cbbd0e7 --- /dev/null +++ b/keystore/memkeystore_test.go @@ -0,0 +1,99 @@ +package keystore + +import ( + "sort" + "testing" +) + +func TestMemKeyStoreBasics(t *testing.T) { + ks := NewMemKeystore() + + l, err := ks.List() + if err != nil { + t.Fatal(err) + } + + if len(l) != 0 { + t.Fatal("expected no keys") + } + + k1 := privKeyOrFatal(t) + k2 := privKeyOrFatal(t) + k3 := privKeyOrFatal(t) + k4 := privKeyOrFatal(t) + + err = ks.Put("foo", k1) + if err != nil { + t.Fatal(err) + } + + err = ks.Put("bar", k2) + if err != nil { + t.Fatal(err) + } + + l, err = ks.List() + if err != nil { + t.Fatal(err) + } + + sort.Strings(l) + if l[0] != "bar" || l[1] != "foo" { + t.Fatal("wrong entries listed") + } + + err = ks.Put("foo", k3) + if err == nil { + t.Fatal("should not be able to overwrite key") + } + + exist, err := ks.Has("foo") + if !exist { + t.Fatal("should know it has a key named foo") + } + if err != nil { + t.Fatal(err) + } + + exist, err = ks.Has("nonexistingkey") + if exist { + t.Fatal("should know it doesn't have a key named nonexistingkey") + } + if err != nil { + t.Fatal(err) + } + + if err := ks.Delete("bar"); err != nil { + t.Fatal(err) + } + if err := ks.Put("beep", k3); err != nil { + t.Fatal(err) + } + + if err := ks.Put("boop", k4); err != nil { + t.Fatal(err) + } + if err := assertGetKey(ks, "foo", k1); err != nil { + t.Fatal(err) + } + + if err := assertGetKey(ks, "beep", k3); err != nil { + t.Fatal(err) + } + + if err := assertGetKey(ks, "boop", k4); err != nil { + t.Fatal(err) + } + + if err := ks.Put("..///foo/", k1); err != nil { + t.Fatal(err) + } + + if err := ks.Put("", k1); err == nil { + t.Fatal("shouldn't be able to put a key with no name") + } + + if err := ks.Put(".foo", k1); err != nil { + t.Fatal(err) + } +} diff --git a/mfs/dir.go b/mfs/dir.go new file mode 100644 index 0000000000..5295b3e698 --- /dev/null +++ b/mfs/dir.go @@ -0,0 +1,427 @@ +package mfs + +import ( + "context" + "errors" + "fmt" + "os" + "path" + "sync" + "time" + + ft "github.com/ipfs/go-libipfs/unixfs" + uio "github.com/ipfs/go-libipfs/unixfs/io" + dag "github.com/ipfs/go-merkledag" + + cid "github.com/ipfs/go-cid" + ipld "github.com/ipfs/go-ipld-format" +) + +var ErrNotYetImplemented = errors.New("not yet implemented") +var ErrInvalidChild = errors.New("invalid child node") +var ErrDirExists = errors.New("directory already has entry by that name") + +// TODO: There's too much functionality associated with this structure, +// let's organize it (and if possible extract part of it elsewhere) +// and document the main features of `Directory` here. +type Directory struct { + inode + + // Internal cache with added entries to the directory, its cotents + // are synched with the underlying `unixfsDir` node in `sync()`. + entriesCache map[string]FSNode + + lock sync.Mutex + // TODO: What content is being protected here exactly? The entire directory? + + ctx context.Context + + // UnixFS directory implementation used for creating, + // reading and editing directories. + unixfsDir uio.Directory + + modTime time.Time +} + +// NewDirectory constructs a new MFS directory. +// +// You probably don't want to call this directly. Instead, construct a new root +// using NewRoot. +func NewDirectory(ctx context.Context, name string, node ipld.Node, parent parent, dserv ipld.DAGService) (*Directory, error) { + db, err := uio.NewDirectoryFromNode(dserv, node) + if err != nil { + return nil, err + } + + return &Directory{ + inode: inode{ + name: name, + parent: parent, + dagService: dserv, + }, + ctx: ctx, + unixfsDir: db, + entriesCache: make(map[string]FSNode), + modTime: time.Now(), + }, nil +} + +// GetCidBuilder gets the CID builder of the root node +func (d *Directory) GetCidBuilder() cid.Builder { + return d.unixfsDir.GetCidBuilder() +} + +// SetCidBuilder sets the CID builder +func (d *Directory) SetCidBuilder(b cid.Builder) { + d.unixfsDir.SetCidBuilder(b) +} + +// This method implements the `parent` interface. It first does the local +// update of the child entry in the underlying UnixFS directory and saves +// the newly created directory node with the updated entry in the DAG +// service. Then it propagates the update upwards (through this same +// interface) repeating the whole process in the parent. +func (d *Directory) updateChildEntry(c child) error { + newDirNode, err := d.localUpdate(c) + if err != nil { + return err + } + + // Continue to propagate the update process upwards + // (all the way up to the root). + return d.parent.updateChildEntry(child{d.name, newDirNode}) +} + +// This method implements the part of `updateChildEntry` that needs +// to be locked around: in charge of updating the UnixFS layer and +// generating the new node reflecting the update. It also stores the +// new node in the DAG layer. +func (d *Directory) localUpdate(c child) (*dag.ProtoNode, error) { + d.lock.Lock() + defer d.lock.Unlock() + + err := d.updateChild(c) + if err != nil { + return nil, err + } + // TODO: Clearly define how are we propagating changes to lower layers + // like UnixFS. + + nd, err := d.unixfsDir.GetNode() + if err != nil { + return nil, err + } + + pbnd, ok := nd.(*dag.ProtoNode) + if !ok { + return nil, dag.ErrNotProtobuf + } + + err = d.dagService.Add(d.ctx, nd) + if err != nil { + return nil, err + } + + return pbnd.Copy().(*dag.ProtoNode), nil + // TODO: Why do we need a copy? +} + +// Update child entry in the underlying UnixFS directory. +func (d *Directory) updateChild(c child) error { + err := d.unixfsDir.AddChild(d.ctx, c.Name, c.Node) + if err != nil { + return err + } + + d.modTime = time.Now() + + return nil +} + +func (d *Directory) Type() NodeType { + return TDir +} + +// childNode returns a FSNode under this directory by the given name if it exists. +// it does *not* check the cached dirs and files +func (d *Directory) childNode(name string) (FSNode, error) { + nd, err := d.childFromDag(name) + if err != nil { + return nil, err + } + + return d.cacheNode(name, nd) +} + +// cacheNode caches a node into d.childDirs or d.files and returns the FSNode. +func (d *Directory) cacheNode(name string, nd ipld.Node) (FSNode, error) { + switch nd := nd.(type) { + case *dag.ProtoNode: + fsn, err := ft.FSNodeFromBytes(nd.Data()) + if err != nil { + return nil, err + } + + switch fsn.Type() { + case ft.TDirectory, ft.THAMTShard: + ndir, err := NewDirectory(d.ctx, name, nd, d, d.dagService) + if err != nil { + return nil, err + } + + d.entriesCache[name] = ndir + return ndir, nil + case ft.TFile, ft.TRaw, ft.TSymlink: + nfi, err := NewFile(name, nd, d, d.dagService) + if err != nil { + return nil, err + } + d.entriesCache[name] = nfi + return nfi, nil + case ft.TMetadata: + return nil, ErrNotYetImplemented + default: + return nil, ErrInvalidChild + } + case *dag.RawNode: + nfi, err := NewFile(name, nd, d, d.dagService) + if err != nil { + return nil, err + } + d.entriesCache[name] = nfi + return nfi, nil + default: + return nil, fmt.Errorf("unrecognized node type in cache node") + } +} + +// Child returns the child of this directory by the given name +func (d *Directory) Child(name string) (FSNode, error) { + d.lock.Lock() + defer d.lock.Unlock() + return d.childUnsync(name) +} + +func (d *Directory) Uncache(name string) { + d.lock.Lock() + defer d.lock.Unlock() + delete(d.entriesCache, name) +} + +// childFromDag searches through this directories dag node for a child link +// with the given name +func (d *Directory) childFromDag(name string) (ipld.Node, error) { + return d.unixfsDir.Find(d.ctx, name) +} + +// childUnsync returns the child under this directory by the given name +// without locking, useful for operations which already hold a lock +func (d *Directory) childUnsync(name string) (FSNode, error) { + entry, ok := d.entriesCache[name] + if ok { + return entry, nil + } + + return d.childNode(name) +} + +type NodeListing struct { + Name string + Type int + Size int64 + Hash string +} + +func (d *Directory) ListNames(ctx context.Context) ([]string, error) { + d.lock.Lock() + defer d.lock.Unlock() + + var out []string + err := d.unixfsDir.ForEachLink(ctx, func(l *ipld.Link) error { + out = append(out, l.Name) + return nil + }) + if err != nil { + return nil, err + } + + return out, nil +} + +func (d *Directory) List(ctx context.Context) ([]NodeListing, error) { + var out []NodeListing + err := d.ForEachEntry(ctx, func(nl NodeListing) error { + out = append(out, nl) + return nil + }) + return out, err +} + +func (d *Directory) ForEachEntry(ctx context.Context, f func(NodeListing) error) error { + d.lock.Lock() + defer d.lock.Unlock() + return d.unixfsDir.ForEachLink(ctx, func(l *ipld.Link) error { + c, err := d.childUnsync(l.Name) + if err != nil { + return err + } + + nd, err := c.GetNode() + if err != nil { + return err + } + + child := NodeListing{ + Name: l.Name, + Type: int(c.Type()), + Hash: nd.Cid().String(), + } + + if c, ok := c.(*File); ok { + size, err := c.Size() + if err != nil { + return err + } + child.Size = size + } + + return f(child) + }) +} + +func (d *Directory) Mkdir(name string) (*Directory, error) { + d.lock.Lock() + defer d.lock.Unlock() + + fsn, err := d.childUnsync(name) + if err == nil { + switch fsn := fsn.(type) { + case *Directory: + return fsn, os.ErrExist + case *File: + return nil, os.ErrExist + default: + return nil, fmt.Errorf("unrecognized type: %#v", fsn) + } + } + + ndir := ft.EmptyDirNode() + ndir.SetCidBuilder(d.GetCidBuilder()) + + err = d.dagService.Add(d.ctx, ndir) + if err != nil { + return nil, err + } + + err = d.unixfsDir.AddChild(d.ctx, name, ndir) + if err != nil { + return nil, err + } + + dirobj, err := NewDirectory(d.ctx, name, ndir, d, d.dagService) + if err != nil { + return nil, err + } + + d.entriesCache[name] = dirobj + return dirobj, nil +} + +func (d *Directory) Unlink(name string) error { + d.lock.Lock() + defer d.lock.Unlock() + + delete(d.entriesCache, name) + + return d.unixfsDir.RemoveChild(d.ctx, name) +} + +func (d *Directory) Flush() error { + nd, err := d.GetNode() + if err != nil { + return err + } + + return d.parent.updateChildEntry(child{d.name, nd}) +} + +// AddChild adds the node 'nd' under this directory giving it the name 'name' +func (d *Directory) AddChild(name string, nd ipld.Node) error { + d.lock.Lock() + defer d.lock.Unlock() + + _, err := d.childUnsync(name) + if err == nil { + return ErrDirExists + } + + err = d.dagService.Add(d.ctx, nd) + if err != nil { + return err + } + + err = d.unixfsDir.AddChild(d.ctx, name, nd) + if err != nil { + return err + } + + d.modTime = time.Now() + return nil +} + +func (d *Directory) sync() error { + for name, entry := range d.entriesCache { + nd, err := entry.GetNode() + if err != nil { + return err + } + + err = d.updateChild(child{name, nd}) + if err != nil { + return err + } + } + + // TODO: Should we clean the cache here? + + return nil +} + +func (d *Directory) Path() string { + cur := d + var out string + for cur != nil { + switch parent := cur.parent.(type) { + case *Directory: + out = path.Join(cur.name, out) + cur = parent + case *Root: + return "/" + out + default: + panic("directory parent neither a directory nor a root") + } + } + return out +} + +func (d *Directory) GetNode() (ipld.Node, error) { + d.lock.Lock() + defer d.lock.Unlock() + + err := d.sync() + if err != nil { + return nil, err + } + + nd, err := d.unixfsDir.GetNode() + if err != nil { + return nil, err + } + + err = d.dagService.Add(d.ctx, nd) + if err != nil { + return nil, err + } + + return nd.Copy(), err +} diff --git a/mfs/fd.go b/mfs/fd.go new file mode 100644 index 0000000000..1bceb9ad32 --- /dev/null +++ b/mfs/fd.go @@ -0,0 +1,197 @@ +package mfs + +import ( + "fmt" + "io" + + mod "github.com/ipfs/go-libipfs/unixfs/mod" + + context "context" + + ipld "github.com/ipfs/go-ipld-format" +) + +type state uint8 + +const ( + stateCreated state = iota + stateFlushed + stateDirty + stateClosed +) + +// One `File` can have many `FileDescriptor`s associated to it +// (only one if it's RW, many if they are RO, see `File.desclock`). +// A `FileDescriptor` contains the "view" of the file (through an +// instance of a `DagModifier`), that's why it (and not the `File`) +// has the responsibility to `Flush` (which crystallizes that view +// in the `File`'s `Node`). +type FileDescriptor interface { + io.Reader + CtxReadFull(context.Context, []byte) (int, error) + + io.Writer + io.WriterAt + + io.Closer + io.Seeker + + Truncate(int64) error + Size() (int64, error) + Flush() error +} + +type fileDescriptor struct { + inode *File + mod *mod.DagModifier + flags Flags + + state state +} + +func (fi *fileDescriptor) checkWrite() error { + if fi.state == stateClosed { + return ErrClosed + } + if !fi.flags.Write { + return fmt.Errorf("file is read-only") + } + return nil +} + +func (fi *fileDescriptor) checkRead() error { + if fi.state == stateClosed { + return ErrClosed + } + if !fi.flags.Read { + return fmt.Errorf("file is write-only") + } + return nil +} + +// Size returns the size of the file referred to by this descriptor +func (fi *fileDescriptor) Size() (int64, error) { + return fi.mod.Size() +} + +// Truncate truncates the file to size +func (fi *fileDescriptor) Truncate(size int64) error { + if err := fi.checkWrite(); err != nil { + return fmt.Errorf("truncate failed: %s", err) + } + fi.state = stateDirty + return fi.mod.Truncate(size) +} + +// Write writes the given data to the file at its current offset +func (fi *fileDescriptor) Write(b []byte) (int, error) { + if err := fi.checkWrite(); err != nil { + return 0, fmt.Errorf("write failed: %s", err) + } + fi.state = stateDirty + return fi.mod.Write(b) +} + +// Read reads into the given buffer from the current offset +func (fi *fileDescriptor) Read(b []byte) (int, error) { + if err := fi.checkRead(); err != nil { + return 0, fmt.Errorf("read failed: %s", err) + } + return fi.mod.Read(b) +} + +// Read reads into the given buffer from the current offset +func (fi *fileDescriptor) CtxReadFull(ctx context.Context, b []byte) (int, error) { + if err := fi.checkRead(); err != nil { + return 0, fmt.Errorf("read failed: %s", err) + } + return fi.mod.CtxReadFull(ctx, b) +} + +// Close flushes, then propogates the modified dag node up the directory structure +// and signals a republish to occur +func (fi *fileDescriptor) Close() error { + if fi.state == stateClosed { + return ErrClosed + } + if fi.flags.Write { + defer fi.inode.desclock.Unlock() + } else if fi.flags.Read { + defer fi.inode.desclock.RUnlock() + } + err := fi.flushUp(fi.flags.Sync) + fi.state = stateClosed + return err +} + +// Flush generates a new version of the node of the underlying +// UnixFS directory (adding it to the DAG service) and updates +// the entry in the parent directory (setting `fullSync` to +// propagate the update all the way to the root). +func (fi *fileDescriptor) Flush() error { + return fi.flushUp(true) +} + +// flushUp syncs the file and adds it to the dagservice +// it *must* be called with the File's lock taken +// If `fullSync` is set the changes are propagated upwards +// (the `Up` part of `flushUp`). +func (fi *fileDescriptor) flushUp(fullSync bool) error { + var nd ipld.Node + switch fi.state { + case stateCreated, stateDirty: + var err error + nd, err = fi.mod.GetNode() + if err != nil { + return err + } + err = fi.inode.dagService.Add(context.TODO(), nd) + if err != nil { + return err + } + + // TODO: Very similar logic to the update process in + // `Directory`, the logic should be unified, both structures + // (`File` and `Directory`) are backed by a IPLD node with + // a UnixFS format that is the actual target of the update + // (regenerating it and adding it to the DAG service). + fi.inode.nodeLock.Lock() + // Always update the file descriptor's inode with the created/modified node. + fi.inode.node = nd + // Save the members to be used for subsequent calls + parent := fi.inode.parent + name := fi.inode.name + fi.inode.nodeLock.Unlock() + + // Bubble up the update's to the parent, only if fullSync is set to true. + if fullSync { + if err := parent.updateChildEntry(child{name, nd}); err != nil { + return err + } + } + + fi.state = stateFlushed + return nil + case stateFlushed: + return nil + default: + panic("invalid state") + } +} + +// Seek implements io.Seeker +func (fi *fileDescriptor) Seek(offset int64, whence int) (int64, error) { + if fi.state == stateClosed { + return 0, fmt.Errorf("seek failed: %s", ErrClosed) + } + return fi.mod.Seek(offset, whence) +} + +// Write At writes the given bytes at the offset 'at' +func (fi *fileDescriptor) WriteAt(b []byte, at int64) (int, error) { + if err := fi.checkWrite(); err != nil { + return 0, fmt.Errorf("write-at failed: %s", err) + } + fi.state = stateDirty + return fi.mod.WriteAt(b, at) +} diff --git a/mfs/file.go b/mfs/file.go new file mode 100644 index 0000000000..d3d1f4252a --- /dev/null +++ b/mfs/file.go @@ -0,0 +1,179 @@ +package mfs + +import ( + "context" + "fmt" + "sync" + + ft "github.com/ipfs/go-libipfs/unixfs" + mod "github.com/ipfs/go-libipfs/unixfs/mod" + dag "github.com/ipfs/go-merkledag" + + ipld "github.com/ipfs/go-ipld-format" + chunker "github.com/ipfs/go-libipfs/chunker" +) + +// File represents a file in the MFS, its logic its mainly targeted +// to coordinating (potentially many) `FileDescriptor`s pointing to +// it. +type File struct { + inode + + // Lock to coordinate the `FileDescriptor`s associated to this file. + desclock sync.RWMutex + + // This isn't any node, it's the root node that represents the + // entire DAG of nodes that comprise the file. + // TODO: Rename, there should be an explicit term for these root nodes + // of a particular sub-DAG that abstract an upper layer's entity. + node ipld.Node + + // Lock around the `node` that represents this file, necessary because + // there may be many `FileDescriptor`s operating on this `File`. + nodeLock sync.RWMutex + + RawLeaves bool +} + +// NewFile returns a NewFile object with the given parameters. If the +// Cid version is non-zero RawLeaves will be enabled. +func NewFile(name string, node ipld.Node, parent parent, dserv ipld.DAGService) (*File, error) { + fi := &File{ + inode: inode{ + name: name, + parent: parent, + dagService: dserv, + }, + node: node, + } + if node.Cid().Prefix().Version > 0 { + fi.RawLeaves = true + } + return fi, nil +} + +func (fi *File) Open(flags Flags) (_ FileDescriptor, _retErr error) { + if flags.Write { + fi.desclock.Lock() + defer func() { + if _retErr != nil { + fi.desclock.Unlock() + } + }() + } else if flags.Read { + fi.desclock.RLock() + defer func() { + if _retErr != nil { + fi.desclock.RUnlock() + } + }() + } else { + return nil, fmt.Errorf("file opened for neither reading nor writing") + } + + fi.nodeLock.RLock() + node := fi.node + fi.nodeLock.RUnlock() + + // TODO: Move this `switch` logic outside (maybe even + // to another package, this seems like a job of UnixFS), + // `NewDagModifier` uses the IPLD node, we're not + // extracting anything just doing a safety check. + switch node := node.(type) { + case *dag.ProtoNode: + fsn, err := ft.FSNodeFromBytes(node.Data()) + if err != nil { + return nil, err + } + + switch fsn.Type() { + default: + return nil, fmt.Errorf("unsupported fsnode type for 'file'") + case ft.TSymlink: + return nil, fmt.Errorf("symlinks not yet supported") + case ft.TFile, ft.TRaw: + // OK case + } + case *dag.RawNode: + // Ok as well. + } + + dmod, err := mod.NewDagModifier(context.TODO(), node, fi.dagService, chunker.DefaultSplitter) + // TODO: Remove the use of the `chunker` package here, add a new `NewDagModifier` in + // `go-unixfs` with the `DefaultSplitter` already included. + if err != nil { + return nil, err + } + dmod.RawLeaves = fi.RawLeaves + + return &fileDescriptor{ + inode: fi, + flags: flags, + mod: dmod, + state: stateCreated, + }, nil +} + +// Size returns the size of this file +// TODO: Should we be providing this API? +// TODO: There's already a `FileDescriptor.Size()` that +// through the `DagModifier`'s `fileSize` function is doing +// pretty much the same thing as here, we should at least call +// that function and wrap the `ErrNotUnixfs` with an MFS text. +func (fi *File) Size() (int64, error) { + fi.nodeLock.RLock() + defer fi.nodeLock.RUnlock() + switch nd := fi.node.(type) { + case *dag.ProtoNode: + fsn, err := ft.FSNodeFromBytes(nd.Data()) + if err != nil { + return 0, err + } + return int64(fsn.FileSize()), nil + case *dag.RawNode: + return int64(len(nd.RawData())), nil + default: + return 0, fmt.Errorf("unrecognized node type in mfs/file.Size()") + } +} + +// GetNode returns the dag node associated with this file +// TODO: Use this method and do not access the `nodeLock` directly anywhere else. +func (fi *File) GetNode() (ipld.Node, error) { + fi.nodeLock.RLock() + defer fi.nodeLock.RUnlock() + return fi.node, nil +} + +// TODO: Tight coupling with the `FileDescriptor`, at the +// very least this should be an independent function that +// takes a `File` argument and automates the open/flush/close +// operations. +// TODO: Why do we need to flush a file that isn't opened? +// (the `OpenWriteOnly` seems to implicitly be targeting a +// closed file, a file we forgot to flush? can we close +// a file without flushing?) +func (fi *File) Flush() error { + // open the file in fullsync mode + fd, err := fi.Open(Flags{Write: true, Sync: true}) + if err != nil { + return err + } + + defer fd.Close() + + return fd.Flush() +} + +func (fi *File) Sync() error { + // just being able to take the writelock means the descriptor is synced + // TODO: Why? + fi.desclock.Lock() + defer fi.desclock.Unlock() // Defer works around "empty critical section (SA2001)" + return nil +} + +// Type returns the type FSNode this is +func (fi *File) Type() NodeType { + return TFile +} diff --git a/mfs/inode.go b/mfs/inode.go new file mode 100644 index 0000000000..50bed0b38e --- /dev/null +++ b/mfs/inode.go @@ -0,0 +1,21 @@ +package mfs + +import ( + ipld "github.com/ipfs/go-ipld-format" +) + +// inode abstracts the common characteristics of the MFS `File` +// and `Directory`. All of its attributes are initialized at +// creation. +type inode struct { + // name of this `inode` in the MFS path (the same value + // is also stored as the name of the DAG link). + name string + + // parent directory of this `inode` (which may be the `Root`). + parent parent + + // dagService used to store modifications made to the contents + // of the file or directory the `inode` belongs to. + dagService ipld.DAGService +} diff --git a/mfs/mfs_test.go b/mfs/mfs_test.go new file mode 100644 index 0000000000..e5b571de69 --- /dev/null +++ b/mfs/mfs_test.go @@ -0,0 +1,1421 @@ +package mfs + +import ( + "bytes" + "context" + "encoding/binary" + "errors" + "fmt" + "io" + "math/rand" + "os" + "sort" + "sync" + "testing" + "time" + + path "github.com/ipfs/go-libipfs/path" + + bserv "github.com/ipfs/go-libipfs/blockservice" + ft "github.com/ipfs/go-libipfs/unixfs" + importer "github.com/ipfs/go-libipfs/unixfs/importer" + uio "github.com/ipfs/go-libipfs/unixfs/io" + dag "github.com/ipfs/go-merkledag" + + cid "github.com/ipfs/go-cid" + ds "github.com/ipfs/go-datastore" + dssync "github.com/ipfs/go-datastore/sync" + ipld "github.com/ipfs/go-ipld-format" + bstore "github.com/ipfs/go-libipfs/blockstore" + chunker "github.com/ipfs/go-libipfs/chunker" + offline "github.com/ipfs/go-libipfs/exchange/offline" + u "github.com/ipfs/go-libipfs/util" +) + +func emptyDirNode() *dag.ProtoNode { + return dag.NodeWithData(ft.FolderPBData()) +} + +func getDagserv(t *testing.T) ipld.DAGService { + db := dssync.MutexWrap(ds.NewMapDatastore()) + bs := bstore.NewBlockstore(db) + blockserv := bserv.New(bs, offline.Exchange(bs)) + return dag.NewDAGService(blockserv) +} + +func getRandFile(t *testing.T, ds ipld.DAGService, size int64) ipld.Node { + r := io.LimitReader(u.NewTimeSeededRand(), size) + return fileNodeFromReader(t, ds, r) +} + +func fileNodeFromReader(t *testing.T, ds ipld.DAGService, r io.Reader) ipld.Node { + nd, err := importer.BuildDagFromReader(ds, chunker.DefaultSplitter(r)) + if err != nil { + t.Fatal(err) + } + return nd +} + +func mkdirP(t *testing.T, root *Directory, pth string) *Directory { + dirs := path.SplitList(pth) + cur := root + for _, d := range dirs { + n, err := cur.Mkdir(d) + if err != nil && err != os.ErrExist { + t.Fatal(err) + } + if err == os.ErrExist { + fsn, err := cur.Child(d) + if err != nil { + t.Fatal(err) + } + switch fsn := fsn.(type) { + case *Directory: + n = fsn + case *File: + t.Fatal("tried to make a directory where a file already exists") + } + } + + cur = n + } + return cur +} + +func assertDirNotAtPath(root *Directory, pth string) error { + _, err := DirLookup(root, pth) + if err == nil { + return fmt.Errorf("%s exists in %s", pth, root.name) + } + return nil +} + +func assertDirAtPath(root *Directory, pth string, children []string) error { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + fsn, err := DirLookup(root, pth) + if err != nil { + return err + } + + dir, ok := fsn.(*Directory) + if !ok { + return fmt.Errorf("%s was not a directory", pth) + } + + listing, err := dir.List(ctx) + if err != nil { + return err + } + + var names []string + for _, d := range listing { + names = append(names, d.Name) + } + + sort.Strings(children) + sort.Strings(names) + if !compStrArrs(children, names) { + return errors.New("directories children did not match") + } + + return nil +} + +func compStrArrs(a, b []string) bool { + if len(a) != len(b) { + return false + } + + for i := 0; i < len(a); i++ { + if a[i] != b[i] { + return false + } + } + + return true +} + +func assertFileAtPath(ds ipld.DAGService, root *Directory, expn ipld.Node, pth string) error { + exp, ok := expn.(*dag.ProtoNode) + if !ok { + return dag.ErrNotProtobuf + } + + parts := path.SplitList(pth) + cur := root + for i, d := range parts[:len(parts)-1] { + next, err := cur.Child(d) + if err != nil { + return fmt.Errorf("looking for %s failed: %s", pth, err) + } + + nextDir, ok := next.(*Directory) + if !ok { + return fmt.Errorf("%s points to a non-directory", parts[:i+1]) + } + + cur = nextDir + } + + last := parts[len(parts)-1] + finaln, err := cur.Child(last) + if err != nil { + return err + } + + file, ok := finaln.(*File) + if !ok { + return fmt.Errorf("%s was not a file", pth) + } + + rfd, err := file.Open(Flags{Read: true}) + if err != nil { + return err + } + + out, err := io.ReadAll(rfd) + if err != nil { + return err + } + + expbytes, err := catNode(ds, exp) + if err != nil { + return err + } + + if !bytes.Equal(out, expbytes) { + return fmt.Errorf("incorrect data at path") + } + return nil +} + +func catNode(ds ipld.DAGService, nd *dag.ProtoNode) ([]byte, error) { + r, err := uio.NewDagReader(context.TODO(), nd, ds) + if err != nil { + return nil, err + } + defer r.Close() + + return io.ReadAll(r) +} + +func setupRoot(ctx context.Context, t *testing.T) (ipld.DAGService, *Root) { + ds := getDagserv(t) + + root := emptyDirNode() + rt, err := NewRoot(ctx, ds, root, func(ctx context.Context, c cid.Cid) error { + fmt.Println("PUBLISHED: ", c) + return nil + }) + + if err != nil { + t.Fatal(err) + } + + return ds, rt +} + +func TestBasic(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + ds, rt := setupRoot(ctx, t) + + rootdir := rt.GetDirectory() + + // test making a basic dir + _, err := rootdir.Mkdir("a") + if err != nil { + t.Fatal(err) + } + + path := "a/b/c/d/e/f/g" + d := mkdirP(t, rootdir, path) + + fi := getRandFile(t, ds, 1000) + + // test inserting that file + err = d.AddChild("afile", fi) + if err != nil { + t.Fatal(err) + } + + err = assertFileAtPath(ds, rootdir, fi, "a/b/c/d/e/f/g/afile") + if err != nil { + t.Fatal(err) + } +} + +func TestMkdir(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + _, rt := setupRoot(ctx, t) + + rootdir := rt.GetDirectory() + + dirsToMake := []string{"a", "B", "foo", "bar", "cats", "fish"} + sort.Strings(dirsToMake) // sort for easy comparing later + + for _, d := range dirsToMake { + _, err := rootdir.Mkdir(d) + if err != nil { + t.Fatal(err) + } + } + + err := assertDirAtPath(rootdir, "/", dirsToMake) + if err != nil { + t.Fatal(err) + } + + for _, d := range dirsToMake { + mkdirP(t, rootdir, "a/"+d) + } + + err = assertDirAtPath(rootdir, "/a", dirsToMake) + if err != nil { + t.Fatal(err) + } + + // mkdir over existing dir should fail + _, err = rootdir.Mkdir("a") + if err == nil { + t.Fatal("should have failed!") + } +} + +func TestDirectoryLoadFromDag(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + ds, rt := setupRoot(ctx, t) + + rootdir := rt.GetDirectory() + + nd := getRandFile(t, ds, 1000) + err := ds.Add(ctx, nd) + if err != nil { + t.Fatal(err) + } + + fihash := nd.Cid() + + dir := emptyDirNode() + err = ds.Add(ctx, dir) + if err != nil { + t.Fatal(err) + } + + dirhash := dir.Cid() + + top := emptyDirNode() + top.SetLinks([]*ipld.Link{ + { + Name: "a", + Cid: fihash, + }, + { + Name: "b", + Cid: dirhash, + }, + }) + + err = rootdir.AddChild("foo", top) + if err != nil { + t.Fatal(err) + } + + // get this dir + topi, err := rootdir.Child("foo") + if err != nil { + t.Fatal(err) + } + + topd := topi.(*Directory) + + path := topd.Path() + if path != "/foo" { + t.Fatalf("Expected path '/foo', got '%s'", path) + } + + // mkdir over existing but unloaded child file should fail + _, err = topd.Mkdir("a") + if err == nil { + t.Fatal("expected to fail!") + } + + // mkdir over existing but unloaded child dir should fail + _, err = topd.Mkdir("b") + if err == nil { + t.Fatal("expected to fail!") + } + + // adding a child over an existing path fails + err = topd.AddChild("b", nd) + if err == nil { + t.Fatal("expected to fail!") + } + + err = assertFileAtPath(ds, rootdir, nd, "foo/a") + if err != nil { + t.Fatal(err) + } + + err = assertDirAtPath(rootdir, "foo/b", nil) + if err != nil { + t.Fatal(err) + } + + err = rootdir.Unlink("foo") + if err != nil { + t.Fatal(err) + } + + err = assertDirAtPath(rootdir, "", nil) + if err != nil { + t.Fatal(err) + } +} + +func TestMvFile(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + dagService, rt := setupRoot(ctx, t) + rootDir := rt.GetDirectory() + + fi := getRandFile(t, dagService, 1000) + + err := rootDir.AddChild("afile", fi) + if err != nil { + t.Fatal(err) + } + + err = Mv(rt, "/afile", "/bfile") + if err != nil { + t.Fatal(err) + } + + err = assertFileAtPath(dagService, rootDir, fi, "bfile") + if err != nil { + t.Fatal(err) + } +} + +func TestMvFileToSubdir(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + dagService, rt := setupRoot(ctx, t) + rootDir := rt.GetDirectory() + + _ = mkdirP(t, rootDir, "test1") + + fi := getRandFile(t, dagService, 1000) + + err := rootDir.AddChild("afile", fi) + if err != nil { + t.Fatal(err) + } + + err = Mv(rt, "/afile", "/test1") + if err != nil { + t.Fatal(err) + } + + err = assertFileAtPath(dagService, rootDir, fi, "test1/afile") + if err != nil { + t.Fatal(err) + } +} + +func TestMvFileToSubdirWithRename(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + dagService, rt := setupRoot(ctx, t) + rootDir := rt.GetDirectory() + + _ = mkdirP(t, rootDir, "test1") + + fi := getRandFile(t, dagService, 1000) + + err := rootDir.AddChild("afile", fi) + if err != nil { + t.Fatal(err) + } + + err = Mv(rt, "/afile", "/test1/bfile") + if err != nil { + t.Fatal(err) + } + + err = assertFileAtPath(dagService, rootDir, fi, "test1/bfile") + if err != nil { + t.Fatal(err) + } +} + +func TestMvDir(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + dagService, rt := setupRoot(ctx, t) + rootDir := rt.GetDirectory() + + _ = mkdirP(t, rootDir, "test1") + d2 := mkdirP(t, rootDir, "test2") + + fi := getRandFile(t, dagService, 1000) + + err := d2.AddChild("afile", fi) + if err != nil { + t.Fatal(err) + } + + err = Mv(rt, "/test2", "/test1") + if err != nil { + t.Fatal(err) + } + + err = assertDirNotAtPath(rootDir, "test2") + if err != nil { + t.Fatal(err) + } + + err = assertFileAtPath(dagService, rootDir, fi, "test1/test2/afile") + if err != nil { + t.Fatal(err) + } +} + +func TestMfsFile(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + ds, rt := setupRoot(ctx, t) + + rootdir := rt.GetDirectory() + + fisize := 1000 + nd := getRandFile(t, ds, 1000) + + err := rootdir.AddChild("file", nd) + if err != nil { + t.Fatal(err) + } + + fsn, err := rootdir.Child("file") + if err != nil { + t.Fatal(err) + } + + fi := fsn.(*File) + + if fi.Type() != TFile { + t.Fatal("some is seriously wrong here") + } + + wfd, err := fi.Open(Flags{Read: true, Write: true, Sync: true}) + if err != nil { + t.Fatal(err) + } + + // assert size is as expected + size, err := fi.Size() + if err != nil { + t.Fatal(err) + } + if size != int64(fisize) { + t.Fatal("size isnt correct") + } + + // write to beginning of file + b := []byte("THIS IS A TEST") + n, err := wfd.Write(b) + if err != nil { + t.Fatal(err) + } + + if n != len(b) { + t.Fatal("didnt write correct number of bytes") + } + + // make sure size hasnt changed + size, err = wfd.Size() + if err != nil { + t.Fatal(err) + } + if size != int64(fisize) { + t.Fatal("size isnt correct") + } + + // seek back to beginning + ns, err := wfd.Seek(0, io.SeekStart) + if err != nil { + t.Fatal(err) + } + + if ns != 0 { + t.Fatal("didnt seek to beginning") + } + + // read back bytes we wrote + buf := make([]byte, len(b)) + n, err = wfd.Read(buf) + if err != nil { + t.Fatal(err) + } + + if n != len(buf) { + t.Fatal("didnt read enough") + } + + if !bytes.Equal(buf, b) { + t.Fatal("data read was different than data written") + } + + // truncate file to ten bytes + err = wfd.Truncate(10) + if err != nil { + t.Fatal(err) + } + + size, err = wfd.Size() + if err != nil { + t.Fatal(err) + } + + if size != 10 { + t.Fatal("size was incorrect: ", size) + } + + // 'writeAt' to extend it + data := []byte("this is a test foo foo foo") + nwa, err := wfd.WriteAt(data, 5) + if err != nil { + t.Fatal(err) + } + + if nwa != len(data) { + t.Fatal(err) + } + + // assert size once more + size, err = wfd.Size() + if err != nil { + t.Fatal(err) + } + + if size != int64(5+len(data)) { + t.Fatal("size was incorrect") + } + + // close it out! + err = wfd.Close() + if err != nil { + t.Fatal(err) + } + + // make sure we can get node. TODO: verify it later + _, err = fi.GetNode() + if err != nil { + t.Fatal(err) + } +} + +func TestMfsDirListNames(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + ds, rt := setupRoot(ctx, t) + + rootdir := rt.GetDirectory() + + rand.Seed(time.Now().UTC().UnixNano()) + + total := rand.Intn(10) + 1 + fNames := make([]string, 0, total) + + for i := 0; i < total; i++ { + fn := randomName() + fNames = append(fNames, fn) + nd := getRandFile(t, ds, rand.Int63n(1000)+1) + err := rootdir.AddChild(fn, nd) + if err != nil { + t.Fatal(err) + } + } + + list, err := rootdir.ListNames(ctx) + + if err != nil { + t.Fatal(err) + } + + for _, lName := range list { + found := false + for _, fName := range fNames { + if lName == fName { + found = true + break + } + } + if !found { + t.Fatal(lName + " not found in directory listing") + } + } +} + +func randomWalk(d *Directory, n int) (*Directory, error) { + for i := 0; i < n; i++ { + dirents, err := d.List(context.Background()) + if err != nil { + return nil, err + } + + var childdirs []NodeListing + for _, child := range dirents { + if child.Type == int(TDir) { + childdirs = append(childdirs, child) + } + } + if len(childdirs) == 0 { + return d, nil + } + + next := childdirs[rand.Intn(len(childdirs))].Name + + nextD, err := d.Child(next) + if err != nil { + return nil, err + } + + d = nextD.(*Directory) + } + return d, nil +} + +func randomName() string { + set := "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_" + length := rand.Intn(10) + 2 + var out string + for i := 0; i < length; i++ { + j := rand.Intn(len(set)) + out += set[j : j+1] + } + return out +} + +func actorMakeFile(d *Directory) error { + d, err := randomWalk(d, rand.Intn(7)) + if err != nil { + return err + } + + name := randomName() + f, err := NewFile(name, dag.NodeWithData(ft.FilePBData(nil, 0)), d, d.dagService) + if err != nil { + return err + } + + wfd, err := f.Open(Flags{Write: true, Sync: true}) + if err != nil { + return err + } + + rread := rand.New(rand.NewSource(time.Now().UnixNano())) + r := io.LimitReader(rread, int64(77*rand.Intn(123)+1)) + _, err = io.Copy(wfd, r) + if err != nil { + return err + } + + return wfd.Close() +} + +func actorMkdir(d *Directory) error { + d, err := randomWalk(d, rand.Intn(7)) + if err != nil { + return err + } + + _, err = d.Mkdir(randomName()) + + return err +} + +func randomFile(d *Directory) (*File, error) { + d, err := randomWalk(d, rand.Intn(6)) + if err != nil { + return nil, err + } + + ents, err := d.List(context.Background()) + if err != nil { + return nil, err + } + + var files []string + for _, e := range ents { + if e.Type == int(TFile) { + files = append(files, e.Name) + } + } + + if len(files) == 0 { + return nil, nil + } + + fname := files[rand.Intn(len(files))] + fsn, err := d.Child(fname) + if err != nil { + return nil, err + } + + fi, ok := fsn.(*File) + if !ok { + return nil, errors.New("file wasn't a file, race?") + } + + return fi, nil +} + +func actorWriteFile(d *Directory) error { + fi, err := randomFile(d) + if err != nil { + return err + } + if fi == nil { + return nil + } + + size := rand.Intn(1024) + 1 + buf := make([]byte, size) + rand.Read(buf) + + s, err := fi.Size() + if err != nil { + return err + } + + wfd, err := fi.Open(Flags{Write: true, Sync: true}) + if err != nil { + return err + } + + offset := rand.Int63n(s) + + n, err := wfd.WriteAt(buf, offset) + if err != nil { + return err + } + if n != size { + return fmt.Errorf("didnt write enough") + } + + return wfd.Close() +} + +func actorReadFile(d *Directory) error { + fi, err := randomFile(d) + if err != nil { + return err + } + if fi == nil { + return nil + } + + _, err = fi.Size() + if err != nil { + return err + } + + rfd, err := fi.Open(Flags{Read: true}) + if err != nil { + return err + } + + _, err = io.ReadAll(rfd) + if err != nil { + return err + } + + return rfd.Close() +} + +func testActor(rt *Root, iterations int, errs chan error) { + d := rt.GetDirectory() + for i := 0; i < iterations; i++ { + switch rand.Intn(5) { + case 0: + if err := actorMkdir(d); err != nil { + errs <- err + return + } + case 1, 2: + if err := actorMakeFile(d); err != nil { + errs <- err + return + } + case 3: + if err := actorWriteFile(d); err != nil { + errs <- err + return + } + case 4: + if err := actorReadFile(d); err != nil { + errs <- err + return + } + } + } + errs <- nil +} + +func TestMfsStress(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + _, rt := setupRoot(ctx, t) + + numroutines := 10 + + errs := make(chan error) + for i := 0; i < numroutines; i++ { + go testActor(rt, 50, errs) + } + + for i := 0; i < numroutines; i++ { + err := <-errs + if err != nil { + t.Fatal(err) + } + } +} + +func TestMfsHugeDir(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + _, rt := setupRoot(ctx, t) + + for i := 0; i < 10000; i++ { + err := Mkdir(rt, fmt.Sprintf("/dir%d", i), MkdirOpts{Mkparents: false, Flush: false}) + if err != nil { + t.Fatal(err) + } + } +} + +func TestMkdirP(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + _, rt := setupRoot(ctx, t) + + err := Mkdir(rt, "/a/b/c/d/e/f", MkdirOpts{Mkparents: true, Flush: true}) + if err != nil { + t.Fatal(err) + } +} + +func TestConcurrentWriteAndFlush(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + ds, rt := setupRoot(ctx, t) + + d := mkdirP(t, rt.GetDirectory(), "foo/bar/baz") + fn := fileNodeFromReader(t, ds, bytes.NewBuffer(nil)) + err := d.AddChild("file", fn) + if err != nil { + t.Fatal(err) + } + + nloops := 500 + + wg := new(sync.WaitGroup) + wg.Add(1) + go func() { + defer wg.Done() + for i := 0; i < nloops; i++ { + err := writeFile(rt, "/foo/bar/baz/file", func(_ []byte) []byte { return []byte("STUFF") }) + if err != nil { + t.Error("file write failed: ", err) + return + } + } + }() + + for i := 0; i < nloops; i++ { + _, err := rt.GetDirectory().GetNode() + if err != nil { + t.Fatal(err) + } + } + + wg.Wait() +} + +func TestFlushing(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + _, rt := setupRoot(ctx, t) + + dir := rt.GetDirectory() + c := mkdirP(t, dir, "a/b/c") + d := mkdirP(t, dir, "a/b/d") + e := mkdirP(t, dir, "a/b/e") + + data := []byte("this is a test\n") + nd1 := dag.NodeWithData(ft.FilePBData(data, uint64(len(data)))) + + if err := c.AddChild("TEST", nd1); err != nil { + t.Fatal(err) + } + if err := d.AddChild("TEST", nd1); err != nil { + t.Fatal(err) + } + if err := e.AddChild("TEST", nd1); err != nil { + t.Fatal(err) + } + if err := dir.AddChild("FILE", nd1); err != nil { + t.Fatal(err) + } + + nd, err := FlushPath(ctx, rt, "/a/b/c/TEST") + if err != nil { + t.Fatal(err) + } + if nd.Cid().String() != "QmYi7wrRFKVCcTB56A6Pep2j31Q5mHfmmu21RzHXu25RVR" { + t.Fatalf("unexpected node from FlushPath: %s", nd.Cid()) + } + + if _, err := FlushPath(ctx, rt, "/a/b/d/TEST"); err != nil { + t.Fatal(err) + } + + if _, err := FlushPath(ctx, rt, "/a/b/e/TEST"); err != nil { + t.Fatal(err) + } + + if _, err := FlushPath(ctx, rt, "/FILE"); err != nil { + t.Fatal(err) + } + + rnd, err := dir.GetNode() + if err != nil { + t.Fatal(err) + } + + pbrnd, ok := rnd.(*dag.ProtoNode) + if !ok { + t.Fatal(dag.ErrNotProtobuf) + } + + fsnode, err := ft.FSNodeFromBytes(pbrnd.Data()) + if err != nil { + t.Fatal(err) + } + + if fsnode.Type() != ft.TDirectory { + t.Fatal("root wasnt a directory") + } + + rnk := rnd.Cid() + exp := "QmWMVyhTuyxUrXX3ynz171jq76yY3PktfY9Bxiph7b9ikr" + if rnk.String() != exp { + t.Fatalf("dag looks wrong, expected %s, but got %s", exp, rnk.String()) + } +} + +func readFile(rt *Root, path string, offset int64, buf []byte) error { + n, err := Lookup(rt, path) + if err != nil { + return err + } + + fi, ok := n.(*File) + if !ok { + return fmt.Errorf("%s was not a file", path) + } + + fd, err := fi.Open(Flags{Read: true}) + if err != nil { + return err + } + + _, err = fd.Seek(offset, io.SeekStart) + if err != nil { + return err + } + + nread, err := fd.Read(buf) + if err != nil { + return err + } + if nread != len(buf) { + return fmt.Errorf("didn't read enough") + } + + return fd.Close() +} + +func TestConcurrentReads(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + ds, rt := setupRoot(ctx, t) + + rootdir := rt.GetDirectory() + + path := "a/b/c" + d := mkdirP(t, rootdir, path) + + buf := make([]byte, 2048) + rand.Read(buf) + + fi := fileNodeFromReader(t, ds, bytes.NewReader(buf)) + err := d.AddChild("afile", fi) + if err != nil { + t.Fatal(err) + } + + var wg sync.WaitGroup + nloops := 100 + for i := 0; i < 10; i++ { + wg.Add(1) + go func(me int) { + defer wg.Done() + mybuf := make([]byte, len(buf)) + for j := 0; j < nloops; j++ { + offset := rand.Intn(len(buf)) + length := rand.Intn(len(buf) - offset) + + err := readFile(rt, "/a/b/c/afile", int64(offset), mybuf[:length]) + if err != nil { + t.Error("readfile failed: ", err) + return + } + + if !bytes.Equal(mybuf[:length], buf[offset:offset+length]) { + t.Error("incorrect read!") + } + } + }(i) + } + wg.Wait() +} + +func writeFile(rt *Root, path string, transform func([]byte) []byte) error { + n, err := Lookup(rt, path) + if err != nil { + return err + } + + fi, ok := n.(*File) + if !ok { + return fmt.Errorf("expected to receive a file, but didnt get one") + } + + fd, err := fi.Open(Flags{Read: true, Write: true, Sync: true}) + if err != nil { + return err + } + defer fd.Close() + + data, err := io.ReadAll(fd) + if err != nil { + return err + } + data = transform(data) + + _, err = fd.Seek(0, io.SeekStart) + if err != nil { + return err + } + err = fd.Truncate(0) + if err != nil { + return err + } + + nw, err := fd.Write(data) + if err != nil { + return err + } + + if nw != len(data) { + return fmt.Errorf("wrote incorrect amount: %d != 10", nw) + } + + return nil +} + +func TestConcurrentWrites(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + ds, rt := setupRoot(ctx, t) + + rootdir := rt.GetDirectory() + + path := "a/b/c" + d := mkdirP(t, rootdir, path) + + fi := fileNodeFromReader(t, ds, bytes.NewReader(make([]byte, 0))) + err := d.AddChild("afile", fi) + if err != nil { + t.Fatal(err) + } + + var wg sync.WaitGroup + nloops := 100 + errs := make(chan error, 1000) + for i := 0; i < 10; i++ { + wg.Add(1) + go func() { + defer wg.Done() + var lastSeen uint64 + for j := 0; j < nloops; j++ { + err := writeFile(rt, "a/b/c/afile", func(buf []byte) []byte { + if len(buf) == 0 { + if lastSeen > 0 { + errs <- fmt.Errorf("file corrupted, last seen: %d", lastSeen) + return buf + } + buf = make([]byte, 8) + } else if len(buf) != 8 { + errs <- fmt.Errorf("buf not the right size") + return buf + } + + num := binary.LittleEndian.Uint64(buf) + if num < lastSeen { + errs <- fmt.Errorf("count decreased: was %d, is %d", lastSeen, num) + return buf + } else { + t.Logf("count correct: was %d, is %d", lastSeen, num) + } + num++ + binary.LittleEndian.PutUint64(buf, num) + lastSeen = num + return buf + }) + if err != nil { + errs <- fmt.Errorf("writefile failed: %v", err) + return + } + } + }() + } + wg.Wait() + close(errs) + for e := range errs { + t.Fatal(e) + } + buf := make([]byte, 8) + if err := readFile(rt, "a/b/c/afile", 0, buf); err != nil { + t.Fatal(err) + } + actual := binary.LittleEndian.Uint64(buf) + expected := uint64(10 * nloops) + if actual != expected { + t.Fatalf("iteration mismatch: expect %d, got %d", expected, actual) + } +} + +func TestFileDescriptors(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + ds, rt := setupRoot(ctx, t) + dir := rt.GetDirectory() + + nd := dag.NodeWithData(ft.FilePBData(nil, 0)) + fi, err := NewFile("test", nd, dir, ds) + if err != nil { + t.Fatal(err) + } + + // test read only + rfd1, err := fi.Open(Flags{Read: true}) + if err != nil { + t.Fatal(err) + } + + err = rfd1.Truncate(0) + if err == nil { + t.Fatal("shouldnt be able to truncate readonly fd") + } + + _, err = rfd1.Write([]byte{}) + if err == nil { + t.Fatal("shouldnt be able to write to readonly fd") + } + + _, err = rfd1.Read([]byte{}) + if err != nil { + t.Fatalf("expected to be able to read from file: %s", err) + } + + done := make(chan struct{}) + go func() { + defer close(done) + // can open second readonly file descriptor + rfd2, err := fi.Open(Flags{Read: true}) + if err != nil { + t.Error(err) + return + } + + rfd2.Close() + }() + + select { + case <-time.After(time.Second): + t.Fatal("open second file descriptor failed") + case <-done: + } + + if t.Failed() { + return + } + + // test not being able to open for write until reader are closed + done = make(chan struct{}) + go func() { + defer close(done) + wfd1, err := fi.Open(Flags{Write: true, Sync: true}) + if err != nil { + t.Error(err) + } + + wfd1.Close() + }() + + select { + case <-time.After(time.Millisecond * 200): + case <-done: + if t.Failed() { + return + } + + t.Fatal("shouldnt have been able to open file for writing") + } + + err = rfd1.Close() + if err != nil { + t.Fatal(err) + } + + select { + case <-time.After(time.Second): + t.Fatal("should have been able to open write fd after closing read fd") + case <-done: + } + + wfd, err := fi.Open(Flags{Write: true, Sync: true}) + if err != nil { + t.Fatal(err) + } + + _, err = wfd.Read([]byte{}) + if err == nil { + t.Fatal("shouldnt have been able to read from write only filedescriptor") + } + + _, err = wfd.Write([]byte{}) + if err != nil { + t.Fatal(err) + } +} + +func TestTruncateAtSize(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + ds, rt := setupRoot(ctx, t) + + dir := rt.GetDirectory() + + nd := dag.NodeWithData(ft.FilePBData(nil, 0)) + fi, err := NewFile("test", nd, dir, ds) + if err != nil { + t.Fatal(err) + } + + fd, err := fi.Open(Flags{Read: true, Write: true, Sync: true}) + if err != nil { + t.Fatal(err) + } + defer fd.Close() + _, err = fd.Write([]byte("test")) + if err != nil { + t.Fatal(err) + } + fd.Truncate(4) +} + +func TestTruncateAndWrite(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + ds, rt := setupRoot(ctx, t) + + dir := rt.GetDirectory() + + nd := dag.NodeWithData(ft.FilePBData(nil, 0)) + fi, err := NewFile("test", nd, dir, ds) + if err != nil { + t.Fatal(err) + } + + fd, err := fi.Open(Flags{Read: true, Write: true, Sync: true}) + if err != nil { + t.Fatal(err) + } + defer fd.Close() + for i := 0; i < 200; i++ { + err = fd.Truncate(0) + if err != nil { + t.Fatal(err) + } + l, err := fd.Write([]byte("test")) + if err != nil { + t.Fatal(err) + } + if l != len("test") { + t.Fatal("incorrect write length") + } + + _, err = fd.Seek(0, io.SeekStart) + if err != nil { + t.Fatal(err) + } + + data, err := io.ReadAll(fd) + if err != nil { + t.Fatal(err) + } + if string(data) != "test" { + t.Fatalf("read error at read %d, read: %v", i, data) + } + } +} + +func TestFSNodeType(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + ds, rt := setupRoot(ctx, t) + + // check for IsDir + nd := dag.NodeWithData(ft.FolderPBData()) + di, err := NewDirectory(ctx, "test", nd, rt.GetDirectory(), ds) + if err != nil { + t.Fatal(err) + } + ret := IsDir(di) + if !ret { + t.Fatal("FSNode type should be dir, but not") + } + + // check for IsFile + fnd := dag.NodeWithData(ft.FilePBData(nil, 0)) + fi, err := NewFile("test", fnd, rt.GetDirectory(), ds) + if err != nil { + t.Fatal(err) + } + ret = IsFile(fi) + if !ret { + t.Fatal("FSNode type should be file, but not") + } +} diff --git a/mfs/ops.go b/mfs/ops.go new file mode 100644 index 0000000000..bca93cc676 --- /dev/null +++ b/mfs/ops.go @@ -0,0 +1,246 @@ +package mfs + +import ( + "context" + "fmt" + "os" + gopath "path" + "strings" + + path "github.com/ipfs/go-libipfs/path" + + cid "github.com/ipfs/go-cid" + ipld "github.com/ipfs/go-ipld-format" +) + +// TODO: Evaluate moving all this operations to as `Root` +// methods, since all of them use it as its first argument +// and there is no clear documentation that explains this +// separation. + +// Mv moves the file or directory at 'src' to 'dst' +// TODO: Document what the strings 'src' and 'dst' represent. +func Mv(r *Root, src, dst string) error { + srcDirName, srcFname := gopath.Split(src) + + var dstDirName string + var dstFname string + if dst[len(dst)-1] == '/' { + dstDirName = dst + dstFname = srcFname + } else { + dstDirName, dstFname = gopath.Split(dst) + } + + // get parent directories of both src and dest first + dstDir, err := lookupDir(r, dstDirName) + if err != nil { + return err + } + + srcDir, err := lookupDir(r, srcDirName) + if err != nil { + return err + } + + srcObj, err := srcDir.Child(srcFname) + if err != nil { + return err + } + + nd, err := srcObj.GetNode() + if err != nil { + return err + } + + fsn, err := dstDir.Child(dstFname) + if err == nil { + switch n := fsn.(type) { + case *File: + _ = dstDir.Unlink(dstFname) + case *Directory: + dstDir = n + dstFname = srcFname + default: + return fmt.Errorf("unexpected type at path: %s", dst) + } + } else if err != os.ErrNotExist { + return err + } + + err = dstDir.AddChild(dstFname, nd) + if err != nil { + return err + } + + if srcDir.name == dstDir.name && srcFname == dstFname { + return nil + } + + return srcDir.Unlink(srcFname) +} + +func lookupDir(r *Root, path string) (*Directory, error) { + di, err := Lookup(r, path) + if err != nil { + return nil, err + } + + d, ok := di.(*Directory) + if !ok { + return nil, fmt.Errorf("%s is not a directory", path) + } + + return d, nil +} + +// PutNode inserts 'nd' at 'path' in the given mfs +// TODO: Rename or clearly document that this is not about nodes but actually +// MFS files/directories (that in the underlying representation can be +// considered as just nodes). +// TODO: Document why are we handling IPLD nodes in the first place when we +// are actually referring to files/directories (that is, it can't be any +// node, it has to have a specific format). +// TODO: Can this function add directories or just files? What would be the +// difference between adding a directory with this method and creating it +// with `Mkdir`. +func PutNode(r *Root, path string, nd ipld.Node) error { + dirp, filename := gopath.Split(path) + if filename == "" { + return fmt.Errorf("cannot create file with empty name") + } + + pdir, err := lookupDir(r, dirp) + if err != nil { + return err + } + + return pdir.AddChild(filename, nd) +} + +// MkdirOpts is used by Mkdir +type MkdirOpts struct { + Mkparents bool + Flush bool + CidBuilder cid.Builder +} + +// Mkdir creates a directory at 'path' under the directory 'd', creating +// intermediary directories as needed if 'mkparents' is set to true +func Mkdir(r *Root, pth string, opts MkdirOpts) error { + if pth == "" { + return fmt.Errorf("no path given to Mkdir") + } + parts := path.SplitList(pth) + if parts[0] == "" { + parts = parts[1:] + } + + // allow 'mkdir /a/b/c/' to create c + if parts[len(parts)-1] == "" { + parts = parts[:len(parts)-1] + } + + if len(parts) == 0 { + // this will only happen on 'mkdir /' + if opts.Mkparents { + return nil + } + return fmt.Errorf("cannot create directory '/': Already exists") + } + + cur := r.GetDirectory() + for i, d := range parts[:len(parts)-1] { + fsn, err := cur.Child(d) + if err == os.ErrNotExist && opts.Mkparents { + mkd, err := cur.Mkdir(d) + if err != nil { + return err + } + if opts.CidBuilder != nil { + mkd.SetCidBuilder(opts.CidBuilder) + } + fsn = mkd + } else if err != nil { + return err + } + + next, ok := fsn.(*Directory) + if !ok { + return fmt.Errorf("%s was not a directory", path.Join(parts[:i])) + } + cur = next + } + + final, err := cur.Mkdir(parts[len(parts)-1]) + if err != nil { + if !opts.Mkparents || err != os.ErrExist || final == nil { + return err + } + } + if opts.CidBuilder != nil { + final.SetCidBuilder(opts.CidBuilder) + } + + if opts.Flush { + err := final.Flush() + if err != nil { + return err + } + } + + return nil +} + +// Lookup extracts the root directory and performs a lookup under it. +// TODO: Now that the root is always a directory, can this function +// be collapsed with `DirLookup`? Or at least be made a method of `Root`? +func Lookup(r *Root, path string) (FSNode, error) { + dir := r.GetDirectory() + + return DirLookup(dir, path) +} + +// DirLookup will look up a file or directory at the given path +// under the directory 'd' +func DirLookup(d *Directory, pth string) (FSNode, error) { + pth = strings.Trim(pth, "/") + parts := path.SplitList(pth) + if len(parts) == 1 && parts[0] == "" { + return d, nil + } + + var cur FSNode + cur = d + for i, p := range parts { + chdir, ok := cur.(*Directory) + if !ok { + return nil, fmt.Errorf("cannot access %s: Not a directory", path.Join(parts[:i+1])) + } + + child, err := chdir.Child(p) + if err != nil { + return nil, err + } + + cur = child + } + return cur, nil +} + +// TODO: Document this function and link its functionality +// with the republisher. +func FlushPath(ctx context.Context, rt *Root, pth string) (ipld.Node, error) { + nd, err := Lookup(rt, pth) + if err != nil { + return nil, err + } + + err = nd.Flush() + if err != nil { + return nil, err + } + + rt.repub.WaitPub(ctx) + return nd.GetNode() +} diff --git a/mfs/options.go b/mfs/options.go new file mode 100644 index 0000000000..6bdcd71006 --- /dev/null +++ b/mfs/options.go @@ -0,0 +1,7 @@ +package mfs + +type Flags struct { + Read bool + Write bool + Sync bool +} diff --git a/mfs/repub.go b/mfs/repub.go new file mode 100644 index 0000000000..463810414c --- /dev/null +++ b/mfs/repub.go @@ -0,0 +1,197 @@ +package mfs + +import ( + "context" + "time" + + cid "github.com/ipfs/go-cid" +) + +// PubFunc is the user-defined function that determines exactly what +// logic entails "publishing" a `Cid` value. +type PubFunc func(context.Context, cid.Cid) error + +// Republisher manages when to publish a given entry. +type Republisher struct { + TimeoutLong time.Duration + TimeoutShort time.Duration + RetryTimeout time.Duration + pubfunc PubFunc + + update chan cid.Cid + immediatePublish chan chan struct{} + + ctx context.Context + cancel func() +} + +// NewRepublisher creates a new Republisher object to republish the given root +// using the given short and long time intervals. +func NewRepublisher(ctx context.Context, pf PubFunc, tshort, tlong time.Duration) *Republisher { + ctx, cancel := context.WithCancel(ctx) + return &Republisher{ + TimeoutShort: tshort, + TimeoutLong: tlong, + RetryTimeout: tlong, + update: make(chan cid.Cid, 1), + pubfunc: pf, + immediatePublish: make(chan chan struct{}), + ctx: ctx, + cancel: cancel, + } +} + +// WaitPub waits for the current value to be published (or returns early +// if it already has). +func (rp *Republisher) WaitPub(ctx context.Context) error { + wait := make(chan struct{}) + select { + case rp.immediatePublish <- wait: + case <-ctx.Done(): + return ctx.Err() + } + select { + case <-wait: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +func (rp *Republisher) Close() error { + // TODO(steb): Wait for `Run` to stop + err := rp.WaitPub(rp.ctx) + rp.cancel() + return err +} + +// Update the current value. The value will be published after a delay but each +// consecutive call to Update may extend this delay up to TimeoutLong. +func (rp *Republisher) Update(c cid.Cid) { + select { + case <-rp.update: + select { + case rp.update <- c: + default: + // Don't try again. If we hit this case, there's a + // concurrent publish and we can safely let that + // concurrent publish win. + } + case rp.update <- c: + } +} + +// Run contains the core logic of the `Republisher`. It calls the user-defined +// `pubfunc` function whenever the `Cid` value is updated to a *new* value. The +// complexity comes from the fact that `pubfunc` may be slow so we need to batch +// updates. +// +// Algorithm: +// 1. When we receive the first update after publishing, we set a `longer` timer. +// 2. When we receive any update, we reset the `quick` timer. +// 3. If either the `quick` timeout or the `longer` timeout elapses, +// we call `publish` with the latest updated value. +// +// The `longer` timer ensures that we delay publishing by at most +// `TimeoutLong`. The `quick` timer allows us to publish sooner if +// it looks like there are no more updates coming down the pipe. +// +// Note: If a publish fails, we retry repeatedly every TimeoutRetry. +func (rp *Republisher) Run(lastPublished cid.Cid) { + quick := time.NewTimer(0) + if !quick.Stop() { + <-quick.C + } + longer := time.NewTimer(0) + if !longer.Stop() { + <-longer.C + } + + var toPublish cid.Cid + for rp.ctx.Err() == nil { + var waiter chan struct{} + + select { + case <-rp.ctx.Done(): + return + case newValue := <-rp.update: + // Skip already published values. + if lastPublished.Equals(newValue) { + // Break to the end of the switch to cleanup any + // timers. + toPublish = cid.Undef + break + } + + // If we aren't already waiting to publish something, + // reset the long timeout. + if !toPublish.Defined() { + longer.Reset(rp.TimeoutLong) + } + + // Always reset the short timeout. + quick.Reset(rp.TimeoutShort) + + // Finally, set the new value to publish. + toPublish = newValue + continue + case waiter = <-rp.immediatePublish: + // Make sure to grab the *latest* value to publish. + select { + case toPublish = <-rp.update: + default: + } + + // Avoid publishing duplicate values + if lastPublished.Equals(toPublish) { + toPublish = cid.Undef + } + case <-quick.C: + case <-longer.C: + } + + // Cleanup, publish, and close waiters. + + // 1. Stop any timers. Don't use the `if !t.Stop() { ... }` + // idiom as these timers may not be running. + + quick.Stop() + select { + case <-quick.C: + default: + } + + longer.Stop() + select { + case <-longer.C: + default: + } + + // 2. If we have a value to publish, publish it now. + if toPublish.Defined() { + for { + err := rp.pubfunc(rp.ctx, toPublish) + if err == nil { + break + } + // Keep retrying until we succeed or we abort. + // TODO(steb): We could try pulling new values + // off `update` but that's not critical (and + // complicates this code a bit). We'll pull off + // a new value on the next loop through. + select { + case <-time.After(rp.RetryTimeout): + case <-rp.ctx.Done(): + return + } + } + lastPublished = toPublish + toPublish = cid.Undef + } + + // 3. Trigger anything waiting in `WaitPub`. + if waiter != nil { + close(waiter) + } + } +} diff --git a/mfs/repub_test.go b/mfs/repub_test.go new file mode 100644 index 0000000000..6be5624ab1 --- /dev/null +++ b/mfs/repub_test.go @@ -0,0 +1,75 @@ +package mfs + +import ( + "context" + "testing" + "time" + + cid "github.com/ipfs/go-cid" + ci "github.com/libp2p/go-libp2p-testing/ci" +) + +func TestRepublisher(t *testing.T) { + if ci.IsRunning() { + t.Skip("dont run timing tests in CI") + } + + ctx := context.TODO() + + pub := make(chan struct{}) + + pf := func(ctx context.Context, c cid.Cid) error { + pub <- struct{}{} + return nil + } + + testCid1, _ := cid.Parse("QmeomffUNfmQy76CQGy9NdmqEnnHU9soCexBnGU3ezPHVH") + testCid2, _ := cid.Parse("QmeomffUNfmQy76CQGy9NdmqEnnHU9soCexBnGU3ezPHVX") + + tshort := time.Millisecond * 50 + tlong := time.Second / 2 + + rp := NewRepublisher(ctx, pf, tshort, tlong) + go rp.Run(cid.Undef) + + rp.Update(testCid1) + + // should hit short timeout + select { + case <-time.After(tshort * 2): + t.Fatal("publish didnt happen in time") + case <-pub: + } + + cctx, cancel := context.WithCancel(context.Background()) + + go func() { + for { + rp.Update(testCid2) + time.Sleep(time.Millisecond * 10) + select { + case <-cctx.Done(): + return + default: + } + } + }() + + select { + case <-pub: + t.Fatal("shouldnt have received publish yet!") + case <-time.After((tlong * 9) / 10): + } + select { + case <-pub: + case <-time.After(tlong / 2): + t.Fatal("waited too long for pub!") + } + + cancel() + + err := rp.Close() + if err != nil { + t.Fatal(err) + } +} diff --git a/mfs/root.go b/mfs/root.go new file mode 100644 index 0000000000..5f1bcbef23 --- /dev/null +++ b/mfs/root.go @@ -0,0 +1,218 @@ +// package mfs implements an in memory model of a mutable IPFS filesystem. +// TODO: Develop on this line (and move it to `doc.go`). + +package mfs + +import ( + "context" + "errors" + "fmt" + "time" + + ft "github.com/ipfs/go-libipfs/unixfs" + dag "github.com/ipfs/go-merkledag" + + ipld "github.com/ipfs/go-ipld-format" + logging "github.com/ipfs/go-log" +) + +// TODO: Remove if not used. +var ErrNotExist = errors.New("no such rootfs") +var ErrClosed = errors.New("file closed") + +var log = logging.Logger("mfs") + +// TODO: Remove if not used. +var ErrIsDirectory = errors.New("error: is a directory") + +// The information that an MFS `Directory` has about its children +// when updating one of its entries: when a child mutates it signals +// its parent directory to update its entry (under `Name`) with the +// new content (in `Node`). +type child struct { + Name string + Node ipld.Node +} + +// This interface represents the basic property of MFS directories of updating +// children entries with modified content. Implemented by both the MFS +// `Directory` and `Root` (which is basically a `Directory` with republishing +// support). +// +// TODO: What is `fullsync`? (unnamed `bool` argument) +// TODO: There are two types of persistence/flush that need to be +// distinguished here, one at the DAG level (when I store the modified +// nodes in the DAG service) and one in the UnixFS/MFS level (when I modify +// the entry/link of the directory that pointed to the modified node). +type parent interface { + // Method called by a child to its parent to signal to update the content + // pointed to in the entry by that child's name. The child sends its own + // information in the `child` structure. As modifying a directory entry + // entails modifying its contents the parent will also call *its* parent's + // `updateChildEntry` to update the entry pointing to the new directory, + // this mechanism is in turn repeated until reaching the `Root`. + updateChildEntry(c child) error +} + +type NodeType int + +const ( + TFile NodeType = iota + TDir +) + +// FSNode abstracts the `Directory` and `File` structures, it represents +// any child node in the MFS (i.e., all the nodes besides the `Root`). It +// is the counterpart of the `parent` interface which represents any +// parent node in the MFS (`Root` and `Directory`). +// (Not to be confused with the `unixfs.FSNode`.) +type FSNode interface { + GetNode() (ipld.Node, error) + + Flush() error + Type() NodeType +} + +// IsDir checks whether the FSNode is dir type +func IsDir(fsn FSNode) bool { + return fsn.Type() == TDir +} + +// IsFile checks whether the FSNode is file type +func IsFile(fsn FSNode) bool { + return fsn.Type() == TFile +} + +// Root represents the root of a filesystem tree. +type Root struct { + + // Root directory of the MFS layout. + dir *Directory + + repub *Republisher +} + +// NewRoot creates a new Root and starts up a republisher routine for it. +func NewRoot(parent context.Context, ds ipld.DAGService, node *dag.ProtoNode, pf PubFunc) (*Root, error) { + + var repub *Republisher + if pf != nil { + repub = NewRepublisher(parent, pf, time.Millisecond*300, time.Second*3) + + // No need to take the lock here since we just created + // the `Republisher` and no one has access to it yet. + + go repub.Run(node.Cid()) + } + + root := &Root{ + repub: repub, + } + + fsn, err := ft.FSNodeFromBytes(node.Data()) + if err != nil { + log.Error("IPNS pointer was not unixfs node") + // TODO: IPNS pointer? + return nil, err + } + + switch fsn.Type() { + case ft.TDirectory, ft.THAMTShard: + newDir, err := NewDirectory(parent, node.String(), node, root, ds) + if err != nil { + return nil, err + } + + root.dir = newDir + case ft.TFile, ft.TMetadata, ft.TRaw: + return nil, fmt.Errorf("root can't be a file (unixfs type: %s)", fsn.Type()) + // TODO: This special error reporting case doesn't seem worth it, we either + // have a UnixFS directory or we don't. + default: + return nil, fmt.Errorf("unrecognized unixfs type: %s", fsn.Type()) + } + return root, nil +} + +// GetDirectory returns the root directory. +func (kr *Root) GetDirectory() *Directory { + return kr.dir +} + +// Flush signals that an update has occurred since the last publish, +// and updates the Root republisher. +// TODO: We are definitely abusing the "flush" terminology here. +func (kr *Root) Flush() error { + nd, err := kr.GetDirectory().GetNode() + if err != nil { + return err + } + + if kr.repub != nil { + kr.repub.Update(nd.Cid()) + } + return nil +} + +// FlushMemFree flushes the root directory and then uncaches all of its links. +// This has the effect of clearing out potentially stale references and allows +// them to be garbage collected. +// CAUTION: Take care not to ever call this while holding a reference to any +// child directories. Those directories will be bad references and using them +// may have unintended racy side effects. +// A better implemented mfs system (one that does smarter internal caching and +// refcounting) shouldnt need this method. +// TODO: Review the motivation behind this method once the cache system is +// refactored. +func (kr *Root) FlushMemFree(ctx context.Context) error { + dir := kr.GetDirectory() + + if err := dir.Flush(); err != nil { + return err + } + + dir.lock.Lock() + defer dir.lock.Unlock() + + for name := range dir.entriesCache { + delete(dir.entriesCache, name) + } + // TODO: Can't we just create new maps? + + return nil +} + +// updateChildEntry implements the `parent` interface, and signals +// to the publisher that there are changes ready to be published. +// This is the only thing that separates a `Root` from a `Directory`. +// TODO: Evaluate merging both. +// TODO: The `sync` argument isn't used here (we've already reached +// the top), document it and maybe make it an anonymous variable (if +// that's possible). +func (kr *Root) updateChildEntry(c child) error { + err := kr.GetDirectory().dagService.Add(context.TODO(), c.Node) + if err != nil { + return err + } + // TODO: Why are we not using the inner directory lock nor + // applying the same procedure as `Directory.updateChildEntry`? + + if kr.repub != nil { + kr.repub.Update(c.Node.Cid()) + } + return nil +} + +func (kr *Root) Close() error { + nd, err := kr.GetDirectory().GetNode() + if err != nil { + return err + } + + if kr.repub != nil { + kr.repub.Update(nd.Cid()) + return kr.repub.Close() + } + + return nil +} diff --git a/namesys/base.go b/namesys/base.go new file mode 100644 index 0000000000..cd94d4d3f7 --- /dev/null +++ b/namesys/base.go @@ -0,0 +1,126 @@ +package namesys + +import ( + "context" + "strings" + "time" + + opts "github.com/ipfs/go-libipfs/coreiface/options/namesys" + path "github.com/ipfs/go-libipfs/path" +) + +type onceResult struct { + value path.Path + ttl time.Duration + err error +} + +type resolver interface { + resolveOnceAsync(ctx context.Context, name string, options opts.ResolveOpts) <-chan onceResult +} + +// resolve is a helper for implementing Resolver.ResolveN using resolveOnce. +func resolve(ctx context.Context, r resolver, name string, options opts.ResolveOpts) (path.Path, error) { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + err := ErrResolveFailed + var p path.Path + + resCh := resolveAsync(ctx, r, name, options) + + for res := range resCh { + p, err = res.Path, res.Err + if err != nil { + break + } + } + + return p, err +} + +func resolveAsync(ctx context.Context, r resolver, name string, options opts.ResolveOpts) <-chan Result { + ctx, span := StartSpan(ctx, "ResolveAsync") + defer span.End() + + resCh := r.resolveOnceAsync(ctx, name, options) + depth := options.Depth + outCh := make(chan Result, 1) + + go func() { + defer close(outCh) + ctx, span := StartSpan(ctx, "ResolveAsync.Worker") + defer span.End() + + var subCh <-chan Result + var cancelSub context.CancelFunc + defer func() { + if cancelSub != nil { + cancelSub() + } + }() + + for { + select { + case res, ok := <-resCh: + if !ok { + resCh = nil + break + } + + if res.err != nil { + emitResult(ctx, outCh, Result{Err: res.err}) + return + } + log.Debugf("resolved %s to %s", name, res.value.String()) + if !strings.HasPrefix(res.value.String(), ipnsPrefix) { + emitResult(ctx, outCh, Result{Path: res.value}) + break + } + + if depth == 1 { + emitResult(ctx, outCh, Result{Path: res.value, Err: ErrResolveRecursion}) + break + } + + subopts := options + if subopts.Depth > 1 { + subopts.Depth-- + } + + var subCtx context.Context + if cancelSub != nil { + // Cancel previous recursive resolve since it won't be used anyways + cancelSub() + } + subCtx, cancelSub = context.WithCancel(ctx) + _ = cancelSub + + p := strings.TrimPrefix(res.value.String(), ipnsPrefix) + subCh = resolveAsync(subCtx, r, p, subopts) + case res, ok := <-subCh: + if !ok { + subCh = nil + break + } + + // We don't bother returning here in case of context timeout as there is + // no good reason to do that, and we may still be able to emit a result + emitResult(ctx, outCh, res) + case <-ctx.Done(): + return + } + if resCh == nil && subCh == nil { + return + } + } + }() + return outCh +} + +func emitResult(ctx context.Context, outCh chan<- Result, r Result) { + select { + case outCh <- r: + case <-ctx.Done(): + } +} diff --git a/namesys/cache.go b/namesys/cache.go new file mode 100644 index 0000000000..52cb29bc5f --- /dev/null +++ b/namesys/cache.go @@ -0,0 +1,62 @@ +package namesys + +import ( + "time" + + path "github.com/ipfs/go-libipfs/path" +) + +func (ns *mpns) cacheGet(name string) (path.Path, bool) { + // existence of optional mapping defined via IPFS_NS_MAP is checked first + if ns.staticMap != nil { + val, ok := ns.staticMap[name] + if ok { + return val, true + } + } + + if ns.cache == nil { + return "", false + } + + ientry, ok := ns.cache.Get(name) + if !ok { + return "", false + } + + entry, ok := ientry.(cacheEntry) + if !ok { + // should never happen, purely for sanity + log.Panicf("unexpected type %T in cache for %q.", ientry, name) + } + + if time.Now().Before(entry.eol) { + return entry.val, true + } + + ns.cache.Remove(name) + + return "", false +} + +func (ns *mpns) cacheSet(name string, val path.Path, ttl time.Duration) { + if ns.cache == nil || ttl <= 0 { + return + } + ns.cache.Add(name, cacheEntry{ + val: val, + eol: time.Now().Add(ttl), + }) +} + +func (ns *mpns) cacheInvalidate(name string) { + if ns.cache == nil { + return + } + ns.cache.Remove(name) +} + +type cacheEntry struct { + val path.Path + eol time.Time +} diff --git a/namesys/dns.go b/namesys/dns.go new file mode 100644 index 0000000000..c901fe904b --- /dev/null +++ b/namesys/dns.go @@ -0,0 +1,195 @@ +package namesys + +import ( + "context" + "errors" + "fmt" + "net" + gpath "path" + "strings" + + opts "github.com/ipfs/go-libipfs/coreiface/options/namesys" + path "github.com/ipfs/go-libipfs/path" + dns "github.com/miekg/dns" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" +) + +// LookupTXTFunc is a function that lookups TXT record values. +type LookupTXTFunc func(ctx context.Context, name string) (txt []string, err error) + +// DNSResolver implements a Resolver on DNS domains +type DNSResolver struct { + lookupTXT LookupTXTFunc + // TODO: maybe some sort of caching? + // cache would need a timeout +} + +// NewDNSResolver constructs a name resolver using DNS TXT records. +func NewDNSResolver(lookup LookupTXTFunc) *DNSResolver { + return &DNSResolver{lookupTXT: lookup} +} + +// Resolve implements Resolver. +func (r *DNSResolver) Resolve(ctx context.Context, name string, options ...opts.ResolveOpt) (path.Path, error) { + ctx, span := StartSpan(ctx, "DNSResolver.Resolve") + defer span.End() + + return resolve(ctx, r, name, opts.ProcessOpts(options)) +} + +// ResolveAsync implements Resolver. +func (r *DNSResolver) ResolveAsync(ctx context.Context, name string, options ...opts.ResolveOpt) <-chan Result { + ctx, span := StartSpan(ctx, "DNSResolver.ResolveAsync") + defer span.End() + + return resolveAsync(ctx, r, name, opts.ProcessOpts(options)) +} + +type lookupRes struct { + path path.Path + error error +} + +// resolveOnce implements resolver. +// TXT records for a given domain name should contain a b58 +// encoded multihash. +func (r *DNSResolver) resolveOnceAsync(ctx context.Context, name string, options opts.ResolveOpts) <-chan onceResult { + ctx, span := StartSpan(ctx, "DNSResolver.ResolveOnceAsync") + defer span.End() + + var fqdn string + out := make(chan onceResult, 1) + segments := strings.SplitN(name, "/", 2) + domain := segments[0] + + if _, ok := dns.IsDomainName(domain); !ok { + out <- onceResult{err: fmt.Errorf("not a valid domain name: %s", domain)} + close(out) + return out + } + log.Debugf("DNSResolver resolving %s", domain) + + if strings.HasSuffix(domain, ".") { + fqdn = domain + } else { + fqdn = domain + "." + } + + rootChan := make(chan lookupRes, 1) + go workDomain(ctx, r, fqdn, rootChan) + + subChan := make(chan lookupRes, 1) + go workDomain(ctx, r, "_dnslink."+fqdn, subChan) + + appendPath := func(p path.Path) (path.Path, error) { + if len(segments) > 1 { + return path.FromSegments("", strings.TrimRight(p.String(), "/"), segments[1]) + } + return p, nil + } + + go func() { + defer close(out) + ctx, span := StartSpan(ctx, "DNSResolver.ResolveOnceAsync.Worker") + defer span.End() + + var rootResErr, subResErr error + for { + select { + case subRes, ok := <-subChan: + if !ok { + subChan = nil + break + } + if subRes.error == nil { + p, err := appendPath(subRes.path) + emitOnceResult(ctx, out, onceResult{value: p, err: err}) + // Return without waiting for rootRes, since this result + // (for "_dnslink."+fqdn) takes precedence + return + } + subResErr = subRes.error + case rootRes, ok := <-rootChan: + if !ok { + rootChan = nil + break + } + if rootRes.error == nil { + p, err := appendPath(rootRes.path) + emitOnceResult(ctx, out, onceResult{value: p, err: err}) + // Do not return here. Wait for subRes so that it is + // output last if good, thereby giving subRes precedence. + } else { + rootResErr = rootRes.error + } + case <-ctx.Done(): + return + } + if subChan == nil && rootChan == nil { + // If here, then both lookups are done + // + // If both lookups failed due to no TXT records with a + // dnslink, then output a more specific error message + if rootResErr == ErrResolveFailed && subResErr == ErrResolveFailed { + // Wrap error so that it can be tested if it is a ErrResolveFailed + err := fmt.Errorf("%w: _dnslink subdomain at %q is missing a TXT record (https://docs.ipfs.tech/concepts/dnslink/)", ErrResolveFailed, gpath.Base(name)) + emitOnceResult(ctx, out, onceResult{err: err}) + } + return + } + } + }() + + return out +} + +func workDomain(ctx context.Context, r *DNSResolver, name string, res chan lookupRes) { + ctx, span := StartSpan(ctx, "DNSResolver.WorkDomain", trace.WithAttributes(attribute.String("Name", name))) + defer span.End() + + defer close(res) + + txt, err := r.lookupTXT(ctx, name) + if err != nil { + if dnsErr, ok := err.(*net.DNSError); ok { + // If no TXT records found, return same error as when no text + // records contain dnslink. Otherwise, return the actual error. + if dnsErr.IsNotFound { + err = ErrResolveFailed + } + } + // Could not look up any text records for name + res <- lookupRes{"", err} + return + } + + for _, t := range txt { + p, err := parseEntry(t) + if err == nil { + res <- lookupRes{p, nil} + return + } + } + + // There were no TXT records with a dnslink + res <- lookupRes{"", ErrResolveFailed} +} + +func parseEntry(txt string) (path.Path, error) { + p, err := path.ParseCidToPath(txt) // bare IPFS multihashes + if err == nil { + return p, nil + } + + return tryParseDNSLink(txt) +} + +func tryParseDNSLink(txt string) (path.Path, error) { + parts := strings.SplitN(txt, "=", 2) + if len(parts) == 2 && parts[0] == "dnslink" { + return path.ParsePath(parts[1]) + } + + return "", errors.New("not a valid dnslink entry") +} diff --git a/namesys/dns_test.go b/namesys/dns_test.go new file mode 100644 index 0000000000..aee0e150e2 --- /dev/null +++ b/namesys/dns_test.go @@ -0,0 +1,184 @@ +package namesys + +import ( + "context" + "fmt" + "testing" + + opts "github.com/ipfs/go-libipfs/coreiface/options/namesys" +) + +type mockDNS struct { + entries map[string][]string +} + +func (m *mockDNS) lookupTXT(ctx context.Context, name string) (txt []string, err error) { + txt, ok := m.entries[name] + if !ok { + return nil, fmt.Errorf("no TXT entry for %s", name) + } + return txt, nil +} + +func TestDnsEntryParsing(t *testing.T) { + goodEntries := []string{ + "QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", + "dnslink=/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", + "dnslink=/ipns/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", + "dnslink=/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/foo", + "dnslink=/ipns/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/bar", + "dnslink=/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/foo/bar/baz", + "dnslink=/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/foo/bar/baz/", + "dnslink=/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", + } + + badEntries := []string{ + "QmYhE8xgFCjGcz6PHgnvJz5NOTCORRECT", + "quux=/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", + "dnslink=", + "dnslink=/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/foo", + "dnslink=ipns/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/bar", + } + + for _, e := range goodEntries { + _, err := parseEntry(e) + if err != nil { + t.Log("expected entry to parse correctly!") + t.Log(e) + t.Fatal(err) + } + } + + for _, e := range badEntries { + _, err := parseEntry(e) + if err == nil { + t.Log("expected entry parse to fail!") + t.Fatal(err) + } + } +} + +func newMockDNS() *mockDNS { + return &mockDNS{ + entries: map[string][]string{ + "multihash.example.com.": { + "dnslink=QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", + }, + "ipfs.example.com.": { + "dnslink=/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", + }, + "_dnslink.dipfs.example.com.": { + "dnslink=/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", + }, + "dns1.example.com.": { + "dnslink=/ipns/ipfs.example.com", + }, + "dns2.example.com.": { + "dnslink=/ipns/dns1.example.com", + }, + "multi.example.com.": { + "some stuff", + "dnslink=/ipns/dns1.example.com", + "masked dnslink=/ipns/example.invalid", + }, + "equals.example.com.": { + "dnslink=/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/=equals", + }, + "loop1.example.com.": { + "dnslink=/ipns/loop2.example.com", + }, + "loop2.example.com.": { + "dnslink=/ipns/loop1.example.com", + }, + "_dnslink.dloop1.example.com.": { + "dnslink=/ipns/loop2.example.com", + }, + "_dnslink.dloop2.example.com.": { + "dnslink=/ipns/loop1.example.com", + }, + "bad.example.com.": { + "dnslink=", + }, + "withsegment.example.com.": { + "dnslink=/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/sub/segment", + }, + "withrecsegment.example.com.": { + "dnslink=/ipns/withsegment.example.com/subsub", + }, + "withtrailing.example.com.": { + "dnslink=/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/sub/", + }, + "withtrailingrec.example.com.": { + "dnslink=/ipns/withtrailing.example.com/segment/", + }, + "double.example.com.": { + "dnslink=/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", + }, + "_dnslink.double.example.com.": { + "dnslink=/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", + }, + "double.conflict.com.": { + "dnslink=/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", + }, + "_dnslink.conflict.example.com.": { + "dnslink=/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjE", + }, + "fqdn.example.com.": { + "dnslink=/ipfs/QmYvMB9yrsSf7RKBghkfwmHJkzJhW2ZgVwq3LxBXXPasFr", + }, + "en.wikipedia-on-ipfs.org.": { + "dnslink=/ipfs/bafybeiaysi4s6lnjev27ln5icwm6tueaw2vdykrtjkwiphwekaywqhcjze", + }, + "custom.non-icann.tldextravaganza.": { + "dnslink=/ipfs/bafybeieto6mcuvqlechv4iadoqvnffondeiwxc2bcfcewhvpsd2odvbmvm", + }, + "singlednslabelshouldbeok.": { + "dnslink=/ipfs/bafybeih4a6ylafdki6ailjrdvmr7o4fbbeceeeuty4v3qyyouiz5koqlpi", + }, + "www.wealdtech.eth.": { + "dnslink=/ipns/ipfs.example.com", + }, + }, + } +} + +func TestDNSResolution(t *testing.T) { + mock := newMockDNS() + r := &DNSResolver{lookupTXT: mock.lookupTXT} + testResolution(t, r, "multihash.example.com", opts.DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", nil) + testResolution(t, r, "ipfs.example.com", opts.DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", nil) + testResolution(t, r, "dipfs.example.com", opts.DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", nil) + testResolution(t, r, "dns1.example.com", opts.DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", nil) + testResolution(t, r, "dns1.example.com", 1, "/ipns/ipfs.example.com", ErrResolveRecursion) + testResolution(t, r, "dns2.example.com", opts.DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", nil) + testResolution(t, r, "dns2.example.com", 1, "/ipns/dns1.example.com", ErrResolveRecursion) + testResolution(t, r, "dns2.example.com", 2, "/ipns/ipfs.example.com", ErrResolveRecursion) + testResolution(t, r, "multi.example.com", opts.DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", nil) + testResolution(t, r, "multi.example.com", 1, "/ipns/dns1.example.com", ErrResolveRecursion) + testResolution(t, r, "multi.example.com", 2, "/ipns/ipfs.example.com", ErrResolveRecursion) + testResolution(t, r, "equals.example.com", opts.DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/=equals", nil) + testResolution(t, r, "loop1.example.com", 1, "/ipns/loop2.example.com", ErrResolveRecursion) + testResolution(t, r, "loop1.example.com", 2, "/ipns/loop1.example.com", ErrResolveRecursion) + testResolution(t, r, "loop1.example.com", 3, "/ipns/loop2.example.com", ErrResolveRecursion) + testResolution(t, r, "loop1.example.com", opts.DefaultDepthLimit, "/ipns/loop1.example.com", ErrResolveRecursion) + testResolution(t, r, "dloop1.example.com", 1, "/ipns/loop2.example.com", ErrResolveRecursion) + testResolution(t, r, "dloop1.example.com", 2, "/ipns/loop1.example.com", ErrResolveRecursion) + testResolution(t, r, "dloop1.example.com", 3, "/ipns/loop2.example.com", ErrResolveRecursion) + testResolution(t, r, "dloop1.example.com", opts.DefaultDepthLimit, "/ipns/loop1.example.com", ErrResolveRecursion) + testResolution(t, r, "bad.example.com", opts.DefaultDepthLimit, "", ErrResolveFailed) + testResolution(t, r, "withsegment.example.com", opts.DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/sub/segment", nil) + testResolution(t, r, "withrecsegment.example.com", opts.DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/sub/segment/subsub", nil) + testResolution(t, r, "withsegment.example.com/test1", opts.DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/sub/segment/test1", nil) + testResolution(t, r, "withrecsegment.example.com/test2", opts.DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/sub/segment/subsub/test2", nil) + testResolution(t, r, "withrecsegment.example.com/test3/", opts.DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/sub/segment/subsub/test3/", nil) + testResolution(t, r, "withtrailingrec.example.com", opts.DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/sub/segment/", nil) + testResolution(t, r, "double.example.com", opts.DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", nil) + testResolution(t, r, "conflict.example.com", opts.DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjE", nil) + testResolution(t, r, "fqdn.example.com.", opts.DefaultDepthLimit, "/ipfs/QmYvMB9yrsSf7RKBghkfwmHJkzJhW2ZgVwq3LxBXXPasFr", nil) + testResolution(t, r, "en.wikipedia-on-ipfs.org", 2, "/ipfs/bafybeiaysi4s6lnjev27ln5icwm6tueaw2vdykrtjkwiphwekaywqhcjze", nil) + testResolution(t, r, "custom.non-icann.tldextravaganza.", 2, "/ipfs/bafybeieto6mcuvqlechv4iadoqvnffondeiwxc2bcfcewhvpsd2odvbmvm", nil) + testResolution(t, r, "singlednslabelshouldbeok", 2, "/ipfs/bafybeih4a6ylafdki6ailjrdvmr7o4fbbeceeeuty4v3qyyouiz5koqlpi", nil) + testResolution(t, r, "www.wealdtech.eth", 1, "/ipns/ipfs.example.com", ErrResolveRecursion) + testResolution(t, r, "www.wealdtech.eth", 2, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", nil) + testResolution(t, r, "www.wealdtech.eth", 2, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", nil) +} diff --git a/namesys/interface.go b/namesys/interface.go new file mode 100644 index 0000000000..067291951b --- /dev/null +++ b/namesys/interface.go @@ -0,0 +1,100 @@ +/* +Package namesys implements resolvers and publishers for the IPFS +naming system (IPNS). + +The core of IPFS is an immutable, content-addressable Merkle graph. +That works well for many use cases, but doesn't allow you to answer +questions like "what is Alice's current homepage?". The mutable name +system allows Alice to publish information like: + + The current homepage for alice.example.com is + /ipfs/Qmcqtw8FfrVSBaRmbWwHxt3AuySBhJLcvmFYi3Lbc4xnwj + +or: + + The current homepage for node + QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy + is + /ipfs/Qmcqtw8FfrVSBaRmbWwHxt3AuySBhJLcvmFYi3Lbc4xnwj + +The mutable name system also allows users to resolve those references +to find the immutable IPFS object currently referenced by a given +mutable name. + +For command-line bindings to this functionality, see: + + ipfs name + ipfs dns + ipfs resolve +*/ +package namesys + +import ( + "errors" + + "context" + + opts "github.com/ipfs/go-libipfs/coreiface/options/namesys" + "github.com/ipfs/go-libipfs/path" + ci "github.com/libp2p/go-libp2p/core/crypto" +) + +// ErrResolveFailed signals an error when attempting to resolve. +var ErrResolveFailed = errors.New("could not resolve name") + +// ErrResolveRecursion signals a recursion-depth limit. +var ErrResolveRecursion = errors.New( + "could not resolve name (recursion limit exceeded)") + +// ErrPublishFailed signals an error when attempting to publish. +var ErrPublishFailed = errors.New("could not publish name") + +// NameSystem represents a cohesive name publishing and resolving system. +// +// Publishing a name is the process of establishing a mapping, a key-value +// pair, according to naming rules and databases. +// +// Resolving a name is the process of looking up the value associated with the +// key (name). +type NameSystem interface { + Resolver + Publisher +} + +// Result is the return type for Resolver.ResolveAsync. +type Result struct { + Path path.Path + Err error +} + +// Resolver is an object capable of resolving names. +type Resolver interface { + + // Resolve performs a recursive lookup, returning the dereferenced + // path. For example, if ipfs.io has a DNS TXT record pointing to + // /ipns/QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy + // and there is a DHT IPNS entry for + // QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy + // -> /ipfs/Qmcqtw8FfrVSBaRmbWwHxt3AuySBhJLcvmFYi3Lbc4xnwj + // then + // Resolve(ctx, "/ipns/ipfs.io") + // will resolve both names, returning + // /ipfs/Qmcqtw8FfrVSBaRmbWwHxt3AuySBhJLcvmFYi3Lbc4xnwj + // + // There is a default depth-limit to avoid infinite recursion. Most + // users will be fine with this default limit, but if you need to + // adjust the limit you can specify it as an option. + Resolve(ctx context.Context, name string, options ...opts.ResolveOpt) (value path.Path, err error) + + // ResolveAsync performs recursive name lookup, like Resolve, but it returns + // entries as they are discovered in the DHT. Each returned result is guaranteed + // to be "better" (which usually means newer) than the previous one. + ResolveAsync(ctx context.Context, name string, options ...opts.ResolveOpt) <-chan Result +} + +// Publisher is an object capable of publishing particular names. +type Publisher interface { + // Publish establishes a name-value mapping. + // TODO make this not PrivKey specific. + Publish(ctx context.Context, name ci.PrivKey, value path.Path, options ...opts.PublishOption) error +} diff --git a/namesys/ipns_resolver_validation_test.go b/namesys/ipns_resolver_validation_test.go new file mode 100644 index 0000000000..edf8156a6b --- /dev/null +++ b/namesys/ipns_resolver_validation_test.go @@ -0,0 +1,209 @@ +package namesys + +import ( + "context" + "testing" + "time" + + ds "github.com/ipfs/go-datastore" + dssync "github.com/ipfs/go-datastore/sync" + opts "github.com/ipfs/go-libipfs/coreiface/options/namesys" + "github.com/ipfs/go-libipfs/ipns" + ipns_pb "github.com/ipfs/go-libipfs/ipns/pb" + "github.com/ipfs/go-libipfs/path" + mockrouting "github.com/ipfs/go-libipfs/routing/mock" + "github.com/ipfs/go-libipfs/routing/offline" + record "github.com/libp2p/go-libp2p-record" + testutil "github.com/libp2p/go-libp2p-testing/net" + ci "github.com/libp2p/go-libp2p/core/crypto" + "github.com/libp2p/go-libp2p/core/peer" + pstore "github.com/libp2p/go-libp2p/core/peerstore" + "github.com/libp2p/go-libp2p/core/routing" + "github.com/libp2p/go-libp2p/core/test" + "github.com/libp2p/go-libp2p/p2p/host/peerstore/pstoremem" +) + +func TestResolverValidation(t *testing.T) { + t.Run("RSA", + func(t *testing.T) { + testResolverValidation(t, ci.RSA) + }) + t.Run("Ed25519", + func(t *testing.T) { + testResolverValidation(t, ci.Ed25519) + }) + t.Run("ECDSA", + func(t *testing.T) { + testResolverValidation(t, ci.ECDSA) + }) + t.Run("Secp256k1", + func(t *testing.T) { + testResolverValidation(t, ci.Secp256k1) + }) +} + +func testResolverValidation(t *testing.T, keyType int) { + ctx := context.Background() + rid := testutil.RandIdentityOrFatal(t) + dstore := dssync.MutexWrap(ds.NewMapDatastore()) + peerstore, err := pstoremem.NewPeerstore() + if err != nil { + t.Fatal(err) + } + + vstore := newMockValueStore(rid, dstore, peerstore) + resolver := NewIpnsResolver(vstore) + + nvVstore := offline.NewOfflineRouter(dstore, mockrouting.MockValidator{}) + + // Create entry with expiry in one hour + priv, id, _, ipnsDHTPath := genKeys(t, keyType) + ts := time.Now() + p := []byte("/ipfs/QmfM2r8seH2GiRaC4esTjeraXEachRt8ZsSeGaWTPLyMoG") + entry, err := createIPNSRecordWithEmbeddedPublicKey(priv, p, 1, ts.Add(time.Hour), 0) + if err != nil { + t.Fatal(err) + } + + // Publish entry + err = PublishEntry(ctx, vstore, ipnsDHTPath, entry) + if err != nil { + t.Fatal(err) + } + + // Resolve entry + resp, err := resolve(ctx, resolver, id.Pretty(), opts.DefaultResolveOpts()) + if err != nil { + t.Fatal(err) + } + if resp != path.Path(p) { + t.Fatalf("Mismatch between published path %s and resolved path %s", p, resp) + } + // Create expired entry + expiredEntry, err := createIPNSRecordWithEmbeddedPublicKey(priv, p, 1, ts.Add(-1*time.Hour), 0) + if err != nil { + t.Fatal(err) + } + + // Publish entry + err = PublishEntry(ctx, nvVstore, ipnsDHTPath, expiredEntry) + if err != nil { + t.Fatal(err) + } + + // Record should fail validation because entry is expired + _, err = resolve(ctx, resolver, id.Pretty(), opts.DefaultResolveOpts()) + if err == nil { + t.Fatal("ValidateIpnsRecord should have returned error") + } + + // Create IPNS record path with a different private key + priv2, id2, _, ipnsDHTPath2 := genKeys(t, keyType) + + // Publish entry + err = PublishEntry(ctx, nvVstore, ipnsDHTPath2, entry) + if err != nil { + t.Fatal(err) + } + + // Record should fail validation because public key defined by + // ipns path doesn't match record signature + _, err = resolve(ctx, resolver, id2.Pretty(), opts.DefaultResolveOpts()) + if err == nil { + t.Fatal("ValidateIpnsRecord should have failed signature verification") + } + + // Try embedding the incorrect private key inside the entry + if err := ipns.EmbedPublicKey(priv2.GetPublic(), entry); err != nil { + t.Fatal(err) + } + + // Publish entry + err = PublishEntry(ctx, nvVstore, ipnsDHTPath2, entry) + if err != nil { + t.Fatal(err) + } + + // Record should fail validation because public key defined by + // ipns path doesn't match record signature + _, err = resolve(ctx, resolver, id2.Pretty(), opts.DefaultResolveOpts()) + if err == nil { + t.Fatal("ValidateIpnsRecord should have failed signature verification") + } +} + +func genKeys(t *testing.T, keyType int) (ci.PrivKey, peer.ID, string, string) { + bits := 0 + if keyType == ci.RSA { + bits = 2048 + } + + sk, pk, err := test.RandTestKeyPair(keyType, bits) + if err != nil { + t.Fatal(err) + } + id, err := peer.IDFromPublicKey(pk) + if err != nil { + t.Fatal(err) + } + return sk, id, PkKeyForID(id), ipns.RecordKey(id) +} + +func createIPNSRecordWithEmbeddedPublicKey(sk ci.PrivKey, val []byte, seq uint64, eol time.Time, ttl time.Duration) (*ipns_pb.IpnsEntry, error) { + entry, err := ipns.Create(sk, val, seq, eol, ttl) + if err != nil { + return nil, err + } + if err := ipns.EmbedPublicKey(sk.GetPublic(), entry); err != nil { + return nil, err + } + + return entry, nil +} + +type mockValueStore struct { + r routing.ValueStore + kbook pstore.KeyBook +} + +func newMockValueStore(id testutil.Identity, dstore ds.Datastore, kbook pstore.KeyBook) *mockValueStore { + return &mockValueStore{ + r: offline.NewOfflineRouter(dstore, record.NamespacedValidator{ + "ipns": ipns.Validator{KeyBook: kbook}, + "pk": record.PublicKeyValidator{}, + }), + kbook: kbook, + } +} + +func (m *mockValueStore) GetValue(ctx context.Context, k string, opts ...routing.Option) ([]byte, error) { + return m.r.GetValue(ctx, k, opts...) +} + +func (m *mockValueStore) SearchValue(ctx context.Context, k string, opts ...routing.Option) (<-chan []byte, error) { + return m.r.SearchValue(ctx, k, opts...) +} + +func (m *mockValueStore) GetPublicKey(ctx context.Context, p peer.ID) (ci.PubKey, error) { + pk := m.kbook.PubKey(p) + if pk != nil { + return pk, nil + } + + pkkey := routing.KeyForPublicKey(p) + val, err := m.GetValue(ctx, pkkey) + if err != nil { + return nil, err + } + + pk, err = ci.UnmarshalPublicKey(val) + if err != nil { + return nil, err + } + + return pk, m.kbook.AddPubKey(p, pk) +} + +func (m *mockValueStore) PutValue(ctx context.Context, k string, d []byte, opts ...routing.Option) error { + return m.r.PutValue(ctx, k, d, opts...) +} diff --git a/namesys/namesys.go b/namesys/namesys.go new file mode 100644 index 0000000000..dbc7731906 --- /dev/null +++ b/namesys/namesys.go @@ -0,0 +1,338 @@ +// Package namesys defines Resolver and Publisher interfaces for IPNS paths, +// that is, IPFS paths in the form of /ipns/. A "resolved" +// IPNS path becomes an /ipfs/ path. +// +// Traditionally, these paths would be in the form of /ipns/peer_id, which +// references an IPNS record in a distributed ValueStore (usually the IPFS +// DHT). +// +// Additionally, the /ipns/ namespace can also be used with domain names that +// use DNSLink (/ipns/, https://docs.ipfs.io/concepts/dnslink/) +// +// The package provides implementations for all three resolvers. +package namesys + +import ( + "context" + "fmt" + "os" + "strings" + "time" + + lru "github.com/hashicorp/golang-lru" + "github.com/ipfs/go-cid" + ds "github.com/ipfs/go-datastore" + dssync "github.com/ipfs/go-datastore/sync" + iface "github.com/ipfs/go-libipfs/coreiface" + opts "github.com/ipfs/go-libipfs/coreiface/options/namesys" + "github.com/ipfs/go-libipfs/path" + ci "github.com/libp2p/go-libp2p/core/crypto" + "github.com/libp2p/go-libp2p/core/peer" + "github.com/libp2p/go-libp2p/core/routing" + "github.com/miekg/dns" + madns "github.com/multiformats/go-multiaddr-dns" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" +) + +// mpns (a multi-protocol NameSystem) implements generic IPFS naming. +// +// Uses several Resolvers: +// (a) IPFS routing naming: SFS-like PKI names. +// (b) dns domains: resolves using links in DNS TXT records +// +// It can only publish to: (a) IPFS routing naming. +type mpns struct { + ds ds.Datastore + + dnsResolver, ipnsResolver resolver + ipnsPublisher Publisher + + staticMap map[string]path.Path + cache *lru.Cache +} + +type Option func(*mpns) error + +// WithCache is an option that instructs the name system to use a (LRU) cache of the given size. +func WithCache(size int) Option { + return func(ns *mpns) error { + if size <= 0 { + return fmt.Errorf("invalid cache size %d; must be > 0", size) + } + + cache, err := lru.New(size) + if err != nil { + return err + } + + ns.cache = cache + return nil + } +} + +// WithDNSResolver is an option that supplies a custom DNS resolver to use instead of the system +// default. +func WithDNSResolver(rslv madns.BasicResolver) Option { + return func(ns *mpns) error { + ns.dnsResolver = NewDNSResolver(rslv.LookupTXT) + return nil + } +} + +// WithDatastore is an option that supplies a datastore to use instead of an in-memory map datastore. The datastore is used to store published IPNS records and make them available for querying. +func WithDatastore(ds ds.Datastore) Option { + return func(ns *mpns) error { + ns.ds = ds + return nil + } +} + +func loadStaticMap(list string) (map[string]path.Path, error) { + staticMap := make(map[string]path.Path) + for _, pair := range strings.Split(list, ",") { + mapping := strings.SplitN(pair, ":", 2) + key := mapping[0] + value := path.FromString(mapping[1]) + + ipnsKey, err := peer.Decode(key) + if err == nil { + key = iface.FormatKeyID(ipnsKey) + } + + staticMap[key] = value + } + return staticMap, nil +} + +// NewNameSystem will construct the IPFS naming system based on Routing +func NewNameSystem(r routing.ValueStore, opts ...Option) (NameSystem, error) { + var staticMap map[string]path.Path + + // Prewarm namesys cache with static records for deterministic tests and debugging. + // Useful for testing things like DNSLink without real DNS lookup. + // Example: + // IPFS_NS_MAP="dnslink-test.example.com:/ipfs/bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am" + if list := os.Getenv("IPFS_NS_MAP"); list != "" { + var err error + staticMap, err = loadStaticMap(list) + + if err != nil { + return nil, err + } + } + + ns := &mpns{ + staticMap: staticMap, + } + + for _, opt := range opts { + err := opt(ns) + if err != nil { + return nil, err + } + } + + if ns.ds == nil { + ns.ds = dssync.MutexWrap(ds.NewMapDatastore()) + } + + if ns.dnsResolver == nil { + ns.dnsResolver = NewDNSResolver(madns.DefaultResolver.LookupTXT) + } + + ns.ipnsResolver = NewIpnsResolver(r) + ns.ipnsPublisher = NewIpnsPublisher(r, ns.ds) + + return ns, nil +} + +// DefaultResolverCacheTTL defines max ttl of a record placed in namesys cache. +const DefaultResolverCacheTTL = time.Minute + +// Resolve implements Resolver. +func (ns *mpns) Resolve(ctx context.Context, name string, options ...opts.ResolveOpt) (path.Path, error) { + ctx, span := StartSpan(ctx, "MPNS.Resolve", trace.WithAttributes(attribute.String("Name", name))) + defer span.End() + + if strings.HasPrefix(name, "/ipfs/") { + return path.ParsePath(name) + } + + if !strings.HasPrefix(name, "/") { + return path.ParsePath("/ipfs/" + name) + } + + return resolve(ctx, ns, name, opts.ProcessOpts(options)) +} + +func (ns *mpns) ResolveAsync(ctx context.Context, name string, options ...opts.ResolveOpt) <-chan Result { + ctx, span := StartSpan(ctx, "MPNS.ResolveAsync", trace.WithAttributes(attribute.String("Name", name))) + defer span.End() + + if strings.HasPrefix(name, "/ipfs/") { + p, err := path.ParsePath(name) + res := make(chan Result, 1) + res <- Result{p, err} + close(res) + return res + } + + if !strings.HasPrefix(name, "/") { + p, err := path.ParsePath("/ipfs/" + name) + res := make(chan Result, 1) + res <- Result{p, err} + close(res) + return res + } + + return resolveAsync(ctx, ns, name, opts.ProcessOpts(options)) +} + +// resolveOnce implements resolver. +func (ns *mpns) resolveOnceAsync(ctx context.Context, name string, options opts.ResolveOpts) <-chan onceResult { + ctx, span := StartSpan(ctx, "MPNS.ResolveOnceAsync") + defer span.End() + + out := make(chan onceResult, 1) + + if !strings.HasPrefix(name, ipnsPrefix) { + name = ipnsPrefix + name + } + segments := strings.SplitN(name, "/", 4) + if len(segments) < 3 || segments[0] != "" { + log.Debugf("invalid name syntax for %s", name) + out <- onceResult{err: ErrResolveFailed} + close(out) + return out + } + + key := segments[2] + + // Resolver selection: + // 1. if it is a PeerID/CID/multihash resolve through "ipns". + // 2. if it is a domain name, resolve through "dns" + + var res resolver + ipnsKey, err := peer.Decode(key) + + // CIDs in IPNS are expected to have libp2p-key multicodec + // We ease the transition by returning a more meaningful error with a valid CID + if err != nil { + ipnsCid, cidErr := cid.Decode(key) + if cidErr == nil && ipnsCid.Version() == 1 && ipnsCid.Type() != cid.Libp2pKey { + fixedCid := cid.NewCidV1(cid.Libp2pKey, ipnsCid.Hash()).String() + codecErr := fmt.Errorf("peer ID represented as CIDv1 require libp2p-key multicodec: retry with /ipns/%s", fixedCid) + log.Debugf("RoutingResolver: could not convert public key hash %q to peer ID: %s\n", key, codecErr) + out <- onceResult{err: codecErr} + close(out) + return out + } + } + + cacheKey := key + if err == nil { + cacheKey = iface.FormatKeyID(ipnsKey) + } + + if p, ok := ns.cacheGet(cacheKey); ok { + var err error + if len(segments) > 3 { + p, err = path.FromSegments("", strings.TrimRight(p.String(), "/"), segments[3]) + } + span.SetAttributes(attribute.Bool("CacheHit", true)) + span.RecordError(err) + + out <- onceResult{value: p, err: err} + close(out) + return out + } + span.SetAttributes(attribute.Bool("CacheHit", false)) + + if err == nil { + res = ns.ipnsResolver + } else if _, ok := dns.IsDomainName(key); ok { + res = ns.dnsResolver + } else { + out <- onceResult{err: fmt.Errorf("invalid IPNS root: %q", key)} + close(out) + return out + } + + resCh := res.resolveOnceAsync(ctx, key, options) + var best onceResult + go func() { + defer close(out) + for { + select { + case res, ok := <-resCh: + if !ok { + if best != (onceResult{}) { + ns.cacheSet(cacheKey, best.value, best.ttl) + } + return + } + if res.err == nil { + best = res + } + p := res.value + err := res.err + ttl := res.ttl + + // Attach rest of the path + if len(segments) > 3 { + p, err = path.FromSegments("", strings.TrimRight(p.String(), "/"), segments[3]) + } + + emitOnceResult(ctx, out, onceResult{value: p, ttl: ttl, err: err}) + case <-ctx.Done(): + return + } + } + }() + + return out +} + +func emitOnceResult(ctx context.Context, outCh chan<- onceResult, r onceResult) { + select { + case outCh <- r: + case <-ctx.Done(): + } +} + +// Publish implements Publisher +func (ns *mpns) Publish(ctx context.Context, name ci.PrivKey, value path.Path, options ...opts.PublishOption) error { + ctx, span := StartSpan(ctx, "MPNS.Publish") + defer span.End() + + // This is a bit hacky. We do this because the EOL is based on the current + // time, but also needed in the end of the function. Therefore, we parse + // the options immediately and add an option PublishWithEOL with the EOL + // calculated in this moment. + publishOpts := opts.ProcessPublishOptions(options) + options = append(options, opts.PublishWithEOL(publishOpts.EOL)) + + id, err := peer.IDFromPrivateKey(name) + if err != nil { + span.RecordError(err) + return err + } + span.SetAttributes(attribute.String("ID", id.String())) + if err := ns.ipnsPublisher.Publish(ctx, name, value, options...); err != nil { + // Invalidate the cache. Publishing may _partially_ succeed but + // still return an error. + ns.cacheInvalidate(string(id)) + span.RecordError(err) + return err + } + ttl := DefaultResolverCacheTTL + if publishOpts.TTL >= 0 { + ttl = publishOpts.TTL + } + if ttEOL := time.Until(publishOpts.EOL); ttEOL < ttl { + ttl = ttEOL + } + ns.cacheSet(string(id), value, ttl) + return nil +} diff --git a/namesys/namesys_test.go b/namesys/namesys_test.go new file mode 100644 index 0000000000..bb2d5a4fe5 --- /dev/null +++ b/namesys/namesys_test.go @@ -0,0 +1,210 @@ +package namesys + +import ( + "context" + "errors" + "fmt" + "testing" + "time" + + ds "github.com/ipfs/go-datastore" + dssync "github.com/ipfs/go-datastore/sync" + opts "github.com/ipfs/go-libipfs/coreiface/options/namesys" + "github.com/ipfs/go-libipfs/ipns" + "github.com/ipfs/go-libipfs/path" + offroute "github.com/ipfs/go-libipfs/routing/offline" + record "github.com/libp2p/go-libp2p-record" + ci "github.com/libp2p/go-libp2p/core/crypto" + "github.com/libp2p/go-libp2p/core/peer" + "github.com/libp2p/go-libp2p/p2p/host/peerstore/pstoremem" +) + +type mockResolver struct { + entries map[string]string +} + +func testResolution(t *testing.T, resolver Resolver, name string, depth uint, expected string, expError error) { + t.Helper() + p, err := resolver.Resolve(context.Background(), name, opts.Depth(depth)) + if !errors.Is(err, expError) { + t.Fatal(fmt.Errorf( + "expected %s with a depth of %d to have a '%s' error, but got '%s'", + name, depth, expError, err)) + } + if p.String() != expected { + t.Fatal(fmt.Errorf( + "%s with depth %d resolved to %s != %s", + name, depth, p.String(), expected)) + } +} + +func (r *mockResolver) resolveOnceAsync(ctx context.Context, name string, options opts.ResolveOpts) <-chan onceResult { + p, err := path.ParsePath(r.entries[name]) + out := make(chan onceResult, 1) + out <- onceResult{value: p, err: err} + close(out) + return out +} + +func mockResolverOne() *mockResolver { + return &mockResolver{ + entries: map[string]string{ + "QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy": "/ipfs/Qmcqtw8FfrVSBaRmbWwHxt3AuySBhJLcvmFYi3Lbc4xnwj", + "QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n": "/ipns/QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy", + "QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD": "/ipns/ipfs.io", + "QmQ4QZh8nrsczdUEwTyfBope4THUhqxqc1fx6qYhhzZQei": "/ipfs/QmP3ouCnU8NNLsW6261pAx2pNLV2E4dQoisB1sgda12Act", + "12D3KooWFB51PRY9BxcXSH6khFXw1BZeszeLDy7C8GciskqCTZn5": "/ipns/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", // ed25519+identity multihash + "bafzbeickencdqw37dpz3ha36ewrh4undfjt2do52chtcky4rxkj447qhdm": "/ipns/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", // cidv1 in base32 with libp2p-key multicodec + }, + } +} + +func mockResolverTwo() *mockResolver { + return &mockResolver{ + entries: map[string]string{ + "ipfs.io": "/ipns/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", + }, + } +} + +func TestNamesysResolution(t *testing.T) { + r := &mpns{ + ipnsResolver: mockResolverOne(), + dnsResolver: mockResolverTwo(), + } + + testResolution(t, r, "Qmcqtw8FfrVSBaRmbWwHxt3AuySBhJLcvmFYi3Lbc4xnwj", opts.DefaultDepthLimit, "/ipfs/Qmcqtw8FfrVSBaRmbWwHxt3AuySBhJLcvmFYi3Lbc4xnwj", nil) + testResolution(t, r, "/ipns/QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy", opts.DefaultDepthLimit, "/ipfs/Qmcqtw8FfrVSBaRmbWwHxt3AuySBhJLcvmFYi3Lbc4xnwj", nil) + testResolution(t, r, "/ipns/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", opts.DefaultDepthLimit, "/ipfs/Qmcqtw8FfrVSBaRmbWwHxt3AuySBhJLcvmFYi3Lbc4xnwj", nil) + testResolution(t, r, "/ipns/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", 1, "/ipns/QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy", ErrResolveRecursion) + testResolution(t, r, "/ipns/ipfs.io", opts.DefaultDepthLimit, "/ipfs/Qmcqtw8FfrVSBaRmbWwHxt3AuySBhJLcvmFYi3Lbc4xnwj", nil) + testResolution(t, r, "/ipns/ipfs.io", 1, "/ipns/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", ErrResolveRecursion) + testResolution(t, r, "/ipns/ipfs.io", 2, "/ipns/QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy", ErrResolveRecursion) + testResolution(t, r, "/ipns/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", opts.DefaultDepthLimit, "/ipfs/Qmcqtw8FfrVSBaRmbWwHxt3AuySBhJLcvmFYi3Lbc4xnwj", nil) + testResolution(t, r, "/ipns/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", 1, "/ipns/ipfs.io", ErrResolveRecursion) + testResolution(t, r, "/ipns/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", 2, "/ipns/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", ErrResolveRecursion) + testResolution(t, r, "/ipns/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", 3, "/ipns/QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy", ErrResolveRecursion) + testResolution(t, r, "/ipns/12D3KooWFB51PRY9BxcXSH6khFXw1BZeszeLDy7C8GciskqCTZn5", 1, "/ipns/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", ErrResolveRecursion) + testResolution(t, r, "/ipns/bafzbeickencdqw37dpz3ha36ewrh4undfjt2do52chtcky4rxkj447qhdm", 1, "/ipns/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", ErrResolveRecursion) +} + +func TestNamesysResolutionWithCache(t *testing.T) { + nsMap := "dnslink-test.example.com:/ipfs/bafyaaeykceeaeeqlgiydemzngazc2mrtbimaw,12D3KooWQbpsnyzdBcxw6GUMbijV8WgXE4L8EtfnbcQWLfyxBKho:/ipfs/bafyaagakcyeaeeqqgiydemzngazc2mrtfvuxa3ttbimba,k51qzi5uqu5dkwkqm42v9j9kqcam2jiuvloi16g72i4i4amoo2m8u3ol3mqu6s:/ipfs/bafyaahikdmeaeeqvgiydemzngazc2mrtfvuxa3ttfvsgk5lybimbk" + + staticMap, err := loadStaticMap(nsMap) + if err != nil { + t.Fatal(err) + } + + r := &mpns{ + ipnsResolver: mockResolverOne(), + dnsResolver: mockResolverTwo(), + staticMap: staticMap, + } + + testResolution(t, r, "/ipns/dnslink-test.example.com", opts.DefaultDepthLimit, "/ipfs/bafyaaeykceeaeeqlgiydemzngazc2mrtbimaw", nil) + + testResolution(t, r, "/ipns/bafzaajaiaejcbw5i6oyqsktsn36r2vxgl2jzosyao46rybqztxt4rx4tfa3hpogg", opts.DefaultDepthLimit, "/ipfs/bafyaagakcyeaeeqqgiydemzngazc2mrtfvuxa3ttbimba", nil) + testResolution(t, r, "/ipns/k51qzi5uqu5dlnojhwrggtpty9c0cp5hvnkdozowth4eqb726jvoros8k9niyu", opts.DefaultDepthLimit, "/ipfs/bafyaagakcyeaeeqqgiydemzngazc2mrtfvuxa3ttbimba", nil) + testResolution(t, r, "/ipns/12D3KooWQbpsnyzdBcxw6GUMbijV8WgXE4L8EtfnbcQWLfyxBKho", opts.DefaultDepthLimit, "/ipfs/bafyaagakcyeaeeqqgiydemzngazc2mrtfvuxa3ttbimba", nil) + + testResolution(t, r, "/ipns/bafzaajaiaejcbpltl72da5f3y7ojrtsa7hsfn5bbnkjbkwyesziqqtdry6vjilku", opts.DefaultDepthLimit, "/ipfs/bafyaahikdmeaeeqvgiydemzngazc2mrtfvuxa3ttfvsgk5lybimbk", nil) + testResolution(t, r, "/ipns/k51qzi5uqu5dkwkqm42v9j9kqcam2jiuvloi16g72i4i4amoo2m8u3ol3mqu6s", opts.DefaultDepthLimit, "/ipfs/bafyaahikdmeaeeqvgiydemzngazc2mrtfvuxa3ttfvsgk5lybimbk", nil) + testResolution(t, r, "/ipns/12D3KooWNZuG8phqhoNK9KWcUhwfzA3biDKNCUNVWEaJgigr6Acj", opts.DefaultDepthLimit, "/ipfs/bafyaahikdmeaeeqvgiydemzngazc2mrtfvuxa3ttfvsgk5lybimbk", nil) + +} + +func TestPublishWithCache0(t *testing.T) { + dst := dssync.MutexWrap(ds.NewMapDatastore()) + priv, _, err := ci.GenerateKeyPair(ci.RSA, 2048) + if err != nil { + t.Fatal(err) + } + ps, err := pstoremem.NewPeerstore() + if err != nil { + t.Fatal(err) + } + pid, err := peer.IDFromPrivateKey(priv) + if err != nil { + t.Fatal(err) + } + err = ps.AddPrivKey(pid, priv) + if err != nil { + t.Fatal(err) + } + + routing := offroute.NewOfflineRouter(dst, record.NamespacedValidator{ + "ipns": ipns.Validator{KeyBook: ps}, + "pk": record.PublicKeyValidator{}, + }) + + nsys, err := NewNameSystem(routing, WithDatastore(dst)) + if err != nil { + t.Fatal(err) + } + + // CID is arbitrary. + p, err := path.ParsePath("QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn") + if err != nil { + t.Fatal(err) + } + err = nsys.Publish(context.Background(), priv, p) + if err != nil { + t.Fatal(err) + } +} + +func TestPublishWithTTL(t *testing.T) { + dst := dssync.MutexWrap(ds.NewMapDatastore()) + priv, _, err := ci.GenerateKeyPair(ci.RSA, 2048) + if err != nil { + t.Fatal(err) + } + ps, err := pstoremem.NewPeerstore() + if err != nil { + t.Fatal(err) + } + pid, err := peer.IDFromPrivateKey(priv) + if err != nil { + t.Fatal(err) + } + err = ps.AddPrivKey(pid, priv) + if err != nil { + t.Fatal(err) + } + + routing := offroute.NewOfflineRouter(dst, record.NamespacedValidator{ + "ipns": ipns.Validator{KeyBook: ps}, + "pk": record.PublicKeyValidator{}, + }) + + nsys, err := NewNameSystem(routing, WithDatastore(dst), WithCache(128)) + if err != nil { + t.Fatal(err) + } + + // CID is arbitrary. + p, err := path.ParsePath("QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn") + if err != nil { + t.Fatal(err) + } + + ttl := 1 * time.Second + eol := time.Now().Add(2 * time.Second) + + err = nsys.Publish(context.Background(), priv, p, opts.PublishWithEOL(eol), opts.PublishWithTTL(ttl)) + if err != nil { + t.Fatal(err) + } + ientry, ok := nsys.(*mpns).cache.Get(string(pid)) + if !ok { + t.Fatal("cache get failed") + } + entry, ok := ientry.(cacheEntry) + if !ok { + t.Fatal("bad cache item returned") + } + if entry.eol.Sub(eol) > 10*time.Millisecond { + t.Fatalf("bad cache ttl: expected %s, got %s", eol, entry.eol) + } +} diff --git a/namesys/publisher.go b/namesys/publisher.go new file mode 100644 index 0000000000..c37f0e0a1f --- /dev/null +++ b/namesys/publisher.go @@ -0,0 +1,285 @@ +package namesys + +import ( + "context" + "strings" + "sync" + "time" + + "github.com/gogo/protobuf/proto" + ds "github.com/ipfs/go-datastore" + dsquery "github.com/ipfs/go-datastore/query" + opts "github.com/ipfs/go-libipfs/coreiface/options/namesys" + "github.com/ipfs/go-libipfs/ipns" + pb "github.com/ipfs/go-libipfs/ipns/pb" + "github.com/ipfs/go-libipfs/path" + "github.com/libp2p/go-libp2p/core/crypto" + "github.com/libp2p/go-libp2p/core/peer" + "github.com/libp2p/go-libp2p/core/routing" + "github.com/whyrusleeping/base32" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" +) + +const ipnsPrefix = "/ipns/" + +// IpnsPublisher is capable of publishing and resolving names to the IPFS +// routing system. +type IpnsPublisher struct { + routing routing.ValueStore + ds ds.Datastore + + // Used to ensure we assign IPNS records *sequential* sequence numbers. + mu sync.Mutex +} + +// NewIpnsPublisher constructs a publisher for the IPFS Routing name system. +func NewIpnsPublisher(route routing.ValueStore, ds ds.Datastore) *IpnsPublisher { + if ds == nil { + panic("nil datastore") + } + return &IpnsPublisher{routing: route, ds: ds} +} + +// Publish implements Publisher. Accepts a keypair and a value, +// and publishes it out to the routing system +func (p *IpnsPublisher) Publish(ctx context.Context, k crypto.PrivKey, value path.Path, options ...opts.PublishOption) error { + log.Debugf("Publish %s", value) + + ctx, span := StartSpan(ctx, "IpnsPublisher.Publish", trace.WithAttributes(attribute.String("Value", value.String()))) + defer span.End() + + record, err := p.updateRecord(ctx, k, value, options...) + if err != nil { + return err + } + + return PutRecordToRouting(ctx, p.routing, k.GetPublic(), record) +} + +// IpnsDsKey returns a datastore key given an IPNS identifier (peer +// ID). Defines the storage key for IPNS records in the local datastore. +func IpnsDsKey(id peer.ID) ds.Key { + return ds.NewKey("/ipns/" + base32.RawStdEncoding.EncodeToString([]byte(id))) +} + +// ListPublished returns the latest IPNS records published by this node and +// their expiration times. +// +// This method will not search the routing system for records published by other +// nodes. +func (p *IpnsPublisher) ListPublished(ctx context.Context) (map[peer.ID]*pb.IpnsEntry, error) { + query, err := p.ds.Query(ctx, dsquery.Query{ + Prefix: ipnsPrefix, + }) + if err != nil { + return nil, err + } + defer query.Close() + + records := make(map[peer.ID]*pb.IpnsEntry) + for { + select { + case result, ok := <-query.Next(): + if !ok { + return records, nil + } + if result.Error != nil { + return nil, result.Error + } + e := new(pb.IpnsEntry) + if err := proto.Unmarshal(result.Value, e); err != nil { + // Might as well return what we can. + log.Error("found an invalid IPNS entry:", err) + continue + } + if !strings.HasPrefix(result.Key, ipnsPrefix) { + log.Errorf("datastore query for keys with prefix %s returned a key: %s", ipnsPrefix, result.Key) + continue + } + k := result.Key[len(ipnsPrefix):] + pid, err := base32.RawStdEncoding.DecodeString(k) + if err != nil { + log.Errorf("ipns ds key invalid: %s", result.Key) + continue + } + records[peer.ID(pid)] = e + case <-ctx.Done(): + return nil, ctx.Err() + } + } +} + +// GetPublished returns the record this node has published corresponding to the +// given peer ID. +// +// If `checkRouting` is true and we have no existing record, this method will +// check the routing system for any existing records. +func (p *IpnsPublisher) GetPublished(ctx context.Context, id peer.ID, checkRouting bool) (*pb.IpnsEntry, error) { + ctx, cancel := context.WithTimeout(ctx, time.Second*30) + defer cancel() + + value, err := p.ds.Get(ctx, IpnsDsKey(id)) + switch err { + case nil: + case ds.ErrNotFound: + if !checkRouting { + return nil, nil + } + ipnskey := ipns.RecordKey(id) + value, err = p.routing.GetValue(ctx, ipnskey) + if err != nil { + // Not found or other network issue. Can't really do + // anything about this case. + if err != routing.ErrNotFound { + log.Debugf("error when determining the last published IPNS record for %s: %s", id, err) + } + + return nil, nil + } + default: + return nil, err + } + e := new(pb.IpnsEntry) + if err := proto.Unmarshal(value, e); err != nil { + return nil, err + } + return e, nil +} + +func (p *IpnsPublisher) updateRecord(ctx context.Context, k crypto.PrivKey, value path.Path, options ...opts.PublishOption) (*pb.IpnsEntry, error) { + id, err := peer.IDFromPrivateKey(k) + if err != nil { + return nil, err + } + + p.mu.Lock() + defer p.mu.Unlock() + + // get previous records sequence number + rec, err := p.GetPublished(ctx, id, true) + if err != nil { + return nil, err + } + + seqno := rec.GetSequence() // returns 0 if rec is nil + if rec != nil && value != path.Path(rec.GetValue()) { + // Don't bother incrementing the sequence number unless the + // value changes. + seqno++ + } + + opts := opts.ProcessPublishOptions(options) + + // Create record + entry, err := ipns.Create(k, []byte(value), seqno, opts.EOL, opts.TTL) + if err != nil { + return nil, err + } + + data, err := proto.Marshal(entry) + if err != nil { + return nil, err + } + + // Put the new record. + key := IpnsDsKey(id) + if err := p.ds.Put(ctx, key, data); err != nil { + return nil, err + } + if err := p.ds.Sync(ctx, key); err != nil { + return nil, err + } + return entry, nil +} + +// PutRecordToRouting publishes the given entry using the provided ValueStore, +// keyed on the ID associated with the provided public key. The public key is +// also made available to the routing system so that entries can be verified. +func PutRecordToRouting(ctx context.Context, r routing.ValueStore, k crypto.PubKey, entry *pb.IpnsEntry) error { + ctx, span := StartSpan(ctx, "PutRecordToRouting") + defer span.End() + + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + errs := make(chan error, 2) // At most two errors (IPNS, and public key) + + if err := ipns.EmbedPublicKey(k, entry); err != nil { + return err + } + + id, err := peer.IDFromPublicKey(k) + if err != nil { + return err + } + + go func() { + errs <- PublishEntry(ctx, r, ipns.RecordKey(id), entry) + }() + + // Publish the public key if a public key cannot be extracted from the ID + // TODO: once v0.4.16 is widespread enough, we can stop doing this + // and at that point we can even deprecate the /pk/ namespace in the dht + // + // NOTE: This check actually checks if the public key has been embedded + // in the IPNS entry. This check is sufficient because we embed the + // public key in the IPNS entry if it can't be extracted from the ID. + if entry.PubKey != nil { + go func() { + errs <- PublishPublicKey(ctx, r, PkKeyForID(id), k) + }() + + if err := waitOnErrChan(ctx, errs); err != nil { + return err + } + } + + return waitOnErrChan(ctx, errs) +} + +func waitOnErrChan(ctx context.Context, errs chan error) error { + select { + case err := <-errs: + return err + case <-ctx.Done(): + return ctx.Err() + } +} + +// PublishPublicKey stores the given public key in the ValueStore with the +// given key. +func PublishPublicKey(ctx context.Context, r routing.ValueStore, k string, pubk crypto.PubKey) error { + ctx, span := StartSpan(ctx, "PublishPublicKey", trace.WithAttributes(attribute.String("Key", k))) + defer span.End() + + log.Debugf("Storing pubkey at: %s", k) + pkbytes, err := crypto.MarshalPublicKey(pubk) + if err != nil { + return err + } + + // Store associated public key + return r.PutValue(ctx, k, pkbytes) +} + +// PublishEntry stores the given IpnsEntry in the ValueStore with the given +// ipnskey. +func PublishEntry(ctx context.Context, r routing.ValueStore, ipnskey string, rec *pb.IpnsEntry) error { + ctx, span := StartSpan(ctx, "PublishEntry", trace.WithAttributes(attribute.String("IPNSKey", ipnskey))) + defer span.End() + + data, err := proto.Marshal(rec) + if err != nil { + return err + } + + log.Debugf("Storing ipns entry at: %x", ipnskey) + // Store ipns entry at "/ipns/"+h(pubkey) + return r.PutValue(ctx, ipnskey, data) +} + +// PkKeyForID returns the public key routing key for the given peer ID. +func PkKeyForID(id peer.ID) string { + return "/pk/" + string(id) +} diff --git a/namesys/publisher_test.go b/namesys/publisher_test.go new file mode 100644 index 0000000000..48a4042ac0 --- /dev/null +++ b/namesys/publisher_test.go @@ -0,0 +1,156 @@ +package namesys + +import ( + "context" + "crypto/rand" + "testing" + "time" + + "github.com/ipfs/go-libipfs/path" + + ds "github.com/ipfs/go-datastore" + dssync "github.com/ipfs/go-datastore/sync" + dshelp "github.com/ipfs/go-libipfs/datastore/dshelp" + "github.com/ipfs/go-libipfs/ipns" + mockrouting "github.com/ipfs/go-libipfs/routing/mock" + testutil "github.com/libp2p/go-libp2p-testing/net" + ci "github.com/libp2p/go-libp2p/core/crypto" + "github.com/libp2p/go-libp2p/core/peer" + ma "github.com/multiformats/go-multiaddr" +) + +type identity struct { + testutil.PeerNetParams +} + +func (p *identity) ID() peer.ID { + return p.PeerNetParams.ID +} + +func (p *identity) Address() ma.Multiaddr { + return p.Addr +} + +func (p *identity) PrivateKey() ci.PrivKey { + return p.PrivKey +} + +func (p *identity) PublicKey() ci.PubKey { + return p.PubKey +} + +func testNamekeyPublisher(t *testing.T, keyType int, expectedErr error, expectedExistence bool) { + // Context + ctx := context.Background() + + // Private key + privKey, pubKey, err := ci.GenerateKeyPairWithReader(keyType, 2048, rand.Reader) + if err != nil { + t.Fatal(err) + } + + // ID + id, err := peer.IDFromPublicKey(pubKey) + if err != nil { + t.Fatal(err) + } + + // Value + value := []byte("ipfs/TESTING") + + // Seqnum + seqnum := uint64(0) + + // Eol + eol := time.Now().Add(24 * time.Hour) + + // Routing value store + p := testutil.PeerNetParams{ + ID: id, + PrivKey: privKey, + PubKey: pubKey, + Addr: testutil.ZeroLocalTCPAddress, + } + + dstore := dssync.MutexWrap(ds.NewMapDatastore()) + serv := mockrouting.NewServer() + r := serv.ClientWithDatastore(context.Background(), &identity{p}, dstore) + + entry, err := ipns.Create(privKey, value, seqnum, eol, 0) + if err != nil { + t.Fatal(err) + } + + err = PutRecordToRouting(ctx, r, pubKey, entry) + if err != nil { + t.Fatal(err) + } + + // Check for namekey existence in value store + namekey := PkKeyForID(id) + _, err = r.GetValue(ctx, namekey) + if err != expectedErr { + t.Fatal(err) + } + + // Also check datastore for completeness + key := dshelp.NewKeyFromBinary([]byte(namekey)) + exists, err := dstore.Has(ctx, key) + if err != nil { + t.Fatal(err) + } + + if exists != expectedExistence { + t.Fatal("Unexpected key existence in datastore") + } +} + +func TestRSAPublisher(t *testing.T) { + testNamekeyPublisher(t, ci.RSA, nil, true) +} + +func TestEd22519Publisher(t *testing.T) { + testNamekeyPublisher(t, ci.Ed25519, ds.ErrNotFound, false) +} + +func TestAsyncDS(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + rt := mockrouting.NewServer().Client(testutil.RandIdentityOrFatal(t)) + ds := &checkSyncDS{ + Datastore: ds.NewMapDatastore(), + syncKeys: make(map[ds.Key]struct{}), + } + publisher := NewIpnsPublisher(rt, ds) + + ipnsFakeID := testutil.RandIdentityOrFatal(t) + ipnsVal, err := path.ParsePath("/ipns/foo.bar") + if err != nil { + t.Fatal(err) + } + + if err := publisher.Publish(ctx, ipnsFakeID.PrivateKey(), ipnsVal); err != nil { + t.Fatal(err) + } + + ipnsKey := IpnsDsKey(ipnsFakeID.ID()) + + for k := range ds.syncKeys { + if k.IsAncestorOf(ipnsKey) || k.Equal(ipnsKey) { + return + } + } + + t.Fatal("ipns key not synced") +} + +type checkSyncDS struct { + ds.Datastore + syncKeys map[ds.Key]struct{} +} + +func (d *checkSyncDS) Sync(ctx context.Context, prefix ds.Key) error { + d.syncKeys[prefix] = struct{}{} + return d.Datastore.Sync(ctx, prefix) +} diff --git a/namesys/republisher/repub.go b/namesys/republisher/repub.go new file mode 100644 index 0000000000..19595f355a --- /dev/null +++ b/namesys/republisher/repub.go @@ -0,0 +1,186 @@ +// Package republisher provides a utility to automatically re-publish IPNS +// records related to the keys in a Keystore. +package republisher + +import ( + "context" + "errors" + "time" + + keystore "github.com/ipfs/go-libipfs/keystore" + "github.com/ipfs/go-libipfs/namesys" + "github.com/ipfs/go-libipfs/path" + "go.opentelemetry.io/otel/attribute" + + "github.com/gogo/protobuf/proto" + ds "github.com/ipfs/go-datastore" + opts "github.com/ipfs/go-libipfs/coreiface/options/namesys" + "github.com/ipfs/go-libipfs/ipns" + pb "github.com/ipfs/go-libipfs/ipns/pb" + logging "github.com/ipfs/go-log" + "github.com/jbenet/goprocess" + gpctx "github.com/jbenet/goprocess/context" + ic "github.com/libp2p/go-libp2p/core/crypto" + "github.com/libp2p/go-libp2p/core/peer" +) + +var errNoEntry = errors.New("no previous entry") + +var log = logging.Logger("ipns-repub") + +// DefaultRebroadcastInterval is the default interval at which we rebroadcast IPNS records +var DefaultRebroadcastInterval = time.Hour * 4 + +// InitialRebroadcastDelay is the delay before first broadcasting IPNS records on start +var InitialRebroadcastDelay = time.Minute * 1 + +// FailureRetryInterval is the interval at which we retry IPNS records broadcasts (when they fail) +var FailureRetryInterval = time.Minute * 5 + +// DefaultRecordLifetime is the default lifetime for IPNS records +const DefaultRecordLifetime = time.Hour * 24 + +// Republisher facilitates the regular publishing of all the IPNS records +// associated to keys in a Keystore. +type Republisher struct { + ns namesys.Publisher + ds ds.Datastore + self ic.PrivKey + ks keystore.Keystore + + Interval time.Duration + + // how long records that are republished should be valid for + RecordLifetime time.Duration +} + +// NewRepublisher creates a new Republisher +func NewRepublisher(ns namesys.Publisher, ds ds.Datastore, self ic.PrivKey, ks keystore.Keystore) *Republisher { + return &Republisher{ + ns: ns, + ds: ds, + self: self, + ks: ks, + Interval: DefaultRebroadcastInterval, + RecordLifetime: DefaultRecordLifetime, + } +} + +// Run starts the republisher facility. It can be stopped by stopping the +// provided proc. +func (rp *Republisher) Run(proc goprocess.Process) { + timer := time.NewTimer(InitialRebroadcastDelay) + defer timer.Stop() + if rp.Interval < InitialRebroadcastDelay { + timer.Reset(rp.Interval) + } + + for { + select { + case <-timer.C: + timer.Reset(rp.Interval) + err := rp.republishEntries(proc) + if err != nil { + log.Info("republisher failed to republish: ", err) + if FailureRetryInterval < rp.Interval { + timer.Reset(FailureRetryInterval) + } + } + case <-proc.Closing(): + return + } + } +} + +func (rp *Republisher) republishEntries(p goprocess.Process) error { + ctx, cancel := context.WithCancel(gpctx.OnClosingContext(p)) + defer cancel() + ctx, span := namesys.StartSpan(ctx, "Republisher.RepublishEntries") + defer span.End() + + // TODO: Use rp.ipns.ListPublished(). We can't currently *do* that + // because: + // 1. There's no way to get keys from the keystore by ID. + // 2. We don't actually have access to the IPNS publisher. + err := rp.republishEntry(ctx, rp.self) + if err != nil { + return err + } + + if rp.ks != nil { + keyNames, err := rp.ks.List() + if err != nil { + return err + } + for _, name := range keyNames { + priv, err := rp.ks.Get(name) + if err != nil { + return err + } + err = rp.republishEntry(ctx, priv) + if err != nil { + return err + } + + } + } + + return nil +} + +func (rp *Republisher) republishEntry(ctx context.Context, priv ic.PrivKey) error { + ctx, span := namesys.StartSpan(ctx, "Republisher.RepublishEntry") + defer span.End() + id, err := peer.IDFromPrivateKey(priv) + if err != nil { + span.RecordError(err) + return err + } + + log.Debugf("republishing ipns entry for %s", id) + + // Look for it locally only + e, err := rp.getLastIPNSEntry(ctx, id) + if err != nil { + if err == errNoEntry { + span.SetAttributes(attribute.Bool("NoEntry", true)) + return nil + } + span.RecordError(err) + return err + } + + p := path.Path(e.GetValue()) + prevEol, err := ipns.GetEOL(e) + if err != nil { + span.RecordError(err) + return err + } + + // update record with same sequence number + eol := time.Now().Add(rp.RecordLifetime) + if prevEol.After(eol) { + eol = prevEol + } + err = rp.ns.Publish(ctx, priv, p, opts.PublishWithEOL(eol)) + span.RecordError(err) + return err +} + +func (rp *Republisher) getLastIPNSEntry(ctx context.Context, id peer.ID) (*pb.IpnsEntry, error) { + // Look for it locally only + val, err := rp.ds.Get(ctx, namesys.IpnsDsKey(id)) + switch err { + case nil: + case ds.ErrNotFound: + return nil, errNoEntry + default: + return nil, err + } + + e := new(pb.IpnsEntry) + if err := proto.Unmarshal(val, e); err != nil { + return nil, err + } + return e, nil +} diff --git a/namesys/republisher/repub_test.go b/namesys/republisher/repub_test.go new file mode 100644 index 0000000000..1bbb4a63e3 --- /dev/null +++ b/namesys/republisher/repub_test.go @@ -0,0 +1,266 @@ +package republisher_test + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/gogo/protobuf/proto" + + "github.com/jbenet/goprocess" + "github.com/libp2p/go-libp2p" + dht "github.com/libp2p/go-libp2p-kad-dht" + ic "github.com/libp2p/go-libp2p/core/crypto" + host "github.com/libp2p/go-libp2p/core/host" + peer "github.com/libp2p/go-libp2p/core/peer" + routing "github.com/libp2p/go-libp2p/core/routing" + + ds "github.com/ipfs/go-datastore" + dssync "github.com/ipfs/go-datastore/sync" + opts "github.com/ipfs/go-libipfs/coreiface/options/namesys" + "github.com/ipfs/go-libipfs/ipns" + ipns_pb "github.com/ipfs/go-libipfs/ipns/pb" + "github.com/ipfs/go-libipfs/path" + + keystore "github.com/ipfs/go-libipfs/keystore" + "github.com/ipfs/go-libipfs/namesys" + . "github.com/ipfs/go-libipfs/namesys/republisher" +) + +type mockNode struct { + h host.Host + id string + privKey ic.PrivKey + store ds.Batching + dht *dht.IpfsDHT + keystore keystore.Keystore +} + +func getMockNode(t *testing.T, ctx context.Context) *mockNode { + t.Helper() + + dstore := dssync.MutexWrap(ds.NewMapDatastore()) + var idht *dht.IpfsDHT + h, err := libp2p.New( + libp2p.ListenAddrStrings("/ip4/127.0.0.1/tcp/0"), + libp2p.Routing(func(h host.Host) (routing.PeerRouting, error) { + rt, err := dht.New(ctx, h, dht.Mode(dht.ModeServer)) + idht = rt + return rt, err + }), + ) + if err != nil { + t.Fatal(err) + } + + return &mockNode{ + h: h, + id: h.ID().Pretty(), + privKey: h.Peerstore().PrivKey(h.ID()), + store: dstore, + dht: idht, + keystore: keystore.NewMemKeystore(), + } +} + +func TestRepublish(t *testing.T) { + // set cache life to zero for testing low-period repubs + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + var nsystems []namesys.NameSystem + var nodes []*mockNode + for i := 0; i < 10; i++ { + n := getMockNode(t, ctx) + ns, err := namesys.NewNameSystem(n.dht, namesys.WithDatastore(n.store)) + if err != nil { + t.Fatal(err) + } + + nsystems = append(nsystems, ns) + nodes = append(nodes, n) + } + + pinfo := host.InfoFromHost(nodes[0].h) + + for _, n := range nodes[1:] { + if err := n.h.Connect(ctx, *pinfo); err != nil { + t.Fatal(err) + } + } + + // have one node publish a record that is valid for 1 second + publisher := nodes[3] + + p := path.FromString("/ipfs/QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn") // does not need to be valid + rp := namesys.NewIpnsPublisher(publisher.dht, publisher.store) + name := "/ipns/" + publisher.id + + // Retry in case the record expires before we can fetch it. This can + // happen when running the test on a slow machine. + var expiration time.Time + timeout := time.Second + for { + expiration = time.Now().Add(time.Second) + err := rp.Publish(ctx, publisher.privKey, p, opts.PublishWithEOL(expiration)) + if err != nil { + t.Fatal(err) + } + + err = verifyResolution(nsystems, name, p) + if err == nil { + break + } + + if time.Now().After(expiration) { + timeout *= 2 + continue + } + t.Fatal(err) + } + + // Now wait a second, the records will be invalid and we should fail to resolve + time.Sleep(timeout) + if err := verifyResolutionFails(nsystems, name); err != nil { + t.Fatal(err) + } + + // The republishers that are contained within the nodes have their timeout set + // to 12 hours. Instead of trying to tweak those, we're just going to pretend + // they don't exist and make our own. + repub := NewRepublisher(rp, publisher.store, publisher.privKey, publisher.keystore) + repub.Interval = time.Second + repub.RecordLifetime = time.Second * 5 + + proc := goprocess.Go(repub.Run) + defer proc.Close() + + // now wait a couple seconds for it to fire + time.Sleep(time.Second * 2) + + // we should be able to resolve them now + if err := verifyResolution(nsystems, name, p); err != nil { + t.Fatal(err) + } +} + +func TestLongEOLRepublish(t *testing.T) { + // set cache life to zero for testing low-period repubs + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + var nsystems []namesys.NameSystem + var nodes []*mockNode + for i := 0; i < 10; i++ { + n := getMockNode(t, ctx) + ns, err := namesys.NewNameSystem(n.dht, namesys.WithDatastore(n.store)) + if err != nil { + t.Fatal(err) + } + + nsystems = append(nsystems, ns) + nodes = append(nodes, n) + } + + pinfo := host.InfoFromHost(nodes[0].h) + + for _, n := range nodes[1:] { + if err := n.h.Connect(ctx, *pinfo); err != nil { + t.Fatal(err) + } + } + + // have one node publish a record that is valid for 1 second + publisher := nodes[3] + p := path.FromString("/ipfs/QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn") // does not need to be valid + rp := namesys.NewIpnsPublisher(publisher.dht, publisher.store) + name := "/ipns/" + publisher.id + + expiration := time.Now().Add(time.Hour) + err := rp.Publish(ctx, publisher.privKey, p, opts.PublishWithEOL(expiration)) + if err != nil { + t.Fatal(err) + } + + err = verifyResolution(nsystems, name, p) + if err != nil { + t.Fatal(err) + } + + // The republishers that are contained within the nodes have their timeout set + // to 12 hours. Instead of trying to tweak those, we're just going to pretend + // they don't exist and make our own. + repub := NewRepublisher(rp, publisher.store, publisher.privKey, publisher.keystore) + repub.Interval = time.Millisecond * 500 + repub.RecordLifetime = time.Second + + proc := goprocess.Go(repub.Run) + defer proc.Close() + + // now wait a couple seconds for it to fire a few times + time.Sleep(time.Second * 2) + + err = verifyResolution(nsystems, name, p) + if err != nil { + t.Fatal(err) + } + + entry, err := getLastIPNSEntry(ctx, publisher.store, publisher.h.ID()) + if err != nil { + t.Fatal(err) + } + + finalEol, err := ipns.GetEOL(entry) + if err != nil { + t.Fatal(err) + } + + if !finalEol.Equal(expiration) { + t.Fatal("expiration time modified") + } +} + +func getLastIPNSEntry(ctx context.Context, dstore ds.Datastore, id peer.ID) (*ipns_pb.IpnsEntry, error) { + // Look for it locally only + val, err := dstore.Get(ctx, namesys.IpnsDsKey(id)) + if err != nil { + return nil, err + } + + e := new(ipns_pb.IpnsEntry) + if err := proto.Unmarshal(val, e); err != nil { + return nil, err + } + return e, nil +} + +func verifyResolution(nsystems []namesys.NameSystem, key string, exp path.Path) error { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + for _, n := range nsystems { + val, err := n.Resolve(ctx, key) + if err != nil { + return err + } + + if val != exp { + return errors.New("resolved wrong record") + } + } + return nil +} + +func verifyResolutionFails(nsystems []namesys.NameSystem, key string) error { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + for _, n := range nsystems { + _, err := n.Resolve(ctx, key) + if err == nil { + return errors.New("expected resolution to fail") + } + } + return nil +} diff --git a/namesys/resolve/resolve.go b/namesys/resolve/resolve.go new file mode 100644 index 0000000000..1b038a8f1a --- /dev/null +++ b/namesys/resolve/resolve.go @@ -0,0 +1,56 @@ +package resolve + +import ( + "context" + "errors" + "fmt" + "strings" + + "github.com/ipfs/go-libipfs/path" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" + + "github.com/ipfs/go-libipfs/namesys" +) + +// ErrNoNamesys is an explicit error for when an IPFS node doesn't +// (yet) have a name system +var ErrNoNamesys = errors.New( + "core/resolve: no Namesys on IpfsNode - can't resolve ipns entry") + +// ResolveIPNS resolves /ipns paths +func ResolveIPNS(ctx context.Context, nsys namesys.NameSystem, p path.Path) (path.Path, error) { + ctx, span := namesys.StartSpan(ctx, "ResolveIPNS", trace.WithAttributes(attribute.String("Path", p.String()))) + defer span.End() + if strings.HasPrefix(p.String(), "/ipns/") { + // TODO(cryptix): we should be able to query the local cache for the path + if nsys == nil { + return "", ErrNoNamesys + } + + seg := p.Segments() + + if len(seg) < 2 || seg[1] == "" { // just "/" without further segments + err := fmt.Errorf("invalid path %q: ipns path missing IPNS ID", p) + return "", err + } + + extensions := seg[2:] + resolvable, err := path.FromSegments("/", seg[0], seg[1]) + if err != nil { + return "", err + } + + respath, err := nsys.Resolve(ctx, resolvable.String()) + if err != nil { + return "", err + } + + segments := append(respath.Segments(), extensions...) + p, err = path.FromSegments("/", segments...) + if err != nil { + return "", err + } + } + return p, nil +} diff --git a/namesys/resolve_test.go b/namesys/resolve_test.go new file mode 100644 index 0000000000..f407d3779c --- /dev/null +++ b/namesys/resolve_test.go @@ -0,0 +1,122 @@ +package namesys + +import ( + "context" + "errors" + "testing" + "time" + + ds "github.com/ipfs/go-datastore" + dssync "github.com/ipfs/go-datastore/sync" + ipns "github.com/ipfs/go-libipfs/ipns" + path "github.com/ipfs/go-libipfs/path" + mockrouting "github.com/ipfs/go-libipfs/routing/mock" + tnet "github.com/libp2p/go-libp2p-testing/net" +) + +func TestRoutingResolve(t *testing.T) { + dstore := dssync.MutexWrap(ds.NewMapDatastore()) + serv := mockrouting.NewServer() + id := tnet.RandIdentityOrFatal(t) + d := serv.ClientWithDatastore(context.Background(), id, dstore) + + resolver := NewIpnsResolver(d) + publisher := NewIpnsPublisher(d, dstore) + + identity := tnet.RandIdentityOrFatal(t) + + h := path.FromString("/ipfs/QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN") + err := publisher.Publish(context.Background(), identity.PrivateKey(), h) + if err != nil { + t.Fatal(err) + } + + res, err := resolver.Resolve(context.Background(), identity.ID().Pretty()) + if err != nil { + t.Fatal(err) + } + + if res != h { + t.Fatal("Got back incorrect value.") + } +} + +func TestPrexistingExpiredRecord(t *testing.T) { + dstore := dssync.MutexWrap(ds.NewMapDatastore()) + d := mockrouting.NewServer().ClientWithDatastore(context.Background(), tnet.RandIdentityOrFatal(t), dstore) + + resolver := NewIpnsResolver(d) + publisher := NewIpnsPublisher(d, dstore) + + identity := tnet.RandIdentityOrFatal(t) + + // Make an expired record and put it in the datastore + h := path.FromString("/ipfs/QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN") + eol := time.Now().Add(time.Hour * -1) + + entry, err := ipns.Create(identity.PrivateKey(), []byte(h), 0, eol, 0) + if err != nil { + t.Fatal(err) + } + err = PutRecordToRouting(context.Background(), d, identity.PublicKey(), entry) + if err != nil { + t.Fatal(err) + } + + // Now, with an old record in the system already, try and publish a new one + err = publisher.Publish(context.Background(), identity.PrivateKey(), h) + if err != nil { + t.Fatal(err) + } + + err = verifyCanResolve(resolver, identity.ID().Pretty(), h) + if err != nil { + t.Fatal(err) + } +} + +func TestPrexistingRecord(t *testing.T) { + dstore := dssync.MutexWrap(ds.NewMapDatastore()) + d := mockrouting.NewServer().ClientWithDatastore(context.Background(), tnet.RandIdentityOrFatal(t), dstore) + + resolver := NewIpnsResolver(d) + publisher := NewIpnsPublisher(d, dstore) + + identity := tnet.RandIdentityOrFatal(t) + + // Make a good record and put it in the datastore + h := path.FromString("/ipfs/QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN") + eol := time.Now().Add(time.Hour) + entry, err := ipns.Create(identity.PrivateKey(), []byte(h), 0, eol, 0) + if err != nil { + t.Fatal(err) + } + err = PutRecordToRouting(context.Background(), d, identity.PublicKey(), entry) + if err != nil { + t.Fatal(err) + } + + // Now, with an old record in the system already, try and publish a new one + err = publisher.Publish(context.Background(), identity.PrivateKey(), h) + if err != nil { + t.Fatal(err) + } + + err = verifyCanResolve(resolver, identity.ID().Pretty(), h) + if err != nil { + t.Fatal(err) + } +} + +func verifyCanResolve(r Resolver, name string, exp path.Path) error { + res, err := r.Resolve(context.Background(), name) + if err != nil { + return err + } + + if res != exp { + return errors.New("got back wrong record") + } + + return nil +} diff --git a/namesys/routing.go b/namesys/routing.go new file mode 100644 index 0000000000..3f556e09cc --- /dev/null +++ b/namesys/routing.go @@ -0,0 +1,160 @@ +package namesys + +import ( + "context" + "strings" + "time" + + "github.com/gogo/protobuf/proto" + "github.com/ipfs/go-cid" + opts "github.com/ipfs/go-libipfs/coreiface/options/namesys" + "github.com/ipfs/go-libipfs/ipns" + pb "github.com/ipfs/go-libipfs/ipns/pb" + "github.com/ipfs/go-libipfs/path" + logging "github.com/ipfs/go-log" + dht "github.com/libp2p/go-libp2p-kad-dht" + "github.com/libp2p/go-libp2p/core/peer" + "github.com/libp2p/go-libp2p/core/routing" + mh "github.com/multiformats/go-multihash" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" +) + +var log = logging.Logger("namesys") + +// IpnsResolver implements NSResolver for the main IPFS SFS-like naming +type IpnsResolver struct { + routing routing.ValueStore +} + +// NewIpnsResolver constructs a name resolver using the IPFS Routing system +// to implement SFS-like naming on top. +func NewIpnsResolver(route routing.ValueStore) *IpnsResolver { + if route == nil { + panic("attempt to create resolver with nil routing system") + } + return &IpnsResolver{ + routing: route, + } +} + +// Resolve implements Resolver. +func (r *IpnsResolver) Resolve(ctx context.Context, name string, options ...opts.ResolveOpt) (path.Path, error) { + ctx, span := StartSpan(ctx, "IpnsResolver.Resolve", trace.WithAttributes(attribute.String("Name", name))) + defer span.End() + return resolve(ctx, r, name, opts.ProcessOpts(options)) +} + +// ResolveAsync implements Resolver. +func (r *IpnsResolver) ResolveAsync(ctx context.Context, name string, options ...opts.ResolveOpt) <-chan Result { + ctx, span := StartSpan(ctx, "IpnsResolver.ResolveAsync", trace.WithAttributes(attribute.String("Name", name))) + defer span.End() + return resolveAsync(ctx, r, name, opts.ProcessOpts(options)) +} + +// resolveOnce implements resolver. Uses the IPFS routing system to +// resolve SFS-like names. +func (r *IpnsResolver) resolveOnceAsync(ctx context.Context, name string, options opts.ResolveOpts) <-chan onceResult { + ctx, span := StartSpan(ctx, "IpnsResolver.ResolveOnceAsync", trace.WithAttributes(attribute.String("Name", name))) + defer span.End() + + out := make(chan onceResult, 1) + log.Debugf("RoutingResolver resolving %s", name) + cancel := func() {} + + if options.DhtTimeout != 0 { + // Resolution must complete within the timeout + ctx, cancel = context.WithTimeout(ctx, options.DhtTimeout) + } + + name = strings.TrimPrefix(name, "/ipns/") + + pid, err := peer.Decode(name) + if err != nil { + log.Debugf("RoutingResolver: could not convert public key hash %s to peer ID: %s\n", name, err) + out <- onceResult{err: err} + close(out) + cancel() + return out + } + + // Use the routing system to get the name. + // Note that the DHT will call the ipns validator when retrieving + // the value, which in turn verifies the ipns record signature + ipnsKey := ipns.RecordKey(pid) + + vals, err := r.routing.SearchValue(ctx, ipnsKey, dht.Quorum(int(options.DhtRecordCount))) + if err != nil { + log.Debugf("RoutingResolver: dht get for name %s failed: %s", name, err) + out <- onceResult{err: err} + close(out) + cancel() + return out + } + + go func() { + defer cancel() + defer close(out) + ctx, span := StartSpan(ctx, "IpnsResolver.ResolveOnceAsync.Worker") + defer span.End() + + for { + select { + case val, ok := <-vals: + if !ok { + return + } + + entry := new(pb.IpnsEntry) + err = proto.Unmarshal(val, entry) + if err != nil { + log.Debugf("RoutingResolver: could not unmarshal value for name %s: %s", name, err) + emitOnceResult(ctx, out, onceResult{err: err}) + return + } + + var p path.Path + // check for old style record: + if valh, err := mh.Cast(entry.GetValue()); err == nil { + // Its an old style multihash record + log.Debugf("encountered CIDv0 ipns entry: %s", valh) + p = path.FromCid(cid.NewCidV0(valh)) + } else { + // Not a multihash, probably a new style record + p, err = path.ParsePath(string(entry.GetValue())) + if err != nil { + emitOnceResult(ctx, out, onceResult{err: err}) + return + } + } + + ttl := DefaultResolverCacheTTL + if entry.Ttl != nil { + ttl = time.Duration(*entry.Ttl) + } + switch eol, err := ipns.GetEOL(entry); err { + case ipns.ErrUnrecognizedValidity: + // No EOL. + case nil: + ttEol := time.Until(eol) + if ttEol < 0 { + // It *was* valid when we first resolved it. + ttl = 0 + } else if ttEol < ttl { + ttl = ttEol + } + default: + log.Errorf("encountered error when parsing EOL: %s", err) + emitOnceResult(ctx, out, onceResult{err: err}) + return + } + + emitOnceResult(ctx, out, onceResult{value: p, ttl: ttl}) + case <-ctx.Done(): + return + } + } + }() + + return out +} diff --git a/namesys/tracing.go b/namesys/tracing.go new file mode 100644 index 0000000000..4ef84294ae --- /dev/null +++ b/namesys/tracing.go @@ -0,0 +1,13 @@ +package namesys + +import ( + "context" + "fmt" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/trace" +) + +func StartSpan(ctx context.Context, name string, opts ...trace.SpanStartOption) (context.Context, trace.Span) { + return otel.Tracer("go-namesys").Start(ctx, fmt.Sprintf("Namesys.%s", name)) +} diff --git a/path/error.go b/path/error.go new file mode 100644 index 0000000000..dafc446b5a --- /dev/null +++ b/path/error.go @@ -0,0 +1,27 @@ +package path + +import ( + "fmt" +) + +type ErrInvalidPath struct { + error error + path string +} + +func (e ErrInvalidPath) Error() string { + return fmt.Sprintf("invalid path %q: %s", e.path, e.error) +} + +func (e ErrInvalidPath) Unwrap() error { + return e.error +} + +func (e ErrInvalidPath) Is(err error) bool { + switch err.(type) { + case ErrInvalidPath: + return true + default: + return false + } +} diff --git a/path/error_test.go b/path/error_test.go new file mode 100644 index 0000000000..07aab64081 --- /dev/null +++ b/path/error_test.go @@ -0,0 +1,16 @@ +package path + +import ( + "errors" + "testing" +) + +func TestErrorIs(t *testing.T) { + if !errors.Is(ErrInvalidPath{path: "foo", error: errors.New("bar")}, ErrInvalidPath{}) { + t.Fatal("error must be error") + } + + if !errors.Is(&ErrInvalidPath{path: "foo", error: errors.New("bar")}, ErrInvalidPath{}) { + t.Fatal("pointer to error must be error") + } +} diff --git a/path/internal/tracing.go b/path/internal/tracing.go new file mode 100644 index 0000000000..f9eda2f92c --- /dev/null +++ b/path/internal/tracing.go @@ -0,0 +1,13 @@ +package internal + +import ( + "context" + "fmt" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/trace" +) + +func StartSpan(ctx context.Context, name string, opts ...trace.SpanStartOption) (context.Context, trace.Span) { + return otel.Tracer("go-path").Start(ctx, fmt.Sprintf("Path.%s", name), opts...) +} diff --git a/path/path.go b/path/path.go new file mode 100644 index 0000000000..6d53ade047 --- /dev/null +++ b/path/path.go @@ -0,0 +1,190 @@ +// Package path contains utilities to work with ipfs paths. +package path + +import ( + "fmt" + "path" + "strings" + + cid "github.com/ipfs/go-cid" +) + +// A Path represents an ipfs content path: +// - /path/to/file +// - /ipfs/ +// - /ipns//path/to/folder +// - etc +type Path string + +// ^^^ +// TODO: debate making this a private struct wrapped in a public interface +// would allow us to control creation, and cache segments. + +// FromString safely converts a string type to a Path type. +func FromString(s string) Path { + return Path(s) +} + +// FromCid safely converts a cid.Cid type to a Path type. +func FromCid(c cid.Cid) Path { + return Path("/ipfs/" + c.String()) +} + +// Segments returns the different elements of a path +// (elements are delimited by a /). +func (p Path) Segments() []string { + cleaned := path.Clean(string(p)) + segments := strings.Split(cleaned, "/") + + // Ignore leading slash + if len(segments[0]) == 0 { + segments = segments[1:] + } + + return segments +} + +// String converts a path to string. +func (p Path) String() string { + return string(p) +} + +// IsJustAKey returns true if the path is of the form or /ipfs/, or +// /ipld/ +func (p Path) IsJustAKey() bool { + parts := p.Segments() + return len(parts) == 2 && (parts[0] == "ipfs" || parts[0] == "ipld") +} + +// PopLastSegment returns a new Path without its final segment, and the final +// segment, separately. If there is no more to pop (the path is just a key), +// the original path is returned. +func (p Path) PopLastSegment() (Path, string, error) { + + if p.IsJustAKey() { + return p, "", nil + } + + segs := p.Segments() + newPath, err := ParsePath("/" + strings.Join(segs[:len(segs)-1], "/")) + if err != nil { + return "", "", err + } + + return newPath, segs[len(segs)-1], nil +} + +// FromSegments returns a path given its different segments. +func FromSegments(prefix string, seg ...string) (Path, error) { + return ParsePath(prefix + strings.Join(seg, "/")) +} + +// ParsePath returns a well-formed ipfs Path. +// The returned path will always be prefixed with /ipfs/ or /ipns/. +// The prefix will be added if not present in the given string. +// This function will return an error when the given string is +// not a valid ipfs path. +func ParsePath(txt string) (Path, error) { + parts := strings.Split(txt, "/") + if len(parts) == 1 { + kp, err := ParseCidToPath(txt) + if err == nil { + return kp, nil + } + } + + // if the path doesnt begin with a '/' + // we expect this to start with a hash, and be an 'ipfs' path + if parts[0] != "" { + if _, err := decodeCid(parts[0]); err != nil { + return "", &ErrInvalidPath{error: err, path: txt} + } + // The case when the path starts with hash without a protocol prefix + return Path("/ipfs/" + txt), nil + } + + if len(parts) < 3 { + return "", &ErrInvalidPath{error: fmt.Errorf("invalid ipfs path"), path: txt} + } + + //TODO: make this smarter + switch parts[1] { + case "ipfs", "ipld": + if parts[2] == "" { + return "", &ErrInvalidPath{error: fmt.Errorf("not enough path components"), path: txt} + } + // Validate Cid. + _, err := decodeCid(parts[2]) + if err != nil { + return "", &ErrInvalidPath{error: fmt.Errorf("invalid CID: %w", err), path: txt} + } + case "ipns": + if parts[2] == "" { + return "", &ErrInvalidPath{error: fmt.Errorf("not enough path components"), path: txt} + } + default: + return "", &ErrInvalidPath{error: fmt.Errorf("unknown namespace %q", parts[1]), path: txt} + } + + return Path(txt), nil +} + +// ParseCidToPath takes a CID in string form and returns a valid ipfs Path. +func ParseCidToPath(txt string) (Path, error) { + if txt == "" { + return "", &ErrInvalidPath{error: fmt.Errorf("empty"), path: txt} + } + + c, err := decodeCid(txt) + if err != nil { + return "", &ErrInvalidPath{error: err, path: txt} + } + + return FromCid(c), nil +} + +// IsValid checks if a path is a valid ipfs Path. +func (p *Path) IsValid() error { + _, err := ParsePath(p.String()) + return err +} + +// Join joins strings slices using / +func Join(pths []string) string { + return strings.Join(pths, "/") +} + +// SplitList splits strings usings / +func SplitList(pth string) []string { + return strings.Split(pth, "/") +} + +// SplitAbsPath clean up and split fpath. It extracts the first component (which +// must be a Multihash) and return it separately. +func SplitAbsPath(fpath Path) (cid.Cid, []string, error) { + parts := fpath.Segments() + if parts[0] == "ipfs" || parts[0] == "ipld" { + parts = parts[1:] + } + + // if nothing, bail. + if len(parts) == 0 { + return cid.Cid{}, nil, &ErrInvalidPath{error: fmt.Errorf("empty"), path: string(fpath)} + } + + c, err := decodeCid(parts[0]) + // first element in the path is a cid + if err != nil { + return cid.Cid{}, nil, &ErrInvalidPath{error: fmt.Errorf("invalid CID: %w", err), path: string(fpath)} + } + + return c, parts[1:], nil +} + +func decodeCid(cstr string) (cid.Cid, error) { + c, err := cid.Decode(cstr) + if err != nil && len(cstr) == 46 && cstr[:2] == "qm" { // https://github.com/ipfs/go-ipfs/issues/7792 + return cid.Cid{}, fmt.Errorf("%v (possible lowercased CIDv0; consider converting to a case-agnostic CIDv1, such as base32)", err) + } + return c, err +} diff --git a/path/path_test.go b/path/path_test.go new file mode 100644 index 0000000000..2b26a56786 --- /dev/null +++ b/path/path_test.go @@ -0,0 +1,128 @@ +package path + +import ( + "strings" + "testing" +) + +func TestPathParsing(t *testing.T) { + cases := map[string]bool{ + "/ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n": true, + "/ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/a": true, + "/ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/a/b/c/d/e/f": true, + "/ipld/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n": true, + "/ipld/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/a": true, + "/ipld/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/a/b/c/d/e/f": true, + "/ipns/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/a/b/c/d/e/f": true, + "/ipns/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n": true, + "QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/a/b/c/d/e/f": true, + "QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n": true, + "/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n": false, + "/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/a": false, + "/ipfs/foo": false, + "/ipfs/": false, + "ipfs/": false, + "ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n": false, + "/ipld/foo": false, + "/ipld/": false, + "ipld/": false, + "ipld/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n": false, + } + + for p, expected := range cases { + _, err := ParsePath(p) + valid := err == nil + if valid != expected { + t.Fatalf("expected %s to have valid == %t", p, expected) + } + } +} + +func TestNoComponents(t *testing.T) { + for _, s := range []string{ + "/ipfs/", + "/ipns/", + "/ipld/", + } { + _, err := ParsePath(s) + if err == nil || !strings.Contains(err.Error(), "not enough path components") || !strings.Contains(err.Error(), s) { + t.Error("wrong error") + } + } +} + +func TestInvalidPaths(t *testing.T) { + for _, s := range []string{ + "/ipfs", + "/testfs", + "/", + } { + _, err := ParsePath(s) + if err == nil || !strings.Contains(err.Error(), "invalid ipfs path") || !strings.Contains(err.Error(), s) { + t.Error("wrong error") + } + } +} + +func TestIsJustAKey(t *testing.T) { + cases := map[string]bool{ + "QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n": true, + "/ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n": true, + "/ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/a": false, + "/ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/a/b": false, + "/ipns/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n": false, + "/ipld/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/a/b": false, + "/ipld/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n": true, + } + + for p, expected := range cases { + path, err := ParsePath(p) + if err != nil { + t.Fatalf("ParsePath failed to parse \"%s\", but should have succeeded", p) + } + result := path.IsJustAKey() + if result != expected { + t.Fatalf("expected IsJustAKey(%s) to return %v, not %v", p, expected, result) + } + } +} + +func TestPopLastSegment(t *testing.T) { + cases := map[string][]string{ + "QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n": {"/ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n", ""}, + "/ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n": {"/ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n", ""}, + "/ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/a": {"/ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n", "a"}, + "/ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/a/b": {"/ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/a", "b"}, + "/ipns/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/x/y/z": {"/ipns/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/x/y", "z"}, + "/ipld/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/x/y/z": {"/ipld/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/x/y", "z"}, + } + + for p, expected := range cases { + path, err := ParsePath(p) + if err != nil { + t.Fatalf("ParsePath failed to parse \"%s\", but should have succeeded", p) + } + head, tail, err := path.PopLastSegment() + if err != nil { + t.Fatalf("PopLastSegment failed, but should have succeeded: %s", err) + } + headStr := head.String() + if headStr != expected[0] { + t.Fatalf("expected head of PopLastSegment(%s) to return %v, not %v", p, expected[0], headStr) + } + if tail != expected[1] { + t.Fatalf("expected tail of PopLastSegment(%s) to return %v, not %v", p, expected[1], tail) + } + } +} + +func TestV0ErrorDueToLowercase(t *testing.T) { + badb58 := "/ipfs/qmbwqxbekc3p8tqskc98xmwnzrzdtrlmimpl8wbutgsmnr" + _, err := ParsePath(badb58) + if err == nil { + t.Fatal("should have failed to decode") + } + if !strings.HasSuffix(err.Error(), "(possible lowercased CIDv0; consider converting to a case-agnostic CIDv1, such as base32)") { + t.Fatal("should have meaningful info about case-insensitive fix") + } +} diff --git a/path/resolver/resolver.go b/path/resolver/resolver.go new file mode 100644 index 0000000000..90445755be --- /dev/null +++ b/path/resolver/resolver.go @@ -0,0 +1,322 @@ +// Package resolver implements utilities for resolving paths within ipfs. +package resolver + +import ( + "context" + "errors" + "fmt" + "time" + + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" + + cid "github.com/ipfs/go-cid" + format "github.com/ipfs/go-ipld-format" + "github.com/ipfs/go-libipfs/fetcher" + fetcherhelpers "github.com/ipfs/go-libipfs/fetcher/helpers" + path "github.com/ipfs/go-libipfs/path" + "github.com/ipfs/go-libipfs/path/internal" + logging "github.com/ipfs/go-log" + "github.com/ipld/go-ipld-prime" + cidlink "github.com/ipld/go-ipld-prime/linking/cid" + basicnode "github.com/ipld/go-ipld-prime/node/basic" + "github.com/ipld/go-ipld-prime/schema" + "github.com/ipld/go-ipld-prime/traversal/selector/builder" +) + +var log = logging.Logger("pathresolv") + +// ErrNoComponents is used when Paths after a protocol +// do not contain at least one component +var ErrNoComponents = errors.New( + "path must contain at least one component") + +// ErrNoLink is returned when a link is not found in a path +type ErrNoLink struct { + Name string + Node cid.Cid +} + +// Error implements the Error interface for ErrNoLink with a useful +// human readable message. +func (e ErrNoLink) Error() string { + return fmt.Sprintf("no link named %q under %s", e.Name, e.Node.String()) +} + +// Resolver provides path resolution to IPFS. +type Resolver interface { + // ResolveToLastNode walks the given path and returns the cid of the + // last block referenced by the path, and the path segments to + // traverse from the final block boundary to the final node within the + // block. + ResolveToLastNode(ctx context.Context, fpath path.Path) (cid.Cid, []string, error) + // ResolvePath fetches the node for given path. It returns the last + // item returned by ResolvePathComponents and the last link traversed + // which can be used to recover the block. + ResolvePath(ctx context.Context, fpath path.Path) (ipld.Node, ipld.Link, error) + // ResolvePathComponents fetches the nodes for each segment of the given path. + // It uses the first path component as a hash (key) of the first node, then + // resolves all other components walking the links via a selector traversal + ResolvePathComponents(ctx context.Context, fpath path.Path) ([]ipld.Node, error) +} + +// basicResolver implements the Resolver interface. +// It references a FetcherFactory, which is uses to resolve nodes. +// TODO: now that this is more modular, try to unify this code with the +// +// the resolvers in namesys. +type basicResolver struct { + FetcherFactory fetcher.Factory +} + +// NewBasicResolver constructs a new basic resolver. +func NewBasicResolver(fetcherFactory fetcher.Factory) Resolver { + return &basicResolver{ + FetcherFactory: fetcherFactory, + } +} + +// ResolveToLastNode walks the given path and returns the cid of the last +// block referenced by the path, and the path segments to traverse from the +// final block boundary to the final node within the block. +func (r *basicResolver) ResolveToLastNode(ctx context.Context, fpath path.Path) (cid.Cid, []string, error) { + ctx, span := internal.StartSpan(ctx, "basicResolver.ResolveToLastNode", trace.WithAttributes(attribute.Stringer("Path", fpath))) + defer span.End() + + c, p, err := path.SplitAbsPath(fpath) + if err != nil { + return cid.Cid{}, nil, err + } + + if len(p) == 0 { + return c, nil, nil + } + + // create a selector to traverse and match all path segments + pathSelector := pathAllSelector(p[:len(p)-1]) + + // create a new cancellable session + ctx, cancel := context.WithTimeout(ctx, time.Minute) + defer cancel() + + // resolve node before last path segment + nodes, lastCid, depth, err := r.resolveNodes(ctx, c, pathSelector) + if err != nil { + return cid.Cid{}, nil, err + } + + if len(nodes) < 1 { + return cid.Cid{}, nil, fmt.Errorf("path %v did not resolve to a node", fpath) + } else if len(nodes) < len(p) { + return cid.Undef, nil, ErrNoLink{Name: p[len(nodes)-1], Node: lastCid} + } + + parent := nodes[len(nodes)-1] + lastSegment := p[len(p)-1] + + // find final path segment within node + nd, err := parent.LookupBySegment(ipld.ParsePathSegment(lastSegment)) + switch err.(type) { + case nil: + case schema.ErrNoSuchField: + return cid.Undef, nil, ErrNoLink{Name: lastSegment, Node: lastCid} + default: + return cid.Cid{}, nil, err + } + + // if last node is not a link, just return it's cid, add path to remainder and return + if nd.Kind() != ipld.Kind_Link { + // return the cid and the remainder of the path + return lastCid, p[len(p)-depth-1:], nil + } + + lnk, err := nd.AsLink() + if err != nil { + return cid.Cid{}, nil, err + } + + clnk, ok := lnk.(cidlink.Link) + if !ok { + return cid.Cid{}, nil, fmt.Errorf("path %v resolves to a link that is not a cid link: %v", fpath, lnk) + } + + return clnk.Cid, []string{}, nil +} + +// ResolvePath fetches the node for given path. It returns the last item +// returned by ResolvePathComponents and the last link traversed which can be used to recover the block. +// +// Note: if/when the context is cancelled or expires then if a multi-block ADL node is returned then it may not be +// possible to load certain values. +func (r *basicResolver) ResolvePath(ctx context.Context, fpath path.Path) (ipld.Node, ipld.Link, error) { + ctx, span := internal.StartSpan(ctx, "basicResolver.ResolvePath", trace.WithAttributes(attribute.Stringer("Path", fpath))) + defer span.End() + + // validate path + if err := fpath.IsValid(); err != nil { + return nil, nil, err + } + + c, p, err := path.SplitAbsPath(fpath) + if err != nil { + return nil, nil, err + } + + // create a selector to traverse all path segments but only match the last + pathSelector := pathLeafSelector(p) + + nodes, c, _, err := r.resolveNodes(ctx, c, pathSelector) + if err != nil { + return nil, nil, err + } + if len(nodes) < 1 { + return nil, nil, fmt.Errorf("path %v did not resolve to a node", fpath) + } + return nodes[len(nodes)-1], cidlink.Link{Cid: c}, nil +} + +// ResolveSingle simply resolves one hop of a path through a graph with no +// extra context (does not opaquely resolve through sharded nodes) +// Deprecated: fetch node as ipld-prime or convert it and then use a selector to traverse through it. +func ResolveSingle(ctx context.Context, ds format.NodeGetter, nd format.Node, names []string) (*format.Link, []string, error) { + _, span := internal.StartSpan(ctx, "ResolveSingle", trace.WithAttributes(attribute.Stringer("CID", nd.Cid()))) + defer span.End() + return nd.ResolveLink(names) +} + +// ResolvePathComponents fetches the nodes for each segment of the given path. +// It uses the first path component as a hash (key) of the first node, then +// resolves all other components walking the links via a selector traversal +// +// Note: if/when the context is cancelled or expires then if a multi-block ADL node is returned then it may not be +// possible to load certain values. +func (r *basicResolver) ResolvePathComponents(ctx context.Context, fpath path.Path) ([]ipld.Node, error) { + ctx, span := internal.StartSpan(ctx, "basicResolver.ResolvePathComponents", trace.WithAttributes(attribute.Stringer("Path", fpath))) + defer span.End() + + //lint:ignore SA1019 TODO: replace EventBegin + evt := log.EventBegin(ctx, "resolvePathComponents", logging.LoggableMap{"fpath": fpath}) + defer evt.Done() + + // validate path + if err := fpath.IsValid(); err != nil { + evt.Append(logging.LoggableMap{"error": err.Error()}) + return nil, err + } + + c, p, err := path.SplitAbsPath(fpath) + if err != nil { + evt.Append(logging.LoggableMap{"error": err.Error()}) + return nil, err + } + + // create a selector to traverse and match all path segments + pathSelector := pathAllSelector(p) + + nodes, _, _, err := r.resolveNodes(ctx, c, pathSelector) + if err != nil { + evt.Append(logging.LoggableMap{"error": err.Error()}) + } + + return nodes, err +} + +// ResolveLinks iteratively resolves names by walking the link hierarchy. +// Every node is fetched from the Fetcher, resolving the next name. +// Returns the list of nodes forming the path, starting with ndd. This list is +// guaranteed never to be empty. +// +// ResolveLinks(nd, []string{"foo", "bar", "baz"}) +// would retrieve "baz" in ("bar" in ("foo" in nd.Links).Links).Links +// +// Note: if/when the context is cancelled or expires then if a multi-block ADL node is returned then it may not be +// possible to load certain values. +func (r *basicResolver) ResolveLinks(ctx context.Context, ndd ipld.Node, names []string) ([]ipld.Node, error) { + ctx, span := internal.StartSpan(ctx, "basicResolver.ResolveLinks") + defer span.End() + + //lint:ignore SA1019 TODO: replace EventBegin + evt := log.EventBegin(ctx, "resolveLinks", logging.LoggableMap{"names": names}) + defer evt.Done() + + // create a selector to traverse and match all path segments + pathSelector := pathAllSelector(names) + + session := r.FetcherFactory.NewSession(ctx) + + // traverse selector + nodes := []ipld.Node{ndd} + err := session.NodeMatching(ctx, ndd, pathSelector, func(res fetcher.FetchResult) error { + nodes = append(nodes, res.Node) + return nil + }) + if err != nil { + evt.Append(logging.LoggableMap{"error": err.Error()}) + return nil, err + } + + return nodes, err +} + +// Finds nodes matching the selector starting with a cid. Returns the matched nodes, the cid of the block containing +// the last node, and the depth of the last node within its block (root is depth 0). +func (r *basicResolver) resolveNodes(ctx context.Context, c cid.Cid, sel ipld.Node) ([]ipld.Node, cid.Cid, int, error) { + ctx, span := internal.StartSpan(ctx, "basicResolver.resolveNodes", trace.WithAttributes(attribute.Stringer("CID", c))) + defer span.End() + session := r.FetcherFactory.NewSession(ctx) + + // traverse selector + lastLink := cid.Undef + depth := 0 + nodes := []ipld.Node{} + err := fetcherhelpers.BlockMatching(ctx, session, cidlink.Link{Cid: c}, sel, func(res fetcher.FetchResult) error { + if res.LastBlockLink == nil { + res.LastBlockLink = cidlink.Link{Cid: c} + } + cidLnk, ok := res.LastBlockLink.(cidlink.Link) + if !ok { + return fmt.Errorf("link is not a cidlink: %v", cidLnk) + } + + // if we hit a block boundary + if !lastLink.Equals(cidLnk.Cid) { + depth = 0 + lastLink = cidLnk.Cid + } else { + depth++ + } + + nodes = append(nodes, res.Node) + return nil + }) + if err != nil { + return nil, cid.Undef, 0, err + } + + return nodes, lastLink, depth, nil +} + +func pathLeafSelector(path []string) ipld.Node { + ssb := builder.NewSelectorSpecBuilder(basicnode.Prototype.Any) + return pathSelector(path, ssb, func(p string, s builder.SelectorSpec) builder.SelectorSpec { + return ssb.ExploreFields(func(efsb builder.ExploreFieldsSpecBuilder) { efsb.Insert(p, s) }) + }) +} + +func pathAllSelector(path []string) ipld.Node { + ssb := builder.NewSelectorSpecBuilder(basicnode.Prototype.Any) + return pathSelector(path, ssb, func(p string, s builder.SelectorSpec) builder.SelectorSpec { + return ssb.ExploreUnion( + ssb.Matcher(), + ssb.ExploreFields(func(efsb builder.ExploreFieldsSpecBuilder) { efsb.Insert(p, s) }), + ) + }) +} + +func pathSelector(path []string, ssb builder.SelectorSpecBuilder, reduce func(string, builder.SelectorSpec) builder.SelectorSpec) ipld.Node { + spec := ssb.Matcher() + for i := len(path) - 1; i >= 0; i-- { + spec = reduce(path[i], spec) + } + return spec.Node() +} diff --git a/path/resolver/resolver_test.go b/path/resolver/resolver_test.go new file mode 100644 index 0000000000..601e32f2c6 --- /dev/null +++ b/path/resolver/resolver_test.go @@ -0,0 +1,293 @@ +package resolver_test + +import ( + "bytes" + "context" + "fmt" + "math/rand" + "strings" + "testing" + "time" + + "github.com/ipfs/go-cid" + blocks "github.com/ipfs/go-libipfs/blocks" + bsfetcher "github.com/ipfs/go-libipfs/fetcher/impl/blockservice" + dagpb "github.com/ipld/go-codec-dagpb" + "github.com/ipld/go-ipld-prime" + cidlink "github.com/ipld/go-ipld-prime/linking/cid" + basicnode "github.com/ipld/go-ipld-prime/node/basic" + "github.com/ipld/go-ipld-prime/schema" + "github.com/multiformats/go-multihash" + + path "github.com/ipfs/go-libipfs/path" + "github.com/ipfs/go-libipfs/path/resolver" + merkledag "github.com/ipfs/go-merkledag" + dagmock "github.com/ipfs/go-merkledag/test" + "github.com/ipfs/go-unixfsnode" + dagcbor "github.com/ipld/go-ipld-prime/codec/dagcbor" + dagjson "github.com/ipld/go-ipld-prime/codec/dagjson" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func randNode() *merkledag.ProtoNode { + node := new(merkledag.ProtoNode) + node.SetData(make([]byte, 32)) + r := rand.New(rand.NewSource(time.Now().UnixNano())) + r.Read(node.Data()) + return node +} + +func TestRecurivePathResolution(t *testing.T) { + ctx := context.Background() + bsrv := dagmock.Bserv() + + a := randNode() + b := randNode() + c := randNode() + + err := b.AddNodeLink("grandchild", c) + if err != nil { + t.Fatal(err) + } + + err = a.AddNodeLink("child", b) + if err != nil { + t.Fatal(err) + } + + for _, n := range []*merkledag.ProtoNode{a, b, c} { + err = bsrv.AddBlock(ctx, n) + if err != nil { + t.Fatal(err) + } + } + + aKey := a.Cid() + + segments := []string{aKey.String(), "child", "grandchild"} + p, err := path.FromSegments("/ipfs/", segments...) + if err != nil { + t.Fatal(err) + } + + fetcherFactory := bsfetcher.NewFetcherConfig(bsrv) + fetcherFactory.NodeReifier = unixfsnode.Reify + fetcherFactory.PrototypeChooser = dagpb.AddSupportToChooser(func(lnk ipld.Link, lnkCtx ipld.LinkContext) (ipld.NodePrototype, error) { + if tlnkNd, ok := lnkCtx.LinkNode.(schema.TypedLinkNode); ok { + return tlnkNd.LinkTargetNodePrototype(), nil + } + return basicnode.Prototype.Any, nil + }) + resolver := resolver.NewBasicResolver(fetcherFactory) + + node, lnk, err := resolver.ResolvePath(ctx, p) + if err != nil { + t.Fatal(err) + } + + uNode, ok := node.(unixfsnode.PathedPBNode) + require.True(t, ok) + fd := uNode.FieldData() + byts, err := fd.Must().AsBytes() + require.NoError(t, err) + + assert.Equal(t, cidlink.Link{Cid: c.Cid()}, lnk) + + assert.Equal(t, c.Data(), byts) + cKey := c.Cid() + + rCid, rest, err := resolver.ResolveToLastNode(ctx, p) + if err != nil { + t.Fatal(err) + } + + if len(rest) != 0 { + t.Error("expected rest to be empty") + } + + if rCid.String() != cKey.String() { + t.Fatal(fmt.Errorf( + "ResolveToLastNode failed for %s: %s != %s", + p.String(), rCid.String(), cKey.String())) + } + + p2, err := path.FromSegments("/ipfs/", aKey.String()) + if err != nil { + t.Fatal(err) + } + + rCid, rest, err = resolver.ResolveToLastNode(ctx, p2) + if err != nil { + t.Fatal(err) + } + + if len(rest) != 0 { + t.Error("expected rest to be empty") + } + + if rCid.String() != aKey.String() { + t.Fatal(fmt.Errorf( + "ResolveToLastNode failed for %s: %s != %s", + p.String(), rCid.String(), cKey.String())) + } +} +func TestResolveToLastNode_ErrNoLink(t *testing.T) { + ctx := context.Background() + bsrv := dagmock.Bserv() + + a := randNode() + b := randNode() + c := randNode() + + err := b.AddNodeLink("grandchild", c) + if err != nil { + t.Fatal(err) + } + + err = a.AddNodeLink("child", b) + if err != nil { + t.Fatal(err) + } + + for _, n := range []*merkledag.ProtoNode{a, b, c} { + err = bsrv.AddBlock(ctx, n) + if err != nil { + t.Fatal(err) + } + } + + aKey := a.Cid() + + fetcherFactory := bsfetcher.NewFetcherConfig(bsrv) + fetcherFactory.PrototypeChooser = dagpb.AddSupportToChooser(func(lnk ipld.Link, lnkCtx ipld.LinkContext) (ipld.NodePrototype, error) { + if tlnkNd, ok := lnkCtx.LinkNode.(schema.TypedLinkNode); ok { + return tlnkNd.LinkTargetNodePrototype(), nil + } + return basicnode.Prototype.Any, nil + }) + fetcherFactory.NodeReifier = unixfsnode.Reify + r := resolver.NewBasicResolver(fetcherFactory) + + // test missing link intermediate segment + segments := []string{aKey.String(), "cheese", "time"} + p, err := path.FromSegments("/ipfs/", segments...) + require.NoError(t, err) + + _, _, err = r.ResolveToLastNode(ctx, p) + require.EqualError(t, err, resolver.ErrNoLink{Name: "cheese", Node: aKey}.Error()) + + // test missing link at end + bKey := b.Cid() + segments = []string{aKey.String(), "child", "apples"} + p, err = path.FromSegments("/ipfs/", segments...) + require.NoError(t, err) + + _, _, err = r.ResolveToLastNode(ctx, p) + require.EqualError(t, err, resolver.ErrNoLink{Name: "apples", Node: bKey}.Error()) +} + +func TestResolveToLastNode_NoUnnecessaryFetching(t *testing.T) { + ctx := context.Background() + bsrv := dagmock.Bserv() + + a := randNode() + b := randNode() + + err := a.AddNodeLink("child", b) + require.NoError(t, err) + + err = bsrv.AddBlock(ctx, a) + require.NoError(t, err) + + aKey := a.Cid() + + segments := []string{aKey.String(), "child"} + p, err := path.FromSegments("/ipfs/", segments...) + require.NoError(t, err) + + fetcherFactory := bsfetcher.NewFetcherConfig(bsrv) + fetcherFactory.PrototypeChooser = dagpb.AddSupportToChooser(func(lnk ipld.Link, lnkCtx ipld.LinkContext) (ipld.NodePrototype, error) { + if tlnkNd, ok := lnkCtx.LinkNode.(schema.TypedLinkNode); ok { + return tlnkNd.LinkTargetNodePrototype(), nil + } + return basicnode.Prototype.Any, nil + }) + fetcherFactory.NodeReifier = unixfsnode.Reify + resolver := resolver.NewBasicResolver(fetcherFactory) + + resolvedCID, remainingPath, err := resolver.ResolveToLastNode(ctx, p) + require.NoError(t, err) + + require.Equal(t, len(remainingPath), 0, "cannot have remaining path") + require.Equal(t, b.Cid(), resolvedCID) +} + +func TestPathRemainder(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + bsrv := dagmock.Bserv() + + nb := basicnode.Prototype.Any.NewBuilder() + err := dagjson.Decode(nb, strings.NewReader(`{"foo": {"bar": "baz"}}`)) + require.NoError(t, err) + out := new(bytes.Buffer) + err = dagcbor.Encode(nb.Build(), out) + require.NoError(t, err) + lnk, err := cid.Prefix{ + Version: 1, + Codec: cid.DagCBOR, + MhType: multihash.SHA2_256, + MhLength: 32, + }.Sum(out.Bytes()) + require.NoError(t, err) + blk, err := blocks.NewBlockWithCid(out.Bytes(), lnk) + require.NoError(t, err) + bsrv.AddBlock(ctx, blk) + fetcherFactory := bsfetcher.NewFetcherConfig(bsrv) + resolver := resolver.NewBasicResolver(fetcherFactory) + + rp1, remainder, err := resolver.ResolveToLastNode(ctx, path.FromString(lnk.String()+"/foo/bar")) + require.NoError(t, err) + + assert.Equal(t, lnk, rp1) + require.Equal(t, "foo/bar", path.Join(remainder)) +} + +func TestResolveToLastNode_MixedSegmentTypes(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + bsrv := dagmock.Bserv() + a := randNode() + err := bsrv.AddBlock(ctx, a) + if err != nil { + t.Fatal(err) + } + + nb := basicnode.Prototype.Any.NewBuilder() + json := `{"foo":{"bar":[0,{"boom":["baz",1,2,{"/":"CID"},"blop"]}]}}` + json = strings.ReplaceAll(json, "CID", a.Cid().String()) + err = dagjson.Decode(nb, strings.NewReader(json)) + require.NoError(t, err) + out := new(bytes.Buffer) + err = dagcbor.Encode(nb.Build(), out) + require.NoError(t, err) + lnk, err := cid.Prefix{ + Version: 1, + Codec: cid.DagCBOR, + MhType: multihash.SHA2_256, + MhLength: 32, + }.Sum(out.Bytes()) + require.NoError(t, err) + blk, err := blocks.NewBlockWithCid(out.Bytes(), lnk) + require.NoError(t, err) + bsrv.AddBlock(ctx, blk) + fetcherFactory := bsfetcher.NewFetcherConfig(bsrv) + resolver := resolver.NewBasicResolver(fetcherFactory) + + cid, remainder, err := resolver.ResolveToLastNode(ctx, path.FromString(lnk.String()+"/foo/bar/1/boom/3")) + require.NoError(t, err) + + assert.Equal(t, 0, len(remainder)) + assert.True(t, cid.Equals(a.Cid())) +} diff --git a/pinning/pinner/.gitignore b/pinning/pinner/.gitignore new file mode 100644 index 0000000000..3c342889d2 --- /dev/null +++ b/pinning/pinner/.gitignore @@ -0,0 +1,8 @@ +*~ +*.log + +# Test binary, build with `go test -c` +*.test + +# Output of the go coverage tool +*.out diff --git a/pinning/pinner/dsindex/error.go b/pinning/pinner/dsindex/error.go new file mode 100644 index 0000000000..f3b685bb95 --- /dev/null +++ b/pinning/pinner/dsindex/error.go @@ -0,0 +1,8 @@ +package dsindex + +import "errors" + +var ( + ErrEmptyKey = errors.New("key is empty") + ErrEmptyValue = errors.New("value is empty") +) diff --git a/pinning/pinner/dsindex/indexer.go b/pinning/pinner/dsindex/indexer.go new file mode 100644 index 0000000000..8384ad5d5a --- /dev/null +++ b/pinning/pinner/dsindex/indexer.go @@ -0,0 +1,277 @@ +// Package dsindex provides secondary indexing functionality for a datastore. +package dsindex + +import ( + "context" + "fmt" + "path" + + ds "github.com/ipfs/go-datastore" + "github.com/ipfs/go-datastore/namespace" + "github.com/ipfs/go-datastore/query" + "github.com/multiformats/go-multibase" +) + +// Indexer maintains a secondary index. An index is a collection of key-value +// mappings where the key is the secondary index that maps to one or more +// values, where each value is a unique key being indexed. +type Indexer interface { + // Add adds the specified value to the key + Add(ctx context.Context, key, value string) error + + // Delete deletes the specified value from the key. If the value is not in + // the datastore, this method returns no error. + Delete(ctx context.Context, key, value string) error + + // DeleteKey deletes all values in the given key. If a key is not in the + // datastore, this method returns no error. Returns a count of values that + // were deleted. + DeleteKey(ctx context.Context, key string) (count int, err error) + + // DeleteAll deletes all keys managed by this Indexer. Returns a count of + // the values that were deleted. + DeleteAll(ctx context.Context) (count int, err error) + + // ForEach calls the function for each value in the specified key, until + // there are no more values, or until the function returns false. If key + // is empty string, then all keys are iterated. + ForEach(ctx context.Context, key string, fn func(key, value string) bool) error + + // HasValue determines if the key contains the specified value + HasValue(ctx context.Context, key, value string) (bool, error) + + // HasAny determines if any value is in the specified key. If key is + // empty string, then all values are searched. + HasAny(ctx context.Context, key string) (bool, error) + + // Search returns all values for the given key + Search(ctx context.Context, key string) (values []string, err error) +} + +// indexer is a simple implementation of Indexer. This implementation relies +// on the underlying data store to support efficient querying by prefix. +// +// TODO: Consider adding caching +type indexer struct { + dstore ds.Datastore +} + +// New creates a new datastore index. All indexes are stored under the +// specified index name. +// +// To persist the actions of calling Indexer functions, it is necessary to call +// dstore.Sync. +func New(dstore ds.Datastore, name ds.Key) Indexer { + return &indexer{ + dstore: namespace.Wrap(dstore, name), + } +} + +func (x *indexer) Add(ctx context.Context, key, value string) error { + if key == "" { + return ErrEmptyKey + } + if value == "" { + return ErrEmptyValue + } + dsKey := ds.NewKey(encode(key)).ChildString(encode(value)) + return x.dstore.Put(ctx, dsKey, []byte{}) +} + +func (x *indexer) Delete(ctx context.Context, key, value string) error { + if key == "" { + return ErrEmptyKey + } + if value == "" { + return ErrEmptyValue + } + return x.dstore.Delete(ctx, ds.NewKey(encode(key)).ChildString(encode(value))) +} + +func (x *indexer) DeleteKey(ctx context.Context, key string) (int, error) { + if key == "" { + return 0, ErrEmptyKey + } + return x.deletePrefix(ctx, encode(key)) +} + +func (x *indexer) DeleteAll(ctx context.Context) (int, error) { + return x.deletePrefix(ctx, "") +} + +func (x *indexer) ForEach(ctx context.Context, key string, fn func(key, value string) bool) error { + if key != "" { + key = encode(key) + } + + q := query.Query{ + Prefix: key, + KeysOnly: true, + } + results, err := x.dstore.Query(ctx, q) + if err != nil { + return err + } + defer results.Close() + + for r := range results.Next() { + if ctx.Err() != nil { + return ctx.Err() + } + if r.Error != nil { + return fmt.Errorf("cannot read index: %v", r.Error) + } + ent := r.Entry + decIdx, err := decode(path.Base(path.Dir(ent.Key))) + if err != nil { + return fmt.Errorf("cannot decode index: %v", err) + } + decKey, err := decode(path.Base(ent.Key)) + if err != nil { + return fmt.Errorf("cannot decode key: %v", err) + } + if !fn(decIdx, decKey) { + return nil + } + } + + return nil +} + +func (x *indexer) HasValue(ctx context.Context, key, value string) (bool, error) { + if key == "" { + return false, ErrEmptyKey + } + if value == "" { + return false, ErrEmptyValue + } + return x.dstore.Has(ctx, ds.NewKey(encode(key)).ChildString(encode(value))) +} + +func (x *indexer) HasAny(ctx context.Context, key string) (bool, error) { + var any bool + err := x.ForEach(ctx, key, func(key, value string) bool { + any = true + return false + }) + return any, err +} + +func (x *indexer) Search(ctx context.Context, key string) ([]string, error) { + if key == "" { + return nil, ErrEmptyKey + } + ents, err := x.queryPrefix(ctx, encode(key)) + if err != nil { + return nil, err + } + if len(ents) == 0 { + return nil, nil + } + + values := make([]string, len(ents)) + for i := range ents { + values[i], err = decode(path.Base(ents[i].Key)) + if err != nil { + return nil, fmt.Errorf("cannot decode value: %v", err) + } + } + return values, nil +} + +// SyncIndex synchronizes the keys in the target Indexer to match those of the +// ref Indexer. This function does not change this indexer's key root (name +// passed into New). +func SyncIndex(ctx context.Context, ref, target Indexer) (bool, error) { + // Build reference index map + refs := map[string]string{} + err := ref.ForEach(ctx, "", func(key, value string) bool { + refs[value] = key + return true + }) + if err != nil { + return false, err + } + if len(refs) == 0 { + return false, nil + } + + // Compare current indexes + dels := map[string]string{} + err = target.ForEach(ctx, "", func(key, value string) bool { + refKey, ok := refs[value] + if ok && refKey == key { + // same in both; delete from refs, do not add to dels + delete(refs, value) + } else { + dels[value] = key + } + return true + }) + if err != nil { + return false, err + } + + // Items in dels are keys that no longer exist + for value, key := range dels { + err = target.Delete(ctx, key, value) + if err != nil { + return false, err + } + } + + // What remains in refs are keys that need to be added + for value, key := range refs { + err = target.Add(ctx, key, value) + if err != nil { + return false, err + } + } + + return len(refs) != 0 || len(dels) != 0, nil +} + +func (x *indexer) deletePrefix(ctx context.Context, prefix string) (int, error) { + ents, err := x.queryPrefix(ctx, prefix) + if err != nil { + return 0, err + } + + for i := range ents { + err = x.dstore.Delete(ctx, ds.NewKey(ents[i].Key)) + if err != nil { + return 0, err + } + } + + return len(ents), nil +} + +func (x *indexer) queryPrefix(ctx context.Context, prefix string) ([]query.Entry, error) { + q := query.Query{ + Prefix: prefix, + KeysOnly: true, + } + results, err := x.dstore.Query(ctx, q) + if err != nil { + return nil, err + } + return results.Rest() +} + +func encode(data string) string { + encData, err := multibase.Encode(multibase.Base64url, []byte(data)) + if err != nil { + // programming error; using unsupported encoding + panic(err.Error()) + } + return encData +} + +func decode(data string) (string, error) { + _, b, err := multibase.Decode(data) + if err != nil { + return "", err + } + return string(b), nil +} diff --git a/pinning/pinner/dsindex/indexer_test.go b/pinning/pinner/dsindex/indexer_test.go new file mode 100644 index 0000000000..45372c6053 --- /dev/null +++ b/pinning/pinner/dsindex/indexer_test.go @@ -0,0 +1,286 @@ +package dsindex + +import ( + "context" + "testing" + + ds "github.com/ipfs/go-datastore" +) + +func createIndexer() Indexer { + dstore := ds.NewMapDatastore() + nameIndex := New(dstore, ds.NewKey("/data/nameindex")) + + ctx := context.Background() + nameIndex.Add(ctx, "alice", "a1") + nameIndex.Add(ctx, "bob", "b1") + nameIndex.Add(ctx, "bob", "b2") + nameIndex.Add(ctx, "cathy", "c1") + + return nameIndex +} + +func TestAdd(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + nameIndex := createIndexer() + err := nameIndex.Add(ctx, "someone", "s1") + if err != nil { + t.Fatal(err) + } + err = nameIndex.Add(ctx, "someone", "s1") + if err != nil { + t.Fatal(err) + } + + err = nameIndex.Add(ctx, "", "noindex") + if err != ErrEmptyKey { + t.Fatal("unexpected error:", err) + } + + err = nameIndex.Add(ctx, "nokey", "") + if err != ErrEmptyValue { + t.Fatal("unexpected error:", err) + } +} + +func TestHasValue(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + nameIndex := createIndexer() + + ok, err := nameIndex.HasValue(ctx, "bob", "b1") + if err != nil { + t.Fatal(err) + } + if !ok { + t.Fatal("missing index") + } + + ok, err = nameIndex.HasValue(ctx, "bob", "b3") + if err != nil { + t.Fatal(err) + } + if ok { + t.Fatal("should not have index") + } + + _, err = nameIndex.HasValue(ctx, "", "b1") + if err != ErrEmptyKey { + t.Fatal("unexpected error:", err) + } + + _, err = nameIndex.HasValue(ctx, "bob", "") + if err != ErrEmptyValue { + t.Fatal("unexpected error:", err) + } +} + +func TestHasAny(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + nameIndex := createIndexer() + + ok, err := nameIndex.HasAny(ctx, "nothere") + if err != nil { + t.Fatal(err) + } + if ok { + t.Fatal("should return false") + } + + for _, idx := range []string{"alice", "bob", ""} { + ok, err = nameIndex.HasAny(ctx, idx) + if err != nil { + t.Fatal(err) + } + if !ok { + t.Fatal("missing index", idx) + } + } + + count, err := nameIndex.DeleteAll(ctx) + if err != nil { + t.Fatal(err) + } + if count != 4 { + t.Fatal("expected 4 deletions") + } + + ok, err = nameIndex.HasAny(ctx, "") + if err != nil { + t.Fatal(err) + } + if ok { + t.Fatal("should return false") + } +} + +func TestForEach(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + nameIndex := createIndexer() + + found := make(map[string]struct{}) + err := nameIndex.ForEach(ctx, "bob", func(key, value string) bool { + found[value] = struct{}{} + return true + }) + if err != nil { + t.Fatal(err) + } + + for _, value := range []string{"b1", "b2"} { + _, ok := found[value] + if !ok { + t.Fatal("missing key for value", value) + } + } + + values := map[string]string{} + err = nameIndex.ForEach(ctx, "", func(key, value string) bool { + values[value] = key + return true + }) + if err != nil { + t.Fatal(err) + } + if len(values) != 4 { + t.Fatal("expected 4 keys") + } + + if values["a1"] != "alice" { + t.Error("expected a1: alice") + } + if values["b1"] != "bob" { + t.Error("expected b1: bob") + } + if values["b2"] != "bob" { + t.Error("expected b2: bob") + } + if values["c1"] != "cathy" { + t.Error("expected c1: cathy") + } +} + +func TestSearch(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + nameIndex := createIndexer() + + ids, err := nameIndex.Search(ctx, "bob") + if err != nil { + t.Fatal(err) + } + if len(ids) != 2 { + t.Fatal("wrong number of ids - expected 2 got", ids) + } + for _, id := range ids { + if id != "b1" && id != "b2" { + t.Fatal("wrong value in id set") + } + } + if ids[0] == ids[1] { + t.Fatal("duplicate id") + } + + ids, err = nameIndex.Search(ctx, "cathy") + if err != nil { + t.Fatal(err) + } + if len(ids) != 1 || ids[0] != "c1" { + t.Fatal("wrong ids") + } + + ids, err = nameIndex.Search(ctx, "amit") + if err != nil { + t.Fatal(err) + } + if len(ids) != 0 { + t.Fatal("unexpected ids returned") + } +} + +func TestDelete(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + nameIndex := createIndexer() + + err := nameIndex.Delete(ctx, "bob", "b3") + if err != nil { + t.Fatal(err) + } + + err = nameIndex.Delete(ctx, "alice", "a1") + if err != nil { + t.Fatal(err) + } + + ok, err := nameIndex.HasValue(ctx, "alice", "a1") + if err != nil { + t.Fatal(err) + } + if ok { + t.Fatal("index key should have been deleted") + } + + count, err := nameIndex.DeleteKey(ctx, "bob") + if err != nil { + t.Fatal(err) + } + if count != 2 { + t.Fatal("wrong deleted count") + } + ok, _ = nameIndex.HasValue(ctx, "bob", "b1") + if ok { + t.Fatal("index not deleted") + } +} + +func TestSyncIndex(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + nameIndex := createIndexer() + + dstore := ds.NewMapDatastore() + refIndex := New(dstore, ds.NewKey("/ref")) + refIndex.Add(ctx, "alice", "a1") + refIndex.Add(ctx, "cathy", "zz") + refIndex.Add(ctx, "dennis", "d1") + + changed, err := SyncIndex(ctx, refIndex, nameIndex) + if err != nil { + t.Fatal(err) + } + if !changed { + t.Error("change was not indicated") + } + + // Create map of id->index in sync target + syncs := map[string]string{} + err = nameIndex.ForEach(ctx, "", func(key, value string) bool { + syncs[value] = key + return true + }) + if err != nil { + t.Fatal(err) + } + + // Iterate items in sync source and make sure they appear in target + var itemCount int + err = refIndex.ForEach(ctx, "", func(key, value string) bool { + itemCount++ + syncKey, ok := syncs[value] + if !ok || key != syncKey { + t.Fatal("key", key, "-->", value, "was not synced") + } + return true + }) + if err != nil { + t.Fatal(err) + } + + if itemCount != len(syncs) { + t.Fatal("different number of items in sync source and target") + } +} diff --git a/pinning/pinner/dspinner/pin.go b/pinning/pinner/dspinner/pin.go new file mode 100644 index 0000000000..6aa567a606 --- /dev/null +++ b/pinning/pinner/dspinner/pin.go @@ -0,0 +1,1031 @@ +// Package dspinner implements structures and methods to keep track of +// which objects a user wants to keep stored locally. This implementation +// stores pin data in a datastore. +package dspinner + +import ( + "context" + "errors" + "fmt" + "path" + "sync" + + "github.com/ipfs/go-cid" + ds "github.com/ipfs/go-datastore" + "github.com/ipfs/go-datastore/query" + ipld "github.com/ipfs/go-ipld-format" + logging "github.com/ipfs/go-log" + "github.com/ipfs/go-merkledag" + "github.com/ipfs/go-merkledag/dagutils" + "github.com/polydawn/refmt/cbor" + "github.com/polydawn/refmt/obj/atlas" + + ipfspinner "github.com/ipfs/go-libipfs/pinning/pinner" + "github.com/ipfs/go-libipfs/pinning/pinner/dsindex" +) + +const ( + basePath = "/pins" + pinKeyPath = "/pins/pin" + indexKeyPath = "/pins/index" + dirtyKeyPath = "/pins/state/dirty" +) + +var ( + log logging.StandardLogger = logging.Logger("pin") + + linkDirect, linkRecursive string + + pinCidDIndexPath string + pinCidRIndexPath string + pinNameIndexPath string + + dirtyKey = ds.NewKey(dirtyKeyPath) + + pinAtl atlas.Atlas +) + +func init() { + directStr, ok := ipfspinner.ModeToString(ipfspinner.Direct) + if !ok { + panic("could not find Direct pin enum") + } + linkDirect = directStr + + recursiveStr, ok := ipfspinner.ModeToString(ipfspinner.Recursive) + if !ok { + panic("could not find Recursive pin enum") + } + linkRecursive = recursiveStr + + pinCidRIndexPath = path.Join(indexKeyPath, "cidRindex") + pinCidDIndexPath = path.Join(indexKeyPath, "cidDindex") + pinNameIndexPath = path.Join(indexKeyPath, "nameIndex") + + pinAtl = atlas.MustBuild( + atlas.BuildEntry(pin{}).StructMap(). + AddField("Cid", atlas.StructMapEntry{SerialName: "cid"}). + AddField("Metadata", atlas.StructMapEntry{SerialName: "metadata", OmitEmpty: true}). + AddField("Mode", atlas.StructMapEntry{SerialName: "mode"}). + AddField("Name", atlas.StructMapEntry{SerialName: "name", OmitEmpty: true}). + Complete(), + atlas.BuildEntry(cid.Cid{}).Transform(). + TransformMarshal(atlas.MakeMarshalTransformFunc(func(live cid.Cid) ([]byte, error) { return live.MarshalBinary() })). + TransformUnmarshal(atlas.MakeUnmarshalTransformFunc(func(serializable []byte) (cid.Cid, error) { + c := cid.Cid{} + err := c.UnmarshalBinary(serializable) + if err != nil { + return cid.Cid{}, err + } + return c, nil + })).Complete(), + ) + pinAtl = pinAtl.WithMapMorphism(atlas.MapMorphism{KeySortMode: atlas.KeySortMode_Strings}) +} + +// pinner implements the Pinner interface +type pinner struct { + autoSync bool + lock sync.RWMutex + + dserv ipld.DAGService + dstore ds.Datastore + + cidDIndex dsindex.Indexer + cidRIndex dsindex.Indexer + nameIndex dsindex.Indexer + + clean int64 + dirty int64 +} + +var _ ipfspinner.Pinner = (*pinner)(nil) + +type pin struct { + Id string + Cid cid.Cid + Metadata map[string]interface{} + Mode ipfspinner.Mode + Name string +} + +func (p *pin) dsKey() ds.Key { + return ds.NewKey(path.Join(pinKeyPath, p.Id)) +} + +func newPin(c cid.Cid, mode ipfspinner.Mode, name string) *pin { + return &pin{ + Id: path.Base(ds.RandomKey().String()), + Cid: c, + Name: name, + Mode: mode, + } +} + +type syncDAGService interface { + ipld.DAGService + Sync() error +} + +// New creates a new pinner and loads its keysets from the given datastore. If +// there is no data present in the datastore, then an empty pinner is returned. +// +// By default, changes are automatically flushed to the datastore. This can be +// disabled by calling SetAutosync(false), which will require that Flush be +// called explicitly. +func New(ctx context.Context, dstore ds.Datastore, dserv ipld.DAGService) (*pinner, error) { + p := &pinner{ + autoSync: true, + cidDIndex: dsindex.New(dstore, ds.NewKey(pinCidDIndexPath)), + cidRIndex: dsindex.New(dstore, ds.NewKey(pinCidRIndexPath)), + nameIndex: dsindex.New(dstore, ds.NewKey(pinNameIndexPath)), + dserv: dserv, + dstore: dstore, + } + + data, err := dstore.Get(ctx, dirtyKey) + if err != nil { + if err == ds.ErrNotFound { + return p, nil + } + return nil, fmt.Errorf("cannot load dirty flag: %v", err) + } + if data[0] == 1 { + p.dirty = 1 + + err = p.rebuildIndexes(ctx) + if err != nil { + return nil, fmt.Errorf("cannot rebuild indexes: %v", err) + } + } + + return p, nil +} + +// SetAutosync allows auto-syncing to be enabled or disabled during runtime. +// This may be used to turn off autosync before doing many repeated pinning +// operations, and then turn it on after. Returns the previous value. +func (p *pinner) SetAutosync(auto bool) bool { + p.lock.Lock() + defer p.lock.Unlock() + + p.autoSync, auto = auto, p.autoSync + return auto +} + +// Pin the given node, optionally recursive +func (p *pinner) Pin(ctx context.Context, node ipld.Node, recurse bool) error { + err := p.dserv.Add(ctx, node) + if err != nil { + return err + } + + if recurse { + return p.doPinRecursive(ctx, node.Cid(), true) + } else { + return p.doPinDirect(ctx, node.Cid()) + } +} + +func (p *pinner) doPinRecursive(ctx context.Context, c cid.Cid, fetch bool) error { + cidKey := c.KeyString() + + p.lock.Lock() + defer p.lock.Unlock() + + found, err := p.cidRIndex.HasAny(ctx, cidKey) + if err != nil { + return err + } + if found { + return nil + } + + dirtyBefore := p.dirty + + if fetch { + // temporary unlock to fetch the entire graph + p.lock.Unlock() + // Fetch graph starting at node identified by cid + err = merkledag.FetchGraph(ctx, c, p.dserv) + p.lock.Lock() + if err != nil { + return err + } + } + + // If autosyncing, sync dag service before making any change to pins + err = p.flushDagService(ctx, false) + if err != nil { + return err + } + + // Only look again if something has changed. + if p.dirty != dirtyBefore { + found, err = p.cidRIndex.HasAny(ctx, cidKey) + if err != nil { + return err + } + if found { + return nil + } + } + + // TODO: remove this to support multiple pins per CID + found, err = p.cidDIndex.HasAny(ctx, cidKey) + if err != nil { + return err + } + if found { + _, err = p.removePinsForCid(ctx, c, ipfspinner.Direct) + if err != nil { + return err + } + } + + _, err = p.addPin(ctx, c, ipfspinner.Recursive, "") + if err != nil { + return err + } + return p.flushPins(ctx, false) +} + +func (p *pinner) doPinDirect(ctx context.Context, c cid.Cid) error { + cidKey := c.KeyString() + + p.lock.Lock() + defer p.lock.Unlock() + + found, err := p.cidRIndex.HasAny(ctx, cidKey) + if err != nil { + return err + } + if found { + return fmt.Errorf("%s already pinned recursively", c.String()) + } + + _, err = p.addPin(ctx, c, ipfspinner.Direct, "") + if err != nil { + return err + } + + return p.flushPins(ctx, false) +} + +func (p *pinner) addPin(ctx context.Context, c cid.Cid, mode ipfspinner.Mode, name string) (string, error) { + // Create new pin and store in datastore + pp := newPin(c, mode, name) + + // Serialize pin + pinData, err := encodePin(pp) + if err != nil { + return "", fmt.Errorf("could not encode pin: %v", err) + } + + p.setDirty(ctx) + + // Store the pin + err = p.dstore.Put(ctx, pp.dsKey(), pinData) + if err != nil { + return "", err + } + + // Store CID index + switch mode { + case ipfspinner.Recursive: + err = p.cidRIndex.Add(ctx, c.KeyString(), pp.Id) + case ipfspinner.Direct: + err = p.cidDIndex.Add(ctx, c.KeyString(), pp.Id) + default: + panic("pin mode must be recursive or direct") + } + if err != nil { + return "", fmt.Errorf("could not add pin cid index: %v", err) + } + + if name != "" { + // Store name index + err = p.nameIndex.Add(ctx, name, pp.Id) + if err != nil { + if mode == ipfspinner.Recursive { + e := p.cidRIndex.Delete(ctx, c.KeyString(), pp.Id) + if e != nil { + log.Errorf("error deleting index: %s", e) + } + } else { + e := p.cidDIndex.Delete(ctx, c.KeyString(), pp.Id) + if e != nil { + log.Errorf("error deleting index: %s", e) + } + } + return "", fmt.Errorf("could not add pin name index: %v", err) + } + } + + return pp.Id, nil +} + +func (p *pinner) removePin(ctx context.Context, pp *pin) error { + p.setDirty(ctx) + var err error + + // Remove cid index from datastore + if pp.Mode == ipfspinner.Recursive { + err = p.cidRIndex.Delete(ctx, pp.Cid.KeyString(), pp.Id) + } else { + err = p.cidDIndex.Delete(ctx, pp.Cid.KeyString(), pp.Id) + } + if err != nil { + return err + } + + if pp.Name != "" { + // Remove name index from datastore + err = p.nameIndex.Delete(ctx, pp.Name, pp.Id) + if err != nil { + return err + } + } + + // The pin is removed last so that an incomplete remove is detected by a + // pin that has a missing index. + err = p.dstore.Delete(ctx, pp.dsKey()) + if err != nil { + return err + } + + return nil +} + +// Unpin a given key +func (p *pinner) Unpin(ctx context.Context, c cid.Cid, recursive bool) error { + cidKey := c.KeyString() + + p.lock.Lock() + defer p.lock.Unlock() + + // TODO: use Ls() to lookup pins when new pinning API available + /* + matchSpec := map[string][]string { + "cid": []string{c.String} + } + matches := p.Ls(matchSpec) + */ + has, err := p.cidRIndex.HasAny(ctx, cidKey) + if err != nil { + return err + } + + if has { + if !recursive { + return fmt.Errorf("%s is pinned recursively", c.String()) + } + } else { + has, err = p.cidDIndex.HasAny(ctx, cidKey) + if err != nil { + return err + } + if !has { + return ipfspinner.ErrNotPinned + } + } + + removed, err := p.removePinsForCid(ctx, c, ipfspinner.Any) + if err != nil { + return err + } + if !removed { + return nil + } + + return p.flushPins(ctx, false) +} + +// IsPinned returns whether or not the given key is pinned +// and an explanation of why its pinned +func (p *pinner) IsPinned(ctx context.Context, c cid.Cid) (string, bool, error) { + p.lock.RLock() + defer p.lock.RUnlock() + return p.isPinnedWithType(ctx, c, ipfspinner.Any) +} + +// IsPinnedWithType returns whether or not the given cid is pinned with the +// given pin type, as well as returning the type of pin its pinned with. +func (p *pinner) IsPinnedWithType(ctx context.Context, c cid.Cid, mode ipfspinner.Mode) (string, bool, error) { + p.lock.RLock() + defer p.lock.RUnlock() + return p.isPinnedWithType(ctx, c, mode) +} + +func (p *pinner) isPinnedWithType(ctx context.Context, c cid.Cid, mode ipfspinner.Mode) (string, bool, error) { + cidKey := c.KeyString() + switch mode { + case ipfspinner.Recursive: + has, err := p.cidRIndex.HasAny(ctx, cidKey) + if err != nil { + return "", false, err + } + if has { + return linkRecursive, true, nil + } + return "", false, nil + case ipfspinner.Direct: + has, err := p.cidDIndex.HasAny(ctx, cidKey) + if err != nil { + return "", false, err + } + if has { + return linkDirect, true, nil + } + return "", false, nil + case ipfspinner.Internal: + return "", false, nil + case ipfspinner.Indirect: + case ipfspinner.Any: + has, err := p.cidRIndex.HasAny(ctx, cidKey) + if err != nil { + return "", false, err + } + if has { + return linkRecursive, true, nil + } + has, err = p.cidDIndex.HasAny(ctx, cidKey) + if err != nil { + return "", false, err + } + if has { + return linkDirect, true, nil + } + default: + err := fmt.Errorf( + "invalid Pin Mode '%d', must be one of {%d, %d, %d, %d, %d}", + mode, ipfspinner.Direct, ipfspinner.Indirect, ipfspinner.Recursive, + ipfspinner.Internal, ipfspinner.Any) + return "", false, err + } + + // Default is Indirect + visitedSet := cid.NewSet() + + // No index for given CID, so search children of all recursive pinned CIDs + var has bool + var rc cid.Cid + var e error + err := p.cidRIndex.ForEach(ctx, "", func(key, value string) bool { + rc, e = cid.Cast([]byte(key)) + if e != nil { + return false + } + has, e = hasChild(ctx, p.dserv, rc, c, visitedSet.Visit) + if e != nil { + return false + } + if has { + return false + } + return true + }) + if err != nil { + return "", false, err + } + if e != nil { + return "", false, e + } + + if has { + return rc.String(), true, nil + } + + return "", false, nil +} + +// CheckIfPinned checks if a set of keys are pinned, more efficient than +// calling IsPinned for each key, returns the pinned status of cid(s) +// +// TODO: If a CID is pinned by multiple pins, should they all be reported? +func (p *pinner) CheckIfPinned(ctx context.Context, cids ...cid.Cid) ([]ipfspinner.Pinned, error) { + pinned := make([]ipfspinner.Pinned, 0, len(cids)) + toCheck := cid.NewSet() + + p.lock.RLock() + defer p.lock.RUnlock() + + // First check for non-Indirect pins directly + for _, c := range cids { + cidKey := c.KeyString() + has, err := p.cidRIndex.HasAny(ctx, cidKey) + if err != nil { + return nil, err + } + if has { + pinned = append(pinned, ipfspinner.Pinned{Key: c, Mode: ipfspinner.Recursive}) + } else { + has, err = p.cidDIndex.HasAny(ctx, cidKey) + if err != nil { + return nil, err + } + if has { + pinned = append(pinned, ipfspinner.Pinned{Key: c, Mode: ipfspinner.Direct}) + } else { + toCheck.Add(c) + } + } + } + + var e error + visited := cid.NewSet() + err := p.cidRIndex.ForEach(ctx, "", func(key, value string) bool { + var rk cid.Cid + rk, e = cid.Cast([]byte(key)) + if e != nil { + return false + } + e = merkledag.Walk(ctx, merkledag.GetLinksWithDAG(p.dserv), rk, func(c cid.Cid) bool { + if toCheck.Len() == 0 || !visited.Visit(c) { + return false + } + + if toCheck.Has(c) { + pinned = append(pinned, ipfspinner.Pinned{Key: c, Mode: ipfspinner.Indirect, Via: rk}) + toCheck.Remove(c) + } + + return true + }, merkledag.Concurrent()) + if e != nil { + return false + } + return toCheck.Len() > 0 + }) + if err != nil { + return nil, err + } + if e != nil { + return nil, e + } + + // Anything left in toCheck is not pinned + for _, k := range toCheck.Keys() { + pinned = append(pinned, ipfspinner.Pinned{Key: k, Mode: ipfspinner.NotPinned}) + } + + return pinned, nil +} + +// removePinsForCid removes all pins for a cid that has the specified mode. +// Returns true if any pins, and all corresponding CID index entries, were +// removed. Otherwise, returns false. +func (p *pinner) removePinsForCid(ctx context.Context, c cid.Cid, mode ipfspinner.Mode) (bool, error) { + // Search for pins by CID + var ids []string + var err error + cidKey := c.KeyString() + switch mode { + case ipfspinner.Recursive: + ids, err = p.cidRIndex.Search(ctx, cidKey) + case ipfspinner.Direct: + ids, err = p.cidDIndex.Search(ctx, cidKey) + case ipfspinner.Any: + ids, err = p.cidRIndex.Search(ctx, cidKey) + if err != nil { + return false, err + } + dIds, err := p.cidDIndex.Search(ctx, cidKey) + if err != nil { + return false, err + } + if len(dIds) != 0 { + ids = append(ids, dIds...) + } + } + if err != nil { + return false, err + } + + var removed bool + + // Remove the pin with the requested mode + for _, pid := range ids { + var pp *pin + pp, err = p.loadPin(ctx, pid) + if err != nil { + if err == ds.ErrNotFound { + p.setDirty(ctx) + // Fix index; remove index for pin that does not exist + switch mode { + case ipfspinner.Recursive: + _, err = p.cidRIndex.DeleteKey(ctx, cidKey) + if err != nil { + return false, fmt.Errorf("error deleting index: %s", err) + } + case ipfspinner.Direct: + _, err = p.cidDIndex.DeleteKey(ctx, cidKey) + if err != nil { + return false, fmt.Errorf("error deleting index: %s", err) + } + case ipfspinner.Any: + _, err = p.cidRIndex.DeleteKey(ctx, cidKey) + if err != nil { + return false, fmt.Errorf("error deleting index: %s", err) + } + _, err = p.cidDIndex.DeleteKey(ctx, cidKey) + if err != nil { + return false, fmt.Errorf("error deleting index: %s", err) + } + } + if err = p.flushPins(ctx, true); err != nil { + return false, err + } + // Mark this as removed since it removed an index, which is + // what prevents determines if an item is pinned. + removed = true + log.Error("found CID index with missing pin") + continue + } + return false, err + } + if mode == ipfspinner.Any || pp.Mode == mode { + err = p.removePin(ctx, pp) + if err != nil { + return false, err + } + removed = true + } + } + return removed, nil +} + +// loadPin loads a single pin from the datastore. +func (p *pinner) loadPin(ctx context.Context, pid string) (*pin, error) { + pinData, err := p.dstore.Get(ctx, ds.NewKey(path.Join(pinKeyPath, pid))) + if err != nil { + return nil, err + } + return decodePin(pid, pinData) +} + +// DirectKeys returns a slice containing the directly pinned keys +func (p *pinner) DirectKeys(ctx context.Context) ([]cid.Cid, error) { + p.lock.RLock() + defer p.lock.RUnlock() + + cidSet := cid.NewSet() + var e error + err := p.cidDIndex.ForEach(ctx, "", func(key, value string) bool { + var c cid.Cid + c, e = cid.Cast([]byte(key)) + if e != nil { + return false + } + cidSet.Add(c) + return true + }) + if err != nil { + return nil, err + } + if e != nil { + return nil, e + } + + return cidSet.Keys(), nil +} + +// RecursiveKeys returns a slice containing the recursively pinned keys +func (p *pinner) RecursiveKeys(ctx context.Context) ([]cid.Cid, error) { + p.lock.RLock() + defer p.lock.RUnlock() + + cidSet := cid.NewSet() + var e error + err := p.cidRIndex.ForEach(ctx, "", func(key, value string) bool { + var c cid.Cid + c, e = cid.Cast([]byte(key)) + if e != nil { + return false + } + cidSet.Add(c) + return true + }) + if err != nil { + return nil, err + } + if e != nil { + return nil, e + } + + return cidSet.Keys(), nil +} + +// InternalPins returns all cids kept pinned for the internal state of the +// pinner +func (p *pinner) InternalPins(ctx context.Context) ([]cid.Cid, error) { + return nil, nil +} + +// Update updates a recursive pin from one cid to another. This is equivalent +// to pinning the new one and unpinning the old one. +// +// TODO: This will not work when multiple pins are supported +func (p *pinner) Update(ctx context.Context, from, to cid.Cid, unpin bool) error { + p.lock.Lock() + defer p.lock.Unlock() + + found, err := p.cidRIndex.HasAny(ctx, from.KeyString()) + if err != nil { + return err + } + if !found { + return errors.New("'from' cid was not recursively pinned already") + } + + // If `from` already recursively pinned and `to` is the same, then all done + if from == to { + return nil + } + + // Check if the `to` cid is already recursively pinned + found, err = p.cidRIndex.HasAny(ctx, to.KeyString()) + if err != nil { + return err + } + if found { + return errors.New("'to' cid was already recursively pinned") + } + + // Temporarily unlock while we fetch the differences. + p.lock.Unlock() + err = dagutils.DiffEnumerate(ctx, p.dserv, from, to) + p.lock.Lock() + + if err != nil { + return err + } + + _, err = p.addPin(ctx, to, ipfspinner.Recursive, "") + if err != nil { + return err + } + + if unpin { + _, err = p.removePinsForCid(ctx, from, ipfspinner.Recursive) + if err != nil { + return err + } + } + + return p.flushPins(ctx, false) +} + +func (p *pinner) flushDagService(ctx context.Context, force bool) error { + if !p.autoSync && !force { + return nil + } + if syncDServ, ok := p.dserv.(syncDAGService); ok { + if err := syncDServ.Sync(); err != nil { + return fmt.Errorf("cannot sync pinned data: %v", err) + } + } + return nil +} + +func (p *pinner) flushPins(ctx context.Context, force bool) error { + if !p.autoSync && !force { + return nil + } + if err := p.dstore.Sync(ctx, ds.NewKey(basePath)); err != nil { + return fmt.Errorf("cannot sync pin state: %v", err) + } + p.setClean(ctx) + return nil +} + +// Flush encodes and writes pinner keysets to the datastore +func (p *pinner) Flush(ctx context.Context) error { + p.lock.Lock() + defer p.lock.Unlock() + + err := p.flushDagService(ctx, true) + if err != nil { + return err + } + + return p.flushPins(ctx, true) +} + +// PinWithMode allows the user to have fine grained control over pin +// counts +func (p *pinner) PinWithMode(ctx context.Context, c cid.Cid, mode ipfspinner.Mode) error { + // TODO: remove his to support multiple pins per CID + switch mode { + case ipfspinner.Recursive: + return p.doPinRecursive(ctx, c, false) + case ipfspinner.Direct: + return p.doPinDirect(ctx, c) + default: + return fmt.Errorf("unrecognized pin mode") + } +} + +// hasChild recursively looks for a Cid among the children of a root Cid. +// The visit function can be used to shortcut already-visited branches. +func hasChild(ctx context.Context, ng ipld.NodeGetter, root cid.Cid, child cid.Cid, visit func(cid.Cid) bool) (bool, error) { + links, err := ipld.GetLinks(ctx, ng, root) + if err != nil { + return false, err + } + for _, lnk := range links { + c := lnk.Cid + if lnk.Cid.Equals(child) { + return true, nil + } + if visit(c) { + has, err := hasChild(ctx, ng, c, child, visit) + if err != nil { + return false, err + } + + if has { + return has, nil + } + } + } + return false, nil +} + +func encodePin(p *pin) ([]byte, error) { + b, err := cbor.MarshalAtlased(p, pinAtl) + if err != nil { + return nil, err + } + return b, nil +} + +func decodePin(pid string, data []byte) (*pin, error) { + p := &pin{Id: pid} + err := cbor.UnmarshalAtlased(cbor.DecodeOptions{}, data, p, pinAtl) + if err != nil { + return nil, err + } + return p, nil +} + +// setDirty updates the dirty counter and saves a dirty state in the datastore +// if the state was previously clean +func (p *pinner) setDirty(ctx context.Context) { + wasClean := p.dirty == p.clean + p.dirty++ + + if !wasClean { + return // do not save; was already dirty + } + + data := []byte{1} + err := p.dstore.Put(ctx, dirtyKey, data) + if err != nil { + log.Errorf("failed to set pin dirty flag: %s", err) + return + } + err = p.dstore.Sync(ctx, dirtyKey) + if err != nil { + log.Errorf("failed to sync pin dirty flag: %s", err) + } +} + +// setClean saves a clean state value in the datastore if the state was +// previously dirty +func (p *pinner) setClean(ctx context.Context) { + if p.dirty == p.clean { + return // already clean + } + + data := []byte{0} + err := p.dstore.Put(ctx, dirtyKey, data) + if err != nil { + log.Errorf("failed to set clear dirty flag: %s", err) + return + } + if err = p.dstore.Sync(ctx, dirtyKey); err != nil { + log.Errorf("failed to sync cleared pin dirty flag: %s", err) + return + } + p.clean = p.dirty // set clean +} + +// sync datastore after every 50 cid repairs +const syncRepairFrequency = 50 + +// rebuildIndexes uses the stored pins to rebuild secondary indexes. This +// resolves any discrepancy between secondary indexes and pins that could +// result from a program termination between saving the two. +func (p *pinner) rebuildIndexes(ctx context.Context) error { + // Load all pins from the datastore. + q := query.Query{ + Prefix: pinKeyPath, + } + results, err := p.dstore.Query(ctx, q) + if err != nil { + return err + } + defer results.Close() + + var checkedCount, repairedCount int + + // Iterate all pins and check if the corresponding recursive or direct + // index is missing. If the index is missing then create the index. + for r := range results.Next() { + if ctx.Err() != nil { + return ctx.Err() + } + if r.Error != nil { + return fmt.Errorf("cannot read index: %v", r.Error) + } + ent := r.Entry + pp, err := decodePin(path.Base(ent.Key), ent.Value) + if err != nil { + return err + } + + indexKey := pp.Cid.KeyString() + + var indexer, staleIndexer dsindex.Indexer + var idxrName, staleIdxrName string + if pp.Mode == ipfspinner.Recursive { + indexer = p.cidRIndex + staleIndexer = p.cidDIndex + idxrName = linkRecursive + staleIdxrName = linkDirect + } else if pp.Mode == ipfspinner.Direct { + indexer = p.cidDIndex + staleIndexer = p.cidRIndex + idxrName = linkDirect + staleIdxrName = linkRecursive + } else { + log.Error("unrecognized pin mode:", pp.Mode) + continue + } + + // Remove any stale index from unused indexer + ok, err := staleIndexer.HasValue(ctx, indexKey, pp.Id) + if err != nil { + return err + } + if ok { + // Delete any stale index + log.Errorf("deleting stale %s pin index for cid %v", staleIdxrName, pp.Cid.String()) + if err = staleIndexer.Delete(ctx, indexKey, pp.Id); err != nil { + return err + } + } + + // Check that the indexer indexes this pin + ok, err = indexer.HasValue(ctx, indexKey, pp.Id) + if err != nil { + return err + } + + var repaired bool + if !ok { + // Do not rebuild if index has an old value with leading slash + ok, err = indexer.HasValue(ctx, indexKey, "/"+pp.Id) + if err != nil { + return err + } + if !ok { + log.Errorf("repairing %s pin index for cid: %s", idxrName, pp.Cid.String()) + // There was no index found for this pin. This was either an + // incomplete add or and incomplete delete of a pin. Either + // way, restore the index to complete the add or to undo the + // incomplete delete. + if err = indexer.Add(ctx, indexKey, pp.Id); err != nil { + return err + } + repaired = true + } + } + // Check for missing name index + if pp.Name != "" { + ok, err = p.nameIndex.HasValue(ctx, pp.Name, pp.Id) + if err != nil { + return err + } + if !ok { + log.Errorf("repairing name pin index for cid: %s", pp.Cid.String()) + if err = p.nameIndex.Add(ctx, pp.Name, pp.Id); err != nil { + return err + } + } + repaired = true + } + + if repaired { + repairedCount++ + } + checkedCount++ + if checkedCount%syncRepairFrequency == 0 { + p.flushPins(ctx, true) + } + } + + log.Errorf("checked %d pins for invalid indexes, repaired %d pins", checkedCount, repairedCount) + return p.flushPins(ctx, true) +} diff --git a/pinning/pinner/dspinner/pin_test.go b/pinning/pinner/dspinner/pin_test.go new file mode 100644 index 0000000000..570479f9a5 --- /dev/null +++ b/pinning/pinner/dspinner/pin_test.go @@ -0,0 +1,1347 @@ +package dspinner + +import ( + "context" + "errors" + "fmt" + "io" + "path" + "testing" + "time" + + bs "github.com/ipfs/go-libipfs/blockservice" + mdag "github.com/ipfs/go-merkledag" + + cid "github.com/ipfs/go-cid" + ds "github.com/ipfs/go-datastore" + "github.com/ipfs/go-datastore/query" + dssync "github.com/ipfs/go-datastore/sync" + lds "github.com/ipfs/go-ds-leveldb" + ipld "github.com/ipfs/go-ipld-format" + blockstore "github.com/ipfs/go-libipfs/blockstore" + offline "github.com/ipfs/go-libipfs/exchange/offline" + util "github.com/ipfs/go-libipfs/util" + logging "github.com/ipfs/go-log" + + ipfspin "github.com/ipfs/go-libipfs/pinning/pinner" +) + +var rand = util.NewTimeSeededRand() + +type fakeLogger struct { + logging.StandardLogger + lastError error +} + +func (f *fakeLogger) Error(args ...interface{}) { + f.lastError = errors.New(fmt.Sprint(args...)) +} + +func (f *fakeLogger) Errorf(format string, args ...interface{}) { + f.lastError = fmt.Errorf(format, args...) +} + +func randNode() (*mdag.ProtoNode, cid.Cid) { + nd := new(mdag.ProtoNode) + nd.SetData(make([]byte, 32)) + _, err := io.ReadFull(rand, nd.Data()) + if err != nil { + panic(err) + } + k := nd.Cid() + return nd, k +} + +func assertPinned(t *testing.T, p ipfspin.Pinner, c cid.Cid, failmsg string) { + _, pinned, err := p.IsPinned(context.Background(), c) + if err != nil { + t.Fatal(err) + } + + if !pinned { + t.Fatal(failmsg) + } +} + +func assertPinnedWithType(t *testing.T, p ipfspin.Pinner, c cid.Cid, mode ipfspin.Mode, failmsg string) { + modeText, pinned, err := p.IsPinnedWithType(context.Background(), c, mode) + if err != nil { + t.Fatal(err) + } + + expect, ok := ipfspin.ModeToString(mode) + if !ok { + t.Fatal("unrecognized pin mode") + } + + if !pinned { + t.Fatal(failmsg) + } + + if mode == ipfspin.Any { + return + } + + if expect != modeText { + t.Fatal("expected", expect, "pin, got", modeText) + } +} + +func assertUnpinned(t *testing.T, p ipfspin.Pinner, c cid.Cid, failmsg string) { + _, pinned, err := p.IsPinned(context.Background(), c) + if err != nil { + t.Fatal(err) + } + + if pinned { + t.Fatal(failmsg) + } +} + +func TestPinnerBasic(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + dstore := dssync.MutexWrap(ds.NewMapDatastore()) + bstore := blockstore.NewBlockstore(dstore) + bserv := bs.New(bstore, offline.Exchange(bstore)) + + dserv := mdag.NewDAGService(bserv) + + p, err := New(ctx, dstore, dserv) + if err != nil { + t.Fatal(err) + } + + a, ak := randNode() + err = dserv.Add(ctx, a) + if err != nil { + t.Fatal(err) + } + + // Pin A{} + err = p.Pin(ctx, a, false) + if err != nil { + t.Fatal(err) + } + + assertPinned(t, p, ak, "Failed to find key") + assertPinnedWithType(t, p, ak, ipfspin.Direct, "Expected direct pin") + + // create new node c, to be indirectly pinned through b + c, _ := randNode() + err = dserv.Add(ctx, c) + if err != nil { + t.Fatal(err) + } + ck := c.Cid() + + // Create new node b, to be parent to a and c + b, _ := randNode() + err = b.AddNodeLink("child", a) + if err != nil { + t.Fatal(err) + } + err = b.AddNodeLink("otherchild", c) + if err != nil { + t.Fatal(err) + } + + err = dserv.Add(ctx, b) + if err != nil { + t.Fatal(err) + } + bk := b.Cid() + + // recursively pin B{A,C} + err = p.Pin(ctx, b, true) + if err != nil { + t.Fatal(err) + } + + assertPinned(t, p, ck, "child of recursively pinned node not found") + + assertPinned(t, p, bk, "Pinned node not found") + assertPinnedWithType(t, p, bk, ipfspin.Recursive, "Recursively pinned node not found") + + d, _ := randNode() + err = d.AddNodeLink("a", a) + if err != nil { + panic(err) + } + err = d.AddNodeLink("c", c) + if err != nil { + panic(err) + } + + e, _ := randNode() + err = d.AddNodeLink("e", e) + if err != nil { + panic(err) + } + + // Must be in dagserv for unpin to work + err = dserv.Add(ctx, e) + if err != nil { + t.Fatal(err) + } + err = dserv.Add(ctx, d) + if err != nil { + t.Fatal(err) + } + + // Add D{A,C,E} + err = p.Pin(ctx, d, true) + if err != nil { + t.Fatal(err) + } + + dk := d.Cid() + assertPinned(t, p, dk, "pinned node not found.") + + cids, err := p.RecursiveKeys(ctx) + if err != nil { + t.Fatal(err) + } + if len(cids) != 2 { + t.Error("expected 2 recursive pins") + } + if !(bk == cids[0] || bk == cids[1]) { + t.Error("expected recursive pin of B") + } + if !(dk == cids[0] || dk == cids[1]) { + t.Error("expected recursive pin of D") + } + + pinned, err := p.CheckIfPinned(ctx, ak, bk, ck, dk) + if err != nil { + t.Fatal(err) + } + if len(pinned) != 4 { + t.Error("incorrect number of results") + } + for _, pn := range pinned { + switch pn.Key { + case ak: + if pn.Mode != ipfspin.Direct { + t.Error("A pinned with wrong mode") + } + case bk: + if pn.Mode != ipfspin.Recursive { + t.Error("B pinned with wrong mode") + } + case ck: + if pn.Mode != ipfspin.Indirect { + t.Error("C should be pinned indirectly") + } + if pn.Via != dk && pn.Via != bk { + t.Error("C should be pinned via D or B") + } + case dk: + if pn.Mode != ipfspin.Recursive { + t.Error("D pinned with wrong mode") + } + } + } + + cids, err = p.DirectKeys(ctx) + if err != nil { + t.Fatal(err) + } + if len(cids) != 1 { + t.Error("expected 1 direct pin") + } + if cids[0] != ak { + t.Error("wrong direct pin") + } + + cids, _ = p.InternalPins(ctx) + if len(cids) != 0 { + t.Error("shound not have internal keys") + } + + err = p.Unpin(ctx, dk, false) + if err == nil { + t.Fatal("expected error unpinning recursive pin without specifying recursive") + } + + // Test recursive unpin + err = p.Unpin(ctx, dk, true) + if err != nil { + t.Fatal(err) + } + + err = p.Unpin(ctx, dk, true) + if err != ipfspin.ErrNotPinned { + t.Fatal("expected error:", ipfspin.ErrNotPinned) + } + + err = p.Flush(ctx) + if err != nil { + t.Fatal(err) + } + + p, err = New(ctx, dstore, dserv) + if err != nil { + t.Fatal(err) + } + + // Test directly pinned + assertPinned(t, p, ak, "Could not find pinned node!") + + // Test recursively pinned + assertPinned(t, p, bk, "could not find recursively pinned node") + + // Remove the pin but not the index to simulate corruption + ids, err := p.cidDIndex.Search(ctx, ak.KeyString()) + if err != nil { + t.Fatal(err) + } + if len(ids) == 0 { + t.Fatal("did not find pin for cid", ak.String()) + } + pp, err := p.loadPin(ctx, ids[0]) + if err != nil { + t.Fatal(err) + } + if pp.Mode != ipfspin.Direct { + t.Error("loaded pin has wrong mode") + } + if pp.Cid != ak { + t.Error("loaded pin has wrong cid") + } + err = p.dstore.Delete(ctx, pp.dsKey()) + if err != nil { + t.Fatal(err) + } + + realLog := log + fakeLog := &fakeLogger{} + fakeLog.StandardLogger = log + log = fakeLog + err = p.Pin(ctx, a, true) + if err != nil { + t.Fatal(err) + } + if fakeLog.lastError == nil { + t.Error("expected error to be logged") + } else if fakeLog.lastError.Error() != "found CID index with missing pin" { + t.Error("did not get expected log message") + } + + log = realLog +} + +func TestAddLoadPin(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + dstore := dssync.MutexWrap(ds.NewMapDatastore()) + bstore := blockstore.NewBlockstore(dstore) + bserv := bs.New(bstore, offline.Exchange(bstore)) + + dserv := mdag.NewDAGService(bserv) + + p, err := New(ctx, dstore, dserv) + if err != nil { + t.Fatal(err) + } + + a, ak := randNode() + err = dserv.Add(ctx, a) + if err != nil { + panic(err) + } + + mode := ipfspin.Recursive + name := "my-pin" + pid, err := p.addPin(ctx, ak, mode, name) + if err != nil { + t.Fatal(err) + } + + // Load pin and check that data decoded correctly + pinData, err := p.loadPin(ctx, pid) + if err != nil { + t.Fatal(err) + } + if pinData.Mode != mode { + t.Error("worng pin mode") + } + if pinData.Cid != ak { + t.Error("wrong pin cid") + } + if pinData.Name != name { + t.Error("wrong pin name; expected", name, "got", pinData.Name) + } +} + +func TestIsPinnedLookup(t *testing.T) { + // Test that lookups work in pins which share + // the same branches. For that construct this tree: + // + // A5->A4->A3->A2->A1->A0 + // / / + // B------- / + // \ / + // C--------------- + // + // This ensures that IsPinned works for all objects both when they + // are pinned and once they have been unpinned. + aBranchLen := 6 + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + dstore := dssync.MutexWrap(ds.NewMapDatastore()) + bstore := blockstore.NewBlockstore(dstore) + bserv := bs.New(bstore, offline.Exchange(bstore)) + + dserv := mdag.NewDAGService(bserv) + + // Create new pinner. New will not load anything since there are + // no pins saved in the datastore yet. + p, err := New(ctx, dstore, dserv) + if err != nil { + t.Fatal(err) + } + + aKeys, bk, ck, err := makeTree(ctx, aBranchLen, dserv, p) + if err != nil { + t.Fatal(err) + } + + assertPinned(t, p, aKeys[0], "A0 should be pinned") + assertPinned(t, p, aKeys[1], "A1 should be pinned") + assertPinned(t, p, ck, "C should be pinned") + assertPinned(t, p, bk, "B should be pinned") + + // Unpin A5 recursively + if err = p.Unpin(ctx, aKeys[5], true); err != nil { + t.Fatal(err) + } + + assertPinned(t, p, aKeys[0], "A0 should still be pinned through B") + assertUnpinned(t, p, aKeys[4], "A4 should be unpinned") + + // Unpin B recursively + if err = p.Unpin(ctx, bk, true); err != nil { + t.Fatal(err) + } + assertUnpinned(t, p, bk, "B should be unpinned") + assertUnpinned(t, p, aKeys[1], "A1 should be unpinned") + assertPinned(t, p, aKeys[0], "A0 should still be pinned through C") +} + +func TestDuplicateSemantics(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + dstore := dssync.MutexWrap(ds.NewMapDatastore()) + bstore := blockstore.NewBlockstore(dstore) + bserv := bs.New(bstore, offline.Exchange(bstore)) + + dserv := mdag.NewDAGService(bserv) + + p, err := New(ctx, dstore, dserv) + if err != nil { + t.Fatal(err) + } + + a, _ := randNode() + err = dserv.Add(ctx, a) + if err != nil { + t.Fatal(err) + } + + // pin is recursively + err = p.Pin(ctx, a, true) + if err != nil { + t.Fatal(err) + } + + // pinning directly should fail + err = p.Pin(ctx, a, false) + if err == nil { + t.Fatal("expected direct pin to fail") + } + + // pinning recursively again should succeed + err = p.Pin(ctx, a, true) + if err != nil { + t.Fatal(err) + } +} + +func TestFlush(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + dstore := dssync.MutexWrap(ds.NewMapDatastore()) + bstore := blockstore.NewBlockstore(dstore) + bserv := bs.New(bstore, offline.Exchange(bstore)) + + dserv := mdag.NewDAGService(bserv) + p, err := New(ctx, dstore, dserv) + if err != nil { + t.Fatal(err) + } + _, k := randNode() + + p.PinWithMode(ctx, k, ipfspin.Recursive) + if err = p.Flush(ctx); err != nil { + t.Fatal(err) + } + assertPinned(t, p, k, "expected key to still be pinned") +} + +func TestPinRecursiveFail(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + dstore := dssync.MutexWrap(ds.NewMapDatastore()) + bstore := blockstore.NewBlockstore(dstore) + bserv := bs.New(bstore, offline.Exchange(bstore)) + dserv := mdag.NewDAGService(bserv) + + p, err := New(ctx, dstore, dserv) + if err != nil { + t.Fatal(err) + } + + a, _ := randNode() + b, _ := randNode() + err = a.AddNodeLink("child", b) + if err != nil { + t.Fatal(err) + } + + // NOTE: This isnt a time based test, we expect the pin to fail + mctx, cancel := context.WithTimeout(ctx, time.Millisecond) + defer cancel() + + err = p.Pin(mctx, a, true) + if err == nil { + t.Fatal("should have failed to pin here") + } + + err = dserv.Add(ctx, b) + if err != nil { + t.Fatal(err) + } + + err = dserv.Add(ctx, a) + if err != nil { + t.Fatal(err) + } + + // this one is time based... but shouldnt cause any issues + mctx, cancel = context.WithTimeout(ctx, time.Second) + defer cancel() + err = p.Pin(mctx, a, true) + if err != nil { + t.Fatal(err) + } +} + +func TestPinUpdate(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + dstore := dssync.MutexWrap(ds.NewMapDatastore()) + bstore := blockstore.NewBlockstore(dstore) + bserv := bs.New(bstore, offline.Exchange(bstore)) + + dserv := mdag.NewDAGService(bserv) + p, err := New(ctx, dstore, dserv) + if err != nil { + t.Fatal(err) + } + n1, c1 := randNode() + n2, c2 := randNode() + _, c3 := randNode() + + if err = dserv.Add(ctx, n1); err != nil { + t.Fatal(err) + } + if err = dserv.Add(ctx, n2); err != nil { + t.Fatal(err) + } + + if err = p.Pin(ctx, n1, true); err != nil { + t.Fatal(err) + } + + if err = p.Update(ctx, c1, c2, true); err != nil { + t.Fatal(err) + } + + assertPinned(t, p, c2, "c2 should be pinned now") + assertUnpinned(t, p, c1, "c1 should no longer be pinned") + + if err = p.Update(ctx, c2, c1, false); err != nil { + t.Fatal(err) + } + + // Test updating same pin that is already pinned. + if err = p.Update(ctx, c2, c2, true); err != nil { + t.Fatal(err) + } + // Check that pin is still pinned. + _, ok, err := p.IsPinned(ctx, c2) + if err != nil { + t.Fatal(err) + } + if !ok { + t.Fatal("c2 should still be pinned") + } + + // Test updating same pin that is not pinned. + if err = p.Update(ctx, c3, c3, false); err == nil { + t.Fatal("expected error updating unpinned cid") + } + _, ok, err = p.IsPinned(ctx, c3) + if err != nil { + t.Fatal(err) + } + if ok { + t.Fatal("c3 should not be pinned") + } + + assertPinned(t, p, c2, "c2 should be pinned still") + assertPinned(t, p, c1, "c1 should be pinned now") +} + +func TestLoadDirty(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + dstore := dssync.MutexWrap(ds.NewMapDatastore()) + bstore := blockstore.NewBlockstore(dstore) + bserv := bs.New(bstore, offline.Exchange(bstore)) + dserv := mdag.NewDAGService(bserv) + + p, err := New(ctx, dstore, dserv) + if err != nil { + t.Fatal(err) + } + prev := p.SetAutosync(false) + if !prev { + t.Fatal("expected previous autosync to be true") + } + prev = p.SetAutosync(false) + if prev { + t.Fatal("expected previous autosync to be false") + } + prev = p.SetAutosync(true) + if prev { + t.Fatal("expected previous autosync to be false") + } + + a, ak := randNode() + err = dserv.Add(ctx, a) + if err != nil { + t.Fatal(err) + } + + _, bk := randNode() + + err = p.Pin(ctx, a, true) + if err != nil { + t.Fatal(err) + } + + cidAKey := ak.KeyString() + cidBKey := bk.KeyString() + + // Corrupt index + cidRIndex := p.cidRIndex + _, err = cidRIndex.DeleteKey(ctx, cidAKey) + if err != nil { + t.Fatal(err) + } + err = cidRIndex.Add(ctx, cidBKey, "not-a-pin-id") + if err != nil { + t.Fatal(err) + } + + // Force dirty, since Pin syncs automatically + p.setDirty(ctx) + + // Verify dirty + data, err := dstore.Get(ctx, dirtyKey) + if err != nil { + t.Fatalf("could not read dirty flag: %v", err) + } + if data[0] != 1 { + t.Fatal("dirty flag not set") + } + + has, err := cidRIndex.HasAny(ctx, cidAKey) + if err != nil { + t.Fatal(err) + } + if has { + t.Fatal("index should be deleted") + } + + // Create new pinner on same datastore that was never flushed. This should + // detect the dirty flag and repair the indexes. + p, err = New(ctx, dstore, dserv) + if err != nil { + t.Fatal(err) + } + + // Verify not dirty + data, err = dstore.Get(ctx, dirtyKey) + if err != nil { + t.Fatalf("could not read dirty flag: %v", err) + } + if data[0] != 0 { + t.Fatal("dirty flag is set") + } + + // Verify index rebuilt + cidRIndex = p.cidRIndex + has, err = cidRIndex.HasAny(ctx, cidAKey) + if err != nil { + t.Fatal(err) + } + if !has { + t.Fatal("index should have been rebuilt") + } + + has, err = p.removePinsForCid(ctx, bk, ipfspin.Any) + if err != nil { + t.Fatal(err) + } + if !has { + t.Fatal("expected Unpin to return true since index removed") + } +} + +func TestEncodeDecodePin(t *testing.T) { + _, c := randNode() + + pin := newPin(c, ipfspin.Recursive, "testpin") + pin.Metadata = make(map[string]interface{}, 2) + pin.Metadata["hello"] = "world" + pin.Metadata["foo"] = "bar" + + encBytes, err := encodePin(pin) + if err != nil { + t.Fatal(err) + } + + decPin, err := decodePin(pin.Id, encBytes) + if err != nil { + t.Fatal(err) + } + + if decPin.Id != pin.Id { + t.Errorf("wrong pin id: expect %q got %q", pin.Id, decPin.Id) + } + if decPin.Cid != pin.Cid { + t.Errorf("wrong pin cid: expect %q got %q", pin.Cid.String(), decPin.Cid.String()) + } + if decPin.Mode != pin.Mode { + expect, _ := ipfspin.ModeToString(pin.Mode) + got, _ := ipfspin.ModeToString(decPin.Mode) + t.Errorf("wrong pin mode: expect %s got %s", expect, got) + } + if decPin.Name != pin.Name { + t.Errorf("wrong pin name: expect %q got %q", pin.Name, decPin.Name) + } + for key, val := range pin.Metadata { + dval, ok := decPin.Metadata[key] + if !ok { + t.Errorf("decoded pin missing metadata key %q", key) + } + if dval != val { + t.Errorf("wrong metadata value: expected %q got %q", val, dval) + } + } +} + +func makeTree(ctx context.Context, aBranchLen int, dserv ipld.DAGService, p ipfspin.Pinner) (aKeys []cid.Cid, bk cid.Cid, ck cid.Cid, err error) { + if aBranchLen < 3 { + err = errors.New("set aBranchLen to at least 3") + return + } + + aNodes := make([]*mdag.ProtoNode, aBranchLen) + aKeys = make([]cid.Cid, aBranchLen) + for i := 0; i < aBranchLen; i++ { + a, _ := randNode() + if i >= 1 { + if err = a.AddNodeLink("child", aNodes[i-1]); err != nil { + return + } + } + + if err = dserv.Add(ctx, a); err != nil { + return + } + aNodes[i] = a + aKeys[i] = a.Cid() + } + + // Pin last A recursively + if err = p.Pin(ctx, aNodes[aBranchLen-1], true); err != nil { + return + } + + // Create node B and add A3 as child + b, _ := randNode() + if err = b.AddNodeLink("mychild", aNodes[3]); err != nil { + return + } + + // Create C node + c, _ := randNode() + // Add A0 as child of C + if err = c.AddNodeLink("child", aNodes[0]); err != nil { + return + } + + // Add C + if err = dserv.Add(ctx, c); err != nil { + return + } + ck = c.Cid() + + // Add C to B and Add B + if err = b.AddNodeLink("myotherchild", c); err != nil { + return + } + if err = dserv.Add(ctx, b); err != nil { + return + } + bk = b.Cid() + + // Pin C recursively + if err = p.Pin(ctx, c, true); err != nil { + return + } + + // Pin B recursively + if err = p.Pin(ctx, b, true); err != nil { + return + } + + if err = p.Flush(ctx); err != nil { + return + } + + return +} + +func makeNodes(count int, dserv ipld.DAGService) []ipld.Node { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + nodes := make([]ipld.Node, count) + for i := 0; i < count; i++ { + n, _ := randNode() + err := dserv.Add(ctx, n) + if err != nil { + panic(err) + } + nodes[i] = n + } + return nodes +} + +func pinNodes(nodes []ipld.Node, p ipfspin.Pinner, recursive bool) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + var err error + + for i := range nodes { + err = p.Pin(ctx, nodes[i], recursive) + if err != nil { + panic(err) + } + } + err = p.Flush(ctx) + if err != nil { + panic(err) + } +} + +func unpinNodes(nodes []ipld.Node, p ipfspin.Pinner) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + var err error + + for i := range nodes { + err = p.Unpin(ctx, nodes[i].Cid(), true) + if err != nil { + panic(err) + } + } + err = p.Flush(ctx) + if err != nil { + panic(err) + } +} + +type batchWrap struct { + ds.Datastore +} + +func (d *batchWrap) Batch(_ context.Context) (ds.Batch, error) { + return ds.NewBasicBatch(d), nil +} + +func makeStore() (ds.Datastore, ipld.DAGService) { + ldstore, err := lds.NewDatastore("", nil) + if err != nil { + panic(err) + } + dstore := &batchWrap{ldstore} + bstore := blockstore.NewBlockstore(dstore) + bserv := bs.New(bstore, offline.Exchange(bstore)) + dserv := mdag.NewDAGService(bserv) + return dstore, dserv +} + +// BenchmarkLoadRebuild loads a pinner that has some number of saved pins, and +// compares the load time when rebuilding indexes to loading without rebuilding +// indexes. +func BenchmarkLoad(b *testing.B) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + dstore, dserv := makeStore() + pinner, err := New(ctx, dstore, dserv) + if err != nil { + panic(err.Error()) + } + + nodes := makeNodes(4096, dserv) + pinNodes(nodes, pinner, true) + + b.Run("RebuildTrue", func(b *testing.B) { + for i := 0; i < b.N; i++ { + err = dstore.Put(ctx, dirtyKey, []byte{1}) + if err != nil { + panic(err.Error()) + } + + _, err = New(ctx, dstore, dserv) + if err != nil { + panic(err.Error()) + } + } + }) + + b.Run("RebuildFalse", func(b *testing.B) { + for i := 0; i < b.N; i++ { + err = dstore.Put(ctx, dirtyKey, []byte{0}) + if err != nil { + panic(err.Error()) + } + + _, err = New(ctx, dstore, dserv) + if err != nil { + panic(err.Error()) + } + } + }) +} + +// BenchmarkNthPins shows the time it takes to create/save 1 pin when a number +// of other pins already exist. Each run in the series shows performance for +// creating a pin in a larger number of existing pins. +func BenchmarkNthPin(b *testing.B) { + dstore, dserv := makeStore() + pinner, err := New(context.Background(), dstore, dserv) + if err != nil { + panic(err.Error()) + } + + for count := 1000; count <= 10000; count += 1000 { + b.Run(fmt.Sprint("PinDS-", count), func(b *testing.B) { + benchmarkNthPin(b, count, pinner, dserv) + }) + } +} + +func benchmarkNthPin(b *testing.B, count int, pinner ipfspin.Pinner, dserv ipld.DAGService) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + nodes := makeNodes(count, dserv) + pinNodes(nodes[:count-1], pinner, true) + b.ResetTimer() + + which := count - 1 + for i := 0; i < b.N; i++ { + // Pin the Nth node and Flush + err := pinner.Pin(ctx, nodes[which], true) + if err != nil { + panic(err) + } + err = pinner.Flush(ctx) + if err != nil { + panic(err) + } + // Unpin the nodes so that it can pinned next iter. + b.StopTimer() + err = pinner.Unpin(ctx, nodes[which].Cid(), true) + if err != nil { + panic(err) + } + err = pinner.Flush(ctx) + if err != nil { + panic(err) + } + b.StartTimer() + } +} + +// BenchmarkNPins demonstrates creating individual pins. Each run in the +// series shows performance for a larger number of individual pins. +func BenchmarkNPins(b *testing.B) { + for count := 128; count < 16386; count <<= 1 { + b.Run(fmt.Sprint("PinDS-", count), func(b *testing.B) { + dstore, dserv := makeStore() + pinner, err := New(context.Background(), dstore, dserv) + if err != nil { + panic(err.Error()) + } + benchmarkNPins(b, count, pinner, dserv) + }) + } +} + +func benchmarkNPins(b *testing.B, count int, pinner ipfspin.Pinner, dserv ipld.DAGService) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + nodes := makeNodes(count, dserv) + b.ResetTimer() + + for i := 0; i < b.N; i++ { + // Pin all the nodes one at a time. + for j := range nodes { + err := pinner.Pin(ctx, nodes[j], true) + if err != nil { + panic(err) + } + err = pinner.Flush(ctx) + if err != nil { + panic(err) + } + } + + // Unpin all nodes so that they can be pinned next iter. + b.StopTimer() + unpinNodes(nodes, pinner) + b.StartTimer() + } +} + +// BenchmarkNUnpins demonstrates unpinning individual pins. Each run in the +// series shows performance for a larger number of individual unpins. +func BenchmarkNUnpins(b *testing.B) { + for count := 128; count < 16386; count <<= 1 { + b.Run(fmt.Sprint("UnpinDS-", count), func(b *testing.B) { + dstore, dserv := makeStore() + pinner, err := New(context.Background(), dstore, dserv) + if err != nil { + panic(err.Error()) + } + benchmarkNUnpins(b, count, pinner, dserv) + }) + } +} + +func benchmarkNUnpins(b *testing.B, count int, pinner ipfspin.Pinner, dserv ipld.DAGService) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + nodes := makeNodes(count, dserv) + pinNodes(nodes, pinner, true) + b.ResetTimer() + + for i := 0; i < b.N; i++ { + for j := range nodes { + // Unpin nodes one at a time. + err := pinner.Unpin(ctx, nodes[j].Cid(), true) + if err != nil { + panic(err) + } + err = pinner.Flush(ctx) + if err != nil { + panic(err) + } + } + // Pin all nodes so that they can be unpinned next iter. + b.StopTimer() + pinNodes(nodes, pinner, true) + b.StartTimer() + } +} + +// BenchmarkPinAllSeries shows times to pin all nodes with only one Flush at +// the end. +func BenchmarkPinAll(b *testing.B) { + for count := 128; count < 16386; count <<= 1 { + b.Run(fmt.Sprint("PinAllDS-", count), func(b *testing.B) { + dstore, dserv := makeStore() + pinner, err := New(context.Background(), dstore, dserv) + if err != nil { + panic(err) + } + benchmarkPinAll(b, count, pinner, dserv) + }) + } +} + +func benchmarkPinAll(b *testing.B, count int, pinner ipfspin.Pinner, dserv ipld.DAGService) { + nodes := makeNodes(count, dserv) + b.ResetTimer() + + for i := 0; i < b.N; i++ { + pinNodes(nodes, pinner, true) + + b.StopTimer() + unpinNodes(nodes, pinner) + b.StartTimer() + } +} + +func BenchmarkRebuild(b *testing.B) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + dstore, dserv := makeStore() + pinIncr := 32768 + + for pins := pinIncr; pins <= pinIncr*5; pins += pinIncr { + pinner, err := New(ctx, dstore, dserv) + if err != nil { + panic(err.Error()) + } + nodes := makeNodes(pinIncr, dserv) + pinNodes(nodes, pinner, true) + + b.Run(fmt.Sprintf("Rebuild %d", pins), func(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + err = dstore.Put(ctx, dirtyKey, []byte{1}) + if err != nil { + panic(err.Error()) + } + + _, err = New(ctx, dstore, dserv) + if err != nil { + panic(err.Error()) + } + } + }) + } +} + +func TestCidIndex(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + dstore, dserv := makeStore() + pinner, err := New(ctx, dstore, dserv) + if err != nil { + t.Fatal(err) + } + nodes := makeNodes(1, dserv) + node := nodes[0] + + c := node.Cid() + cidKey := c.KeyString() + + // Pin the cid + pid, err := pinner.addPin(ctx, c, ipfspin.Recursive, "") + if err != nil { + t.Fatal(err) + } + + t.Log("Added pin:", pid) + t.Log("CID index:", c.String(), "-->", pid) + + // Check that the index exists + ok, err := pinner.cidRIndex.HasAny(ctx, cidKey) + if err != nil { + t.Fatal(err) + } + if !ok { + t.Fatal("R-index has no value for", cidKey) + } + + // Check that searching for the cid returns a value + values, err := pinner.cidRIndex.Search(ctx, cidKey) + if err != nil { + t.Fatal(err) + } + if len(values) != 1 { + t.Fatal("expect index to return one value") + } + if values[0] != pid { + t.Fatal("indexer should have has value", cidKey, "-->", pid) + } + + // Check that index has specific value + ok, err = pinner.cidRIndex.HasValue(ctx, cidKey, pid) + if err != nil { + t.Fatal(err) + } + if !ok { + t.Fatal("indexer should have has value", cidKey, "-->", pid) + } + + // Iterate values of index + var seen bool + err = pinner.cidRIndex.ForEach(ctx, "", func(key, value string) bool { + if seen { + t.Fatal("expected one key-value pair") + } + if key != cidKey { + t.Fatal("unexpected key:", key) + } + if value != pid { + t.Fatal("unexpected value:", value) + } + seen = true + return true + }) + if err != nil { + t.Fatal(err) + } + + // Load all pins from the datastore. + q := query.Query{ + Prefix: pinKeyPath, + } + results, err := pinner.dstore.Query(ctx, q) + if err != nil { + t.Fatal(err) + } + defer results.Close() + + // Iterate all pins and check if the corresponding recursive or direct + // index is missing. If the index is missing then create the index. + seen = false + for r := range results.Next() { + if seen { + t.Fatal("has more than one pin") + } + if r.Error != nil { + t.Fatal(fmt.Errorf("cannot read index: %v", r.Error)) + } + ent := r.Entry + pp, err := decodePin(path.Base(ent.Key), ent.Value) + if err != nil { + t.Fatal(err) + } + t.Log("Found pin:", pp.Id) + if pp.Id != pid { + t.Fatal("ID of loaded pin is not the same known to indexer") + } + seen = true + } +} + +func TestRebuild(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + dstore, dserv := makeStore() + pinner, err := New(ctx, dstore, dserv) + if err != nil { + t.Fatal(err) + } + nodes := makeNodes(3, dserv) + pinNodes(nodes, pinner, true) + + c1 := nodes[0].Cid() + cid1Key := c1.KeyString() + c2 := nodes[1].Cid() + cid2Key := c2.KeyString() + c3 := nodes[2].Cid() + cid3Key := c3.KeyString() + + // Get pin IDs + values, err := pinner.cidRIndex.Search(ctx, cid1Key) + if err != nil { + t.Fatal(err) + } + pid1 := values[0] + values, err = pinner.cidRIndex.Search(ctx, cid2Key) + if err != nil { + t.Fatal(err) + } + pid2 := values[0] + values, err = pinner.cidRIndex.Search(ctx, cid3Key) + if err != nil { + t.Fatal(err) + } + pid3 := values[0] + + // Corrupt by adding direct index when there is already a recursive index + err = pinner.cidDIndex.Add(ctx, cid1Key, pid1) + if err != nil { + t.Fatal(err) + } + + // Corrupt index by deleting cid index 2 to simulate an incomplete add or delete + _, err = pinner.cidRIndex.DeleteKey(ctx, cid2Key) + if err != nil { + t.Fatal(err) + } + + // Corrupt index by deleting pin to simulate corruption + var pp *pin + pp, err = pinner.loadPin(ctx, pid3) + if err != nil { + t.Fatal(err) + } + err = pinner.dstore.Delete(ctx, pp.dsKey()) + if err != nil { + t.Fatal(err) + } + + pinner.setDirty(ctx) + + // Rebuild indexes + pinner, err = New(ctx, dstore, dserv) + if err != nil { + t.Fatal(err) + } + + // Verify that indexes have same values as before + err = verifyIndexValue(ctx, pinner, cid1Key, pid1) + if err != nil { + t.Fatal(err) + } + err = verifyIndexValue(ctx, pinner, cid2Key, pid2) + if err != nil { + t.Fatal(err) + } + err = verifyIndexValue(ctx, pinner, cid3Key, pid3) + if err != nil { + t.Fatal(err) + } +} + +func verifyIndexValue(ctx context.Context, pinner *pinner, cidKey, expectedPid string) error { + values, err := pinner.cidRIndex.Search(ctx, cidKey) + if err != nil { + return err + } + if len(values) != 1 { + return errors.New("expected 1 value") + } + if expectedPid != values[0] { + return errors.New("index has wrong value") + } + ok, err := pinner.cidDIndex.HasAny(ctx, cidKey) + if err != nil { + return err + } + if ok { + return errors.New("should not have a direct index") + } + return nil +} diff --git a/pinning/pinner/dspinner/sync_test.go b/pinning/pinner/dspinner/sync_test.go new file mode 100644 index 0000000000..a59a06c821 --- /dev/null +++ b/pinning/pinner/dspinner/sync_test.go @@ -0,0 +1,88 @@ +package dspinner + +import ( + "context" + "os" + "testing" + + ds "github.com/ipfs/go-datastore" + bds "github.com/ipfs/go-ds-badger" + lds "github.com/ipfs/go-ds-leveldb" + ipld "github.com/ipfs/go-ipld-format" + bs "github.com/ipfs/go-libipfs/blockservice" + blockstore "github.com/ipfs/go-libipfs/blockstore" + offline "github.com/ipfs/go-libipfs/exchange/offline" + mdag "github.com/ipfs/go-merkledag" +) + +func makeStoreLevelDB(dir string) (ds.Datastore, ipld.DAGService) { + ldstore, err := lds.NewDatastore(dir, nil) + if err != nil { + panic(err) + } + // dstore := &batchWrap{ldstore} + dstore := ldstore + bstore := blockstore.NewBlockstore(dstore) + bserv := bs.New(bstore, offline.Exchange(bstore)) + dserv := mdag.NewDAGService(bserv) + return dstore, dserv +} + +func makeStoreBadger(dir string) (ds.Datastore, ipld.DAGService) { + bdstore, err := bds.NewDatastore(dir, nil) + if err != nil { + panic(err) + } + dstore := &batchWrap{bdstore} + bstore := blockstore.NewBlockstore(dstore) + bserv := bs.New(bstore, offline.Exchange(bstore)) + dserv := mdag.NewDAGService(bserv) + return dstore, dserv +} + +func benchAutoSync(b *testing.B, N int, auto bool, dstore ds.Datastore, dserv ipld.DAGService) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + pinner, err := New(ctx, dstore, dserv) + if err != nil { + panic(err.Error()) + } + + nodes := makeNodes(N, dserv) + + pinner.SetAutosync(auto) + pinNodes(nodes, pinner, true) +} + +func BenchmarkSyncOnceBadger(b *testing.B) { + const dsDir = "b-once" + dstoreB1, dservB1 := makeStoreBadger(dsDir) + defer os.RemoveAll(dsDir) + benchAutoSync(b, b.N, false, dstoreB1, dservB1) + dstoreB1.Close() +} + +func BenchmarkSyncEveryBadger(b *testing.B) { + const dsDir = "b-every" + dstoreB2, dservB2 := makeStoreBadger(dsDir) + defer os.RemoveAll(dsDir) + benchAutoSync(b, b.N, true, dstoreB2, dservB2) + dstoreB2.Close() +} + +func BenchmarkSyncOnceLevelDB(b *testing.B) { + const dsDir = "l-once" + dstoreL1, dservL1 := makeStoreLevelDB(dsDir) + defer os.RemoveAll(dsDir) + benchAutoSync(b, b.N, false, dstoreL1, dservL1) + dstoreL1.Close() +} + +func BenchmarkSyncEveryLevelDB(b *testing.B) { + const dsDir = "l-every" + dstoreL2, dservL2 := makeStoreLevelDB(dsDir) + defer os.RemoveAll(dsDir) + benchAutoSync(b, b.N, true, dstoreL2, dservL2) + dstoreL2.Close() +} diff --git a/pinning/pinner/pin.go b/pinning/pinner/pin.go new file mode 100644 index 0000000000..fcf7d764a7 --- /dev/null +++ b/pinning/pinner/pin.go @@ -0,0 +1,158 @@ +// Package pin implements structures and methods to keep track of +// which objects a user wants to keep stored locally. +package pin + +import ( + "context" + "fmt" + + cid "github.com/ipfs/go-cid" + ipld "github.com/ipfs/go-ipld-format" +) + +const ( + linkRecursive = "recursive" + linkDirect = "direct" + linkIndirect = "indirect" + linkInternal = "internal" + linkNotPinned = "not pinned" + linkAny = "any" + linkAll = "all" +) + +// Mode allows to specify different types of pin (recursive, direct etc.). +// See the Pin Modes constants for a full list. +type Mode int + +// Pin Modes +const ( + // Recursive pins pin the target cids along with any reachable children. + Recursive Mode = iota + + // Direct pins pin just the target cid. + Direct + + // Indirect pins are cids who have some ancestor pinned recursively. + Indirect + + // Internal pins are cids used to keep the internal state of the pinner. + Internal + + // NotPinned + NotPinned + + // Any refers to any pinned cid + Any +) + +// ModeToString returns a human-readable name for the Mode. +func ModeToString(mode Mode) (string, bool) { + m := map[Mode]string{ + Recursive: linkRecursive, + Direct: linkDirect, + Indirect: linkIndirect, + Internal: linkInternal, + NotPinned: linkNotPinned, + Any: linkAny, + } + s, ok := m[mode] + return s, ok +} + +// StringToMode parses the result of ModeToString() back to a Mode. +// It returns a boolean which is set to false if the mode is unknown. +func StringToMode(s string) (Mode, bool) { + m := map[string]Mode{ + linkRecursive: Recursive, + linkDirect: Direct, + linkIndirect: Indirect, + linkInternal: Internal, + linkNotPinned: NotPinned, + linkAny: Any, + linkAll: Any, // "all" and "any" means the same thing + } + mode, ok := m[s] + return mode, ok +} + +// ErrNotPinned is returned when trying to unpin items that are not pinned. +var ErrNotPinned = fmt.Errorf("not pinned or pinned indirectly") + +// A Pinner provides the necessary methods to keep track of Nodes which are +// to be kept locally, according to a pin mode. In practice, a Pinner is in +// in charge of keeping the list of items from the local storage that should +// not be garbage-collected. +type Pinner interface { + // IsPinned returns whether or not the given cid is pinned + // and an explanation of why its pinned + IsPinned(ctx context.Context, c cid.Cid) (string, bool, error) + + // IsPinnedWithType returns whether or not the given cid is pinned with the + // given pin type, as well as returning the type of pin its pinned with. + IsPinnedWithType(ctx context.Context, c cid.Cid, mode Mode) (string, bool, error) + + // Pin the given node, optionally recursively. + // Pin will make sure that the given node and its children if recursive is set + // are stored locally. + Pin(ctx context.Context, node ipld.Node, recursive bool) error + + // Unpin the given cid. If recursive is true, removes either a recursive or + // a direct pin. If recursive is false, only removes a direct pin. + // If the pin doesn't exist, return ErrNotPinned + Unpin(ctx context.Context, cid cid.Cid, recursive bool) error + + // Update updates a recursive pin from one cid to another + // this is more efficient than simply pinning the new one and unpinning the + // old one + Update(ctx context.Context, from, to cid.Cid, unpin bool) error + + // Check if a set of keys are pinned, more efficient than + // calling IsPinned for each key + CheckIfPinned(ctx context.Context, cids ...cid.Cid) ([]Pinned, error) + + // PinWithMode is for manually editing the pin structure. Use with + // care! If used improperly, garbage collection may not be + // successful. + PinWithMode(context.Context, cid.Cid, Mode) error + + // Flush writes the pin state to the backing datastore + Flush(ctx context.Context) error + + // DirectKeys returns all directly pinned cids + DirectKeys(ctx context.Context) ([]cid.Cid, error) + + // RecursiveKeys returns all recursively pinned cids + RecursiveKeys(ctx context.Context) ([]cid.Cid, error) + + // InternalPins returns all cids kept pinned for the internal state of the + // pinner + InternalPins(ctx context.Context) ([]cid.Cid, error) +} + +// Pinned represents CID which has been pinned with a pinning strategy. +// The Via field allows to identify the pinning parent of this CID, in the +// case that the item is not pinned directly (but rather pinned recursively +// by some ascendant). +type Pinned struct { + Key cid.Cid + Mode Mode + Via cid.Cid +} + +// Pinned returns whether or not the given cid is pinned +func (p Pinned) Pinned() bool { + return p.Mode != NotPinned +} + +// String Returns pin status as string +func (p Pinned) String() string { + switch p.Mode { + case NotPinned: + return "not pinned" + case Indirect: + return fmt.Sprintf("pinned via %s", p.Via) + default: + modeStr, _ := ModeToString(p.Mode) + return fmt.Sprintf("pinned: %s", modeStr) + } +} diff --git a/pinning/remote/client/.gitignore b/pinning/remote/client/.gitignore new file mode 100644 index 0000000000..66fd13c903 --- /dev/null +++ b/pinning/remote/client/.gitignore @@ -0,0 +1,15 @@ +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib + +# Test binary, built with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Dependency directories (remove the comment below to include it) +# vendor/ diff --git a/pinning/remote/client/README.md b/pinning/remote/client/README.md new file mode 100644 index 0000000000..717d9538b8 --- /dev/null +++ b/pinning/remote/client/README.md @@ -0,0 +1,16 @@ +## Updating Pinning Service Spec + +Download the openapi-generator from https://github.com/OpenAPITools/openapi-generator and generate the code using: + +Current code generated with: openapi-generator 5.0.0-beta + +``` +openapi-generator generate -g go-experimental -i https://raw.githubusercontent.com/ipfs/pinning-services-api-spec/master/ipfs-pinning-service.yaml -o openapi +rm openapi/go.mod openapi/go.sum +``` + +Notes: +Due to https://github.com/OpenAPITools/openapi-generator/issues/7473 the code generator the http error codes processing +may need some manual editing. + +`go-experimental` is becoming mainstream and so in later versions will be replaced with `go` diff --git a/pinning/remote/client/client.go b/pinning/remote/client/client.go new file mode 100644 index 0000000000..c612c24ce3 --- /dev/null +++ b/pinning/remote/client/client.go @@ -0,0 +1,428 @@ +package go_pinning_service_http_client + +import ( + "context" + "fmt" + "net/http" + "time" + + "github.com/pkg/errors" + + "github.com/ipfs/go-cid" + "github.com/ipfs/go-libipfs/pinning/remote/client/openapi" + "github.com/multiformats/go-multiaddr" + "github.com/multiformats/go-multibase" + + logging "github.com/ipfs/go-log/v2" +) + +var logger = logging.Logger("pinning-service-http-client") + +const UserAgent = "go-pinning-service-http-client" + +type Client struct { + client *openapi.APIClient +} + +func NewClient(url, bearerToken string) *Client { + config := openapi.NewConfiguration() + config.UserAgent = UserAgent + config.AddDefaultHeader("Authorization", "Bearer "+bearerToken) + config.Servers = openapi.ServerConfigurations{ + openapi.ServerConfiguration{ + URL: url, + }, + } + + return &Client{client: openapi.NewAPIClient(config)} +} + +// TODO: We should probably make sure there are no duplicates sent +type lsSettings struct { + cids []string + name string + status []Status + before *time.Time + after *time.Time + limit *int32 + meta map[string]string +} + +type LsOption func(options *lsSettings) error + +var PinOpts = pinOpts{} + +type pinOpts struct { + pinLsOpts + pinAddOpts +} + +type pinLsOpts struct{} + +func (pinLsOpts) FilterCIDs(cids ...cid.Cid) LsOption { + return func(options *lsSettings) error { + enc := getCIDEncoder() + for _, c := range cids { + options.cids = append(options.cids, c.Encode(enc)) + } + return nil + } +} + +const maxNameSize = 255 + +func (pinLsOpts) FilterName(name string) LsOption { + return func(options *lsSettings) error { + if len(name) > maxNameSize { + return fmt.Errorf("name cannot be longer than %d", maxNameSize) + } + options.name = name + return nil + } +} + +func (pinLsOpts) FilterStatus(statuses ...Status) LsOption { + return func(options *lsSettings) error { + for _, s := range statuses { + valid := false + for _, existing := range validStatuses { + if existing == s { + valid = true + break + } + } + if !valid { + return fmt.Errorf("invalid status %s", s) + } + } + options.status = append(options.status, statuses...) + return nil + } +} + +func (pinLsOpts) FilterBefore(t time.Time) LsOption { + return func(options *lsSettings) error { + options.before = &t + return nil + } +} + +func (pinLsOpts) FilterAfter(t time.Time) LsOption { + return func(options *lsSettings) error { + options.after = &t + return nil + } +} + +const recordLimit = 1000 +const defaultLimit = 10 + +func (pinLsOpts) Limit(limit int) LsOption { + return func(options *lsSettings) error { + if limit > recordLimit { + return fmt.Errorf("limit exceeded maximum record limit of %d", recordLimit) + } + limitCasted := int32(limit) + options.limit = &limitCasted + return nil + } +} + +func (pinLsOpts) LsMeta(meta map[string]string) LsOption { + return func(options *lsSettings) error { + options.meta = meta + return nil + } +} + +type pinResults = openapi.PinResults + +func (c *Client) Ls(ctx context.Context, opts ...LsOption) (chan PinStatusGetter, chan error) { + res := make(chan PinStatusGetter, 1) + errs := make(chan error, 1) + + settings := new(lsSettings) + for _, o := range opts { + if err := o(settings); err != nil { + close(res) + errs <- err + close(errs) + return res, errs + } + } + + go func() { + defer func() { + if r := recover(); r != nil { + var err error + switch x := r.(type) { + case string: + err = fmt.Errorf("unexpected error while listing remote pins: %s", x) + case error: + err = fmt.Errorf("unexpected error while listing remote pins: %w", x) + default: + err = errors.New("unknown panic while listing remote pins") + } + errs <- err + } + close(errs) + close(res) + }() + + for { + pinRes, err := c.lsInternal(ctx, settings) + if err != nil { + errs <- err + return + } + + results := pinRes.GetResults() + for _, r := range results { + select { + case res <- &pinStatusObject{r}: + case <-ctx.Done(): + errs <- ctx.Err() + return + } + } + + batchSize := len(results) + if int(pinRes.Count) == batchSize { + // no more batches + return + } + + // Better DX/UX for cases like https://github.com/application-research/estuary/issues/124 + if batchSize == 0 && int(pinRes.Count) != 0 { + errs <- fmt.Errorf("invalid pinning service response: PinResults.count=%d but no PinResults.results", int(pinRes.Count)) + return + } + + oldestResult := results[batchSize-1] + settings.before = &oldestResult.Created + } + }() + + return res, errs +} + +func (c *Client) LsSync(ctx context.Context, opts ...LsOption) ([]PinStatusGetter, error) { + resCh, errCh := c.Ls(ctx, opts...) + + var res []PinStatusGetter + for r := range resCh { + res = append(res, r) + } + + return res, <-errCh +} + +// Manual version of Ls that returns a single batch of results and int with total count +func (c *Client) LsBatchSync(ctx context.Context, opts ...LsOption) ([]PinStatusGetter, int, error) { + var res []PinStatusGetter + + settings := new(lsSettings) + for _, o := range opts { + if err := o(settings); err != nil { + return nil, 0, err + } + } + + pinRes, err := c.lsInternal(ctx, settings) + if err != nil { + return nil, 0, err + } + + results := pinRes.GetResults() + for _, r := range results { + res = append(res, &pinStatusObject{r}) + } + + return res, int(pinRes.Count), nil +} + +func (c *Client) lsInternal(ctx context.Context, settings *lsSettings) (pinResults, error) { + getter := c.client.PinsApi.PinsGet(ctx) + if len(settings.cids) > 0 { + getter = getter.Cid(settings.cids) + } + if len(settings.status) > 0 { + statuses := make([]openapi.Status, len(settings.status)) + for i := 0; i < len(statuses); i++ { + statuses[i] = openapi.Status(settings.status[i]) + } + getter = getter.Status(statuses) + } + if settings.limit == nil { + getter = getter.Limit(defaultLimit) + } else { + getter = getter.Limit(*settings.limit) + } + if len(settings.name) > 0 { + getter = getter.Name(settings.name) + } + if settings.before != nil { + getter = getter.Before(*settings.before) + } + if settings.after != nil { + getter = getter.After(*settings.after) + } + if settings.meta != nil { + getter = getter.Meta(settings.meta) + } + + // TODO: Ignoring HTTP Response OK? + results, httpresp, err := getter.Execute() + if err != nil { + err := httperr(httpresp, err) + return pinResults{}, err + } + + return results, nil +} + +// TODO: We should probably make sure there are no duplicates sent +type addSettings struct { + name string + origins []string + meta map[string]string +} + +type AddOption func(options *addSettings) error + +type pinAddOpts struct{} + +func (pinAddOpts) WithName(name string) AddOption { + return func(options *addSettings) error { + if len(name) > maxNameSize { + return fmt.Errorf("name cannot be longer than %d", maxNameSize) + } + options.name = name + return nil + } +} + +func (pinAddOpts) WithOrigins(origins ...multiaddr.Multiaddr) AddOption { + return func(options *addSettings) error { + for _, o := range origins { + options.origins = append(options.origins, o.String()) + } + return nil + } +} + +func (pinAddOpts) AddMeta(meta map[string]string) AddOption { + return func(options *addSettings) error { + options.meta = meta + return nil + } +} + +func (c *Client) Add(ctx context.Context, cid cid.Cid, opts ...AddOption) (PinStatusGetter, error) { + settings := new(addSettings) + for _, o := range opts { + if err := o(settings); err != nil { + return nil, err + } + } + + adder := c.client.PinsApi.PinsPost(ctx) + p := openapi.Pin{ + Cid: cid.Encode(getCIDEncoder()), + } + + if len(settings.origins) > 0 { + p.SetOrigins(settings.origins) + } + if settings.meta != nil { + p.SetMeta(settings.meta) + } + if len(settings.name) > 0 { + p.SetName(settings.name) + } + + result, httpresp, err := adder.Pin(p).Execute() + if err != nil { + err := httperr(httpresp, err) + return nil, err + } + + return &pinStatusObject{result}, nil +} + +func (c *Client) GetStatusByID(ctx context.Context, pinID string) (PinStatusGetter, error) { + getter := c.client.PinsApi.PinsRequestidGet(ctx, pinID) + result, httpresp, err := getter.Execute() + if err != nil { + err := httperr(httpresp, err) + return nil, err + } + + return &pinStatusObject{result}, nil +} + +func (c *Client) DeleteByID(ctx context.Context, pinID string) error { + deleter := c.client.PinsApi.PinsRequestidDelete(ctx, pinID) + httpresp, err := deleter.Execute() + if err != nil { + err := httperr(httpresp, err) + return err + } + return nil +} + +func (c *Client) Replace(ctx context.Context, pinID string, cid cid.Cid, opts ...AddOption) (PinStatusGetter, error) { + settings := new(addSettings) + for _, o := range opts { + if err := o(settings); err != nil { + return nil, err + } + } + + adder := c.client.PinsApi.PinsRequestidPost(ctx, pinID) + p := openapi.Pin{ + Cid: cid.Encode(getCIDEncoder()), + } + + if len(settings.origins) > 0 { + p.SetOrigins(settings.origins) + } + if settings.meta != nil { + p.SetMeta(settings.meta) + } + if len(settings.name) > 0 { + p.SetName(settings.name) + } + + result, httpresp, err := adder.Pin(p).Execute() + if err != nil { + err := httperr(httpresp, err) + return nil, err + } + + return &pinStatusObject{result}, nil +} + +func getCIDEncoder() multibase.Encoder { + enc, err := multibase.NewEncoder(multibase.Base32) + if err != nil { + panic(err) + } + return enc +} + +func httperr(resp *http.Response, e error) error { + oerr, ok := e.(openapi.GenericOpenAPIError) + if ok { + ferr, ok := oerr.Model().(openapi.Failure) + if ok { + return errors.Wrapf(e, "reason: %q, details: %q", ferr.Error.GetReason(), ferr.Error.GetDetails()) + } + } + + if resp == nil { + return errors.Wrapf(e, "empty response from remote pinning service") + } + + return errors.Wrapf(e, "remote pinning service returned http error %d", resp.StatusCode) +} diff --git a/pinning/remote/client/cmd/main.go b/pinning/remote/client/cmd/main.go new file mode 100644 index 0000000000..64c545c6f9 --- /dev/null +++ b/pinning/remote/client/cmd/main.go @@ -0,0 +1,92 @@ +package main + +import ( + "context" + "fmt" + "os" + "time" + + "github.com/ipfs/go-cid" + pinclient "github.com/ipfs/go-libipfs/pinning/remote/client" +) + +func main() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + url, ok := os.LookupEnv("PS_URL") + if !ok { + panic("No Pinning Service URL found") + } + + key, ok := os.LookupEnv("PS_KEY") + if !ok { + panic("No Pinning Service API Key found") + } + + c := pinclient.NewClient(url, key) + + ipfsPgCid, err := cid.Parse("bafybeiayvrj27f65vbecspbnuavehcb3znvnt2strop2rfbczupudoizya") + if err != nil { + panic(err) + } + + libp2pCid, err := cid.Parse("bafybeiejgrxo4p4uofgfzvlg5twrg5w7tfwpf7aciiswfacfbdpevg2xfy") + if err != nil { + panic(err) + } + _ = ipfsPgCid + + listPins(ctx, c) + + fmt.Println("Adding libp2p home page") + ps, err := c.Add(ctx, libp2pCid, pinclient.PinOpts.WithName("libp2p")) + if err == nil { + fmt.Printf("PinStatus: %v \n", ps) + } else { + fmt.Println(err) + } + + listPins(ctx, c) + + fmt.Println("Check on pin status") + if ps == nil { + panic("Skipping pin status check because the pin is null") + } + + var pinned bool + for !pinned { + status, err := c.GetStatusByID(ctx, ps.GetRequestId()) + if err == nil { + fmt.Println(status.GetStatus()) + pinned = status.GetStatus() == pinclient.StatusPinned + } else { + fmt.Println(err) + } + time.Sleep(time.Millisecond * 500) + } + + listPins(ctx, c) + + fmt.Println("Delete pin") + err = c.DeleteByID(ctx, ps.GetRequestId()) + if err == nil { + fmt.Println("Successfully deleted pin") + } else { + fmt.Println(err) + } + + listPins(ctx, c) +} + +func listPins(ctx context.Context, c *pinclient.Client) { + fmt.Println("List all pins") + pins, err := c.LsSync(ctx) + if err != nil { + fmt.Println(err) + } else { + for _, p := range pins { + fmt.Printf("Pin: %v \n", p) + } + } +} diff --git a/pinning/remote/client/model.go b/pinning/remote/client/model.go new file mode 100644 index 0000000000..8f2c685e26 --- /dev/null +++ b/pinning/remote/client/model.go @@ -0,0 +1,167 @@ +package go_pinning_service_http_client + +import ( + "encoding/json" + "fmt" + "time" + + "github.com/ipfs/go-cid" + "github.com/ipfs/go-libipfs/pinning/remote/client/openapi" + "github.com/multiformats/go-multiaddr" +) + +// PinGetter Getter for Pin object +type PinGetter interface { + fmt.Stringer + json.Marshaler + // CID to be pinned recursively + GetCid() cid.Cid + // Optional name for pinned data; can be used for lookups later + GetName() string + // Optional list of multiaddrs known to provide the data + GetOrigins() []string + // Optional metadata for pin object + GetMeta() map[string]string +} + +type pinObject struct { + openapi.Pin +} + +func (p *pinObject) MarshalJSON() ([]byte, error) { + var originsStr string + if o := p.GetOrigins(); o != nil { + originsBytes, err := json.Marshal(o) + if err == nil { + originsStr = string(originsBytes) + } + } + + metaStr := "{}" + if meta := p.GetMeta(); meta != nil { + metaBytes, err := json.Marshal(meta) + if err == nil { + metaStr = string(metaBytes) + } + } + + str := fmt.Sprintf("{ \"Cid\" : \"%v\", \"Name\" : \"%s\", \"Origins\" : %v, \"Meta\" : %v }", + p.GetCid(), p.GetName(), originsStr, metaStr) + return []byte(str), nil +} + +func (p *pinObject) String() string { + marshalled, err := json.MarshalIndent(p, "", "\t") + if err != nil { + return "" + } + + return string(marshalled) +} + +func (p *pinObject) GetCid() cid.Cid { + c, err := cid.Parse(p.Pin.Cid) + if err != nil { + return cid.Undef + } + return c +} + +type Status string + +const ( + StatusUnknown Status = "" + StatusQueued Status = Status(openapi.QUEUED) + StatusPinning Status = Status(openapi.PINNING) + StatusPinned Status = Status(openapi.PINNED) + StatusFailed Status = Status(openapi.FAILED) +) + +func (s Status) String() string { + switch s { + case StatusQueued, StatusPinning, StatusPinned, StatusFailed: + return string(s) + default: + return string(StatusUnknown) + } +} + +var validStatuses = []Status{"queued", "pinning", "pinned", "failed"} + +// PinStatusGetter Getter for Pin object with status +type PinStatusGetter interface { + fmt.Stringer + json.Marshaler + // Globally unique ID of the pin request; can be used to check the status of ongoing pinning, modification of pin object, or pin removal + GetRequestId() string + GetStatus() Status + // Immutable timestamp indicating when a pin request entered a pinning service; can be used for filtering results and pagination + GetCreated() time.Time + GetPin() PinGetter + // List of multiaddrs designated by pinning service for transferring any new data from external peers + GetDelegates() []multiaddr.Multiaddr + // Optional info for PinStatus response + GetInfo() map[string]string +} + +type pinStatusObject struct { + openapi.PinStatus +} + +func (p *pinStatusObject) GetDelegates() []multiaddr.Multiaddr { + delegates := p.PinStatus.GetDelegates() + addrs := make([]multiaddr.Multiaddr, 0, len(delegates)) + for _, d := range delegates { + a, err := multiaddr.NewMultiaddr(d) + if err != nil { + logger.Errorf("returned delegate is an invalid multiaddr: %w", err) + continue + } + addrs = append(addrs, a) + } + return addrs +} + +func (p *pinStatusObject) GetPin() PinGetter { + return &pinObject{p.Pin} +} + +func (p *pinStatusObject) GetStatus() Status { + return Status(p.PinStatus.GetStatus()) +} + +func (p *pinStatusObject) GetRequestId() string { + return p.GetRequestid() +} + +func (p *pinStatusObject) MarshalJSON() ([]byte, error) { + var delegatesStr string + if d := p.GetDelegates(); d != nil { + delegatesBytes, err := json.Marshal(d) + if err == nil { + delegatesStr = string(delegatesBytes) + } + } + + infoStr := "{}" + if info := p.GetInfo(); info != nil { + infoBytes, err := json.Marshal(info) + if err == nil { + infoStr = string(infoBytes) + } + } + + str := fmt.Sprintf("{\"Pin\" : %v, \"RequestID\" : \"%s\", \"Status\" : \"%s\", \"Created\" : \"%v\", \"Delegates\" : %v, \"Info\" : %v }", + p.GetPin(), p.GetRequestId(), p.GetStatus(), p.GetCreated(), delegatesStr, infoStr) + + return []byte(str), nil +} + +func (p *pinStatusObject) String() string { + marshalled, err := json.MarshalIndent(p, "", "\t") + if err != nil { + return "" + } + + return string(marshalled) +} diff --git a/pinning/remote/client/openapi/README.md b/pinning/remote/client/openapi/README.md new file mode 100644 index 0000000000..fe21b9a009 --- /dev/null +++ b/pinning/remote/client/openapi/README.md @@ -0,0 +1,217 @@ +# Go API client for openapi + + + +## About this spec +The IPFS Pinning Service API is intended to be an implementation-agnostic API: +- For use and implementation by pinning service providers +- For use in client mode by IPFS nodes and GUI-based applications + +> **Note**: while ready for implementation, this spec is still a work in progress! 🏗️ **Your input and feedback are welcome and valuable as we develop this API spec. Please join the design discussion at [github.com/ipfs/pinning-services-api-spec](https://github.com/ipfs/pinning-services-api-spec).** + +# Schemas +This section describes the most important object types and conventions. + +A full list of fields and schemas can be found in the `schemas` section of the [YAML file](https://github.com/ipfs/pinning-services-api-spec/blob/master/ipfs-pinning-service.yaml). + +## Identifiers +### cid +[Content Identifier (CID)](https://docs.ipfs.io/concepts/content-addressing/) points at the root of a DAG that is pinned recursively. +### requestid +Unique identifier of a pin request. + +When a pin is created, the service responds with unique `requestid` that can be later used for pin removal. When the same `cid` is pinned again, a different `requestid` is returned to differentiate between those pin requests. + +Service implementation should use UUID, `hash(accessToken,Pin,PinStatus.created)`, or any other opaque identifier that provides equally strong protection against race conditions. + +## Objects +### Pin object + +![pin object](https://bafybeideck2fchyxna4wqwc2mo67yriokehw3yujboc5redjdaajrk2fjq.ipfs.dweb.link/pin.png) + +The `Pin` object is a representation of a pin request. + +It includes the `cid` of data to be pinned, as well as optional metadata in `name`, `origins`, and `meta`. + +### Pin status response + +![pin status response object](https://bafybeideck2fchyxna4wqwc2mo67yriokehw3yujboc5redjdaajrk2fjq.ipfs.dweb.link/pinstatus.png) + +The `PinStatus` object is a representation of the current state of a pinning operation. +It includes the original `pin` object, along with the current `status` and globally unique `requestid` of the entire pinning request, which can be used for future status checks and management. Addresses in the `delegates` array are peers delegated by the pinning service for facilitating direct file transfers (more details in the provider hints section). Any additional vendor-specific information is returned in optional `info`. + +## The pin lifecycle + +![pinning service objects and lifecycle](https://bafybeideck2fchyxna4wqwc2mo67yriokehw3yujboc5redjdaajrk2fjq.ipfs.dweb.link/lifecycle.png) + +### Creating a new pin object +The user sends a `Pin` object to `POST /pins` and receives a `PinStatus` response: +- `requestid` in `PinStatus` is the identifier of the pin operation, which can can be used for checking status, and removing the pin in the future +- `status` in `PinStatus` indicates the current state of a pin + +### Checking status of in-progress pinning +`status` (in `PinStatus`) may indicate a pending state (`queued` or `pinning`). This means the data behind `Pin.cid` was not found on the pinning service and is being fetched from the IPFS network at large, which may take time. + +In this case, the user can periodically check pinning progress via `GET /pins/{requestid}` until pinning is successful, or the user decides to remove the pending pin. + +### Replacing an existing pin object +The user can replace an existing pin object via `POST /pins/{requestid}`. This is a shortcut for removing a pin object identified by `requestid` and creating a new one in a single API call that protects against undesired garbage collection of blocks common to both pins. Useful when updating a pin representing a huge dataset where most of blocks did not change. The new pin object `requestid` is returned in the `PinStatus` response. The old pin object is deleted automatically. + +### Removing a pin object +A pin object can be removed via `DELETE /pins/{requestid}`. + + +## Provider hints +Pinning of new data can be accelerated by providing a list of known data sources in `Pin.origins`, and connecting at least one of them to pinning service nodes at `PinStatus.delegates`. + +The most common scenario is a client putting its own IPFS node's multiaddrs in `Pin.origins`, and then directly connecting to every multiaddr returned by a pinning service in `PinStatus.delegates` to initiate transfer. + +This ensures data transfer starts immediately (without waiting for provider discovery over DHT), and direct dial from a client works around peer routing issues in restrictive network topologies such as NATs. + +## Custom metadata +Pinning services are encouraged to add support for additional features by leveraging the optional `Pin.meta` and `PinStatus.info` fields. While these attributes can be application- or vendor-specific, we encourage the community at large to leverage these attributes as a sandbox to come up with conventions that could become part of future revisions of this API. +### Pin metadata +String keys and values passed in `Pin.meta` are persisted with the pin object. + +Potential uses: +- `Pin.meta[app_id]`: Attaching a unique identifier to pins created by an app enables filtering pins per app via `?meta={\"app_id\":}` +- `Pin.meta[vendor_policy]`: Vendor-specific policy (for example: which region to use, how many copies to keep) + +Note that it is OK for a client to omit or ignore these optional attributes; doing so should not impact the basic pinning functionality. + +### Pin status info +Additional `PinStatus.info` can be returned by pinning service. + +Potential uses: +- `PinStatus.info[status_details]`: more info about the current status (queue position, percentage of transferred data, summary of where data is stored, etc); when `PinStatus.status=failed`, it could provide a reason why a pin operation failed (e.g. lack of funds, DAG too big, etc.) +- `PinStatus.info[dag_size]`: the size of pinned data, along with DAG overhead +- `PinStatus.info[raw_size]`: the size of data without DAG overhead (eg. unixfs) +- `PinStatus.info[pinned_until]`: if vendor supports time-bound pins, this could indicate when the pin will expire + +# Pagination and filtering +Pin objects can be listed by executing `GET /pins` with optional parameters: + +- When no filters are provided, the endpoint will return a small batch of the 10 most recently created items, from the latest to the oldest. +- The number of returned items can be adjusted with the `limit` parameter (implicit default is 10). +- If the value in `PinResults.count` is bigger than the length of `PinResults.results`, the client can infer there are more results that can be queried. +- To read more items, pass the `before` filter with the timestamp from `PinStatus.created` found in the oldest item in the current batch of results. Repeat to read all results. +- Returned results can be fine-tuned by applying optional `after`, `cid`, `name`, `status`, or `meta` filters. + +> **Note**: pagination by the `created` timestamp requires each value to be globally unique. Any future considerations to add support for bulk creation must account for this. + + + +## Overview +This API client was generated by the [OpenAPI Generator](https://openapi-generator.tech) project. By using the [OpenAPI-spec](https://www.openapis.org/) from a remote server, you can easily generate an API client. + +- API version: 0.1.1 +- Package version: 1.0.0 +- Build package: org.openapitools.codegen.languages.GoClientExperimentalCodegen + +## Installation + +Install the following dependencies: + +```shell +go get github.com/stretchr/testify/assert +go get golang.org/x/oauth2 +go get golang.org/x/net/context +``` + +Put the package under your project folder and add the following in import: + +```golang +import sw "./openapi" +``` + +## Configuration of Server URL + +Default configuration comes with `Servers` field that contains server objects as defined in the OpenAPI specification. + +### Select Server Configuration + +For using other server than the one defined on index 0 set context value `sw.ContextServerIndex` of type `int`. + +```golang +ctx := context.WithValue(context.Background(), sw.ContextServerIndex, 1) +``` + +### Templated Server URL + +Templated server URL is formatted using default variables from configuration or from context value `sw.ContextServerVariables` of type `map[string]string`. + +```golang +ctx := context.WithValue(context.Background(), sw.ContextServerVariables, map[string]string{ + "basePath": "v2", +}) +``` + +Note, enum values are always validated and all unused variables are silently ignored. + +### URLs Configuration per Operation + +Each operation can use different server URL defined using `OperationServers` map in the `Configuration`. +An operation is uniquely identifield by `"{classname}Service.{nickname}"` string. +Similar rules for overriding default operation server index and variables applies by using `sw.ContextOperationServerIndices` and `sw.ContextOperationServerVariables` context maps. + +``` +ctx := context.WithValue(context.Background(), sw.ContextOperationServerIndices, map[string]int{ + "{classname}Service.{nickname}": 2, +}) +ctx = context.WithValue(context.Background(), sw.ContextOperationServerVariables, map[string]map[string]string{ + "{classname}Service.{nickname}": { + "port": "8443", + }, +}) +``` + +## Documentation for API Endpoints + +All URIs are relative to *https://pinning-service.example.com* + +Class | Method | HTTP request | Description +------------ | ------------- | ------------- | ------------- +*PinsApi* | [**PinsGet**](docs/PinsApi.md#pinsget) | **Get** /pins | List pin objects +*PinsApi* | [**PinsPost**](docs/PinsApi.md#pinspost) | **Post** /pins | Add pin object +*PinsApi* | [**PinsRequestidDelete**](docs/PinsApi.md#pinsrequestiddelete) | **Delete** /pins/{requestid} | Remove pin object +*PinsApi* | [**PinsRequestidGet**](docs/PinsApi.md#pinsrequestidget) | **Get** /pins/{requestid} | Get pin object +*PinsApi* | [**PinsRequestidPost**](docs/PinsApi.md#pinsrequestidpost) | **Post** /pins/{requestid} | Replace pin object + + +## Documentation For Models + + - [Failure](docs/Failure.md) + - [FailureError](docs/FailureError.md) + - [Pin](docs/Pin.md) + - [PinResults](docs/PinResults.md) + - [PinStatus](docs/PinStatus.md) + - [Status](docs/Status.md) + + +## Documentation For Authorization + + + +### accessToken + + +## Documentation for Utility Methods + +Due to the fact that model structure members are all pointers, this package contains +a number of utility functions to easily obtain pointers to values of basic types. +Each of these functions takes a value of the given basic type and returns a pointer to it: + +* `PtrBool` +* `PtrInt` +* `PtrInt32` +* `PtrInt64` +* `PtrFloat` +* `PtrFloat32` +* `PtrFloat64` +* `PtrString` +* `PtrTime` + +## Author + + + diff --git a/pinning/remote/client/openapi/api_pins.go b/pinning/remote/client/openapi/api_pins.go new file mode 100644 index 0000000000..b2858c5eea --- /dev/null +++ b/pinning/remote/client/openapi/api_pins.go @@ -0,0 +1,654 @@ +/* + * IPFS Pinning Service API + * + * ## About this spec The IPFS Pinning Service API is intended to be an implementation-agnostic API: - For use and implementation by pinning service providers - For use in client mode by IPFS nodes and GUI-based applications > **Note**: while ready for implementation, this spec is still a work in progress! 🏗️ **Your input and feedback are welcome and valuable as we develop this API spec. Please join the design discussion at [github.com/ipfs/pinning-services-api-spec](https://github.com/ipfs/pinning-services-api-spec).** # Schemas This section describes the most important object types and conventions. A full list of fields and schemas can be found in the `schemas` section of the [YAML file](https://github.com/ipfs/pinning-services-api-spec/blob/master/ipfs-pinning-service.yaml). ## Identifiers ### cid [Content Identifier (CID)](https://docs.ipfs.io/concepts/content-addressing/) points at the root of a DAG that is pinned recursively. ### requestid Unique identifier of a pin request. When a pin is created, the service responds with unique `requestid` that can be later used for pin removal. When the same `cid` is pinned again, a different `requestid` is returned to differentiate between those pin requests. Service implementation should use UUID, `hash(accessToken,Pin,PinStatus.created)`, or any other opaque identifier that provides equally strong protection against race conditions. ## Objects ### Pin object ![pin object](https://bafybeideck2fchyxna4wqwc2mo67yriokehw3yujboc5redjdaajrk2fjq.ipfs.dweb.link/pin.png) The `Pin` object is a representation of a pin request. It includes the `cid` of data to be pinned, as well as optional metadata in `name`, `origins`, and `meta`. ### Pin status response ![pin status response object](https://bafybeideck2fchyxna4wqwc2mo67yriokehw3yujboc5redjdaajrk2fjq.ipfs.dweb.link/pinstatus.png) The `PinStatus` object is a representation of the current state of a pinning operation. It includes the original `pin` object, along with the current `status` and globally unique `requestid` of the entire pinning request, which can be used for future status checks and management. Addresses in the `delegates` array are peers delegated by the pinning service for facilitating direct file transfers (more details in the provider hints section). Any additional vendor-specific information is returned in optional `info`. ## The pin lifecycle ![pinning service objects and lifecycle](https://bafybeideck2fchyxna4wqwc2mo67yriokehw3yujboc5redjdaajrk2fjq.ipfs.dweb.link/lifecycle.png) ### Creating a new pin object The user sends a `Pin` object to `POST /pins` and receives a `PinStatus` response: - `requestid` in `PinStatus` is the identifier of the pin operation, which can can be used for checking status, and removing the pin in the future - `status` in `PinStatus` indicates the current state of a pin ### Checking status of in-progress pinning `status` (in `PinStatus`) may indicate a pending state (`queued` or `pinning`). This means the data behind `Pin.cid` was not found on the pinning service and is being fetched from the IPFS network at large, which may take time. In this case, the user can periodically check pinning progress via `GET /pins/{requestid}` until pinning is successful, or the user decides to remove the pending pin. ### Replacing an existing pin object The user can replace an existing pin object via `POST /pins/{requestid}`. This is a shortcut for removing a pin object identified by `requestid` and creating a new one in a single API call that protects against undesired garbage collection of blocks common to both pins. Useful when updating a pin representing a huge dataset where most of blocks did not change. The new pin object `requestid` is returned in the `PinStatus` response. The old pin object is deleted automatically. ### Removing a pin object A pin object can be removed via `DELETE /pins/{requestid}`. ## Provider hints Pinning of new data can be accelerated by providing a list of known data sources in `Pin.origins`, and connecting at least one of them to pinning service nodes at `PinStatus.delegates`. The most common scenario is a client putting its own IPFS node's multiaddrs in `Pin.origins`, and then directly connecting to every multiaddr returned by a pinning service in `PinStatus.delegates` to initiate transfer. This ensures data transfer starts immediately (without waiting for provider discovery over DHT), and direct dial from a client works around peer routing issues in restrictive network topologies such as NATs. ## Custom metadata Pinning services are encouraged to add support for additional features by leveraging the optional `Pin.meta` and `PinStatus.info` fields. While these attributes can be application- or vendor-specific, we encourage the community at large to leverage these attributes as a sandbox to come up with conventions that could become part of future revisions of this API. ### Pin metadata String keys and values passed in `Pin.meta` are persisted with the pin object. Potential uses: - `Pin.meta[app_id]`: Attaching a unique identifier to pins created by an app enables filtering pins per app via `?meta={\"app_id\":}` - `Pin.meta[vendor_policy]`: Vendor-specific policy (for example: which region to use, how many copies to keep) Note that it is OK for a client to omit or ignore these optional attributes; doing so should not impact the basic pinning functionality. ### Pin status info Additional `PinStatus.info` can be returned by pinning service. Potential uses: - `PinStatus.info[status_details]`: more info about the current status (queue position, percentage of transferred data, summary of where data is stored, etc); when `PinStatus.status=failed`, it could provide a reason why a pin operation failed (e.g. lack of funds, DAG too big, etc.) - `PinStatus.info[dag_size]`: the size of pinned data, along with DAG overhead - `PinStatus.info[raw_size]`: the size of data without DAG overhead (eg. unixfs) - `PinStatus.info[pinned_until]`: if vendor supports time-bound pins, this could indicate when the pin will expire # Pagination and filtering Pin objects can be listed by executing `GET /pins` with optional parameters: - When no filters are provided, the endpoint will return a small batch of the 10 most recently created items, from the latest to the oldest. - The number of returned items can be adjusted with the `limit` parameter (implicit default is 10). - If the value in `PinResults.count` is bigger than the length of `PinResults.results`, the client can infer there are more results that can be queried. - To read more items, pass the `before` filter with the timestamp from `PinStatus.created` found in the oldest item in the current batch of results. Repeat to read all results. - Returned results can be fine-tuned by applying optional `after`, `cid`, `name`, `status`, or `meta` filters. > **Note**: pagination by the `created` timestamp requires each value to be globally unique. Any future considerations to add support for bulk creation must account for this. + * + * API version: 0.1.1 + * Generated by: OpenAPI Generator (https://openapi-generator.tech) + */ + +package openapi + +import ( + _context "context" + _io "io" + _nethttp "net/http" + _neturl "net/url" + "strings" + "time" +) + +// Linger please +var ( + _ _context.Context +) + +// PinsApiService PinsApi service +type PinsApiService service + +type apiPinsGetRequest struct { + ctx _context.Context + apiService *PinsApiService + cid *[]string + name *string + status *[]Status + before *time.Time + after *time.Time + limit *int32 + meta *map[string]string +} + +func (r apiPinsGetRequest) Cid(cid []string) apiPinsGetRequest { + r.cid = &cid + return r +} + +func (r apiPinsGetRequest) Name(name string) apiPinsGetRequest { + r.name = &name + return r +} + +func (r apiPinsGetRequest) Status(status []Status) apiPinsGetRequest { + r.status = &status + return r +} + +func (r apiPinsGetRequest) Before(before time.Time) apiPinsGetRequest { + r.before = &before + return r +} + +func (r apiPinsGetRequest) After(after time.Time) apiPinsGetRequest { + r.after = &after + return r +} + +func (r apiPinsGetRequest) Limit(limit int32) apiPinsGetRequest { + r.limit = &limit + return r +} + +func (r apiPinsGetRequest) Meta(meta map[string]string) apiPinsGetRequest { + r.meta = &meta + return r +} + +/* +PinsGet List pin objects +List all the pin objects, matching optional filters; when no filter is provided, only successful pins are returned + - @param ctx _context.Context - for authentication, logging, cancellation, deadlines, tracing, etc. Passed from http.Request or context.Background(). + +@return apiPinsGetRequest +*/ +func (a *PinsApiService) PinsGet(ctx _context.Context) apiPinsGetRequest { + return apiPinsGetRequest{ + apiService: a, + ctx: ctx, + } +} + +/* +Execute executes the request + + @return PinResults +*/ +func (r apiPinsGetRequest) Execute() (PinResults, *_nethttp.Response, error) { + var ( + localVarHTTPMethod = _nethttp.MethodGet + localVarPostBody interface{} + localVarFormFileName string + localVarFileName string + localVarFileBytes []byte + localVarReturnValue PinResults + ) + + localBasePath, err := r.apiService.client.cfg.ServerURLWithContext(r.ctx, "PinsApiService.PinsGet") + if err != nil { + return localVarReturnValue, nil, GenericOpenAPIError{error: err.Error()} + } + + localVarPath := localBasePath + "/pins" + + localVarHeaderParams := make(map[string]string) + localVarQueryParams := _neturl.Values{} + localVarFormParams := _neturl.Values{} + + if r.cid != nil { + localVarQueryParams.Add("cid", parameterToString(*r.cid, "csv")) + } + if r.name != nil { + localVarQueryParams.Add("name", parameterToString(*r.name, "")) + } + if r.status != nil { + localVarQueryParams.Add("status", parameterToString(*r.status, "csv")) + } + if r.before != nil { + localVarQueryParams.Add("before", parameterToString(*r.before, "")) + } + if r.after != nil { + localVarQueryParams.Add("after", parameterToString(*r.after, "")) + } + if r.limit != nil { + localVarQueryParams.Add("limit", parameterToString(*r.limit, "")) + } + if r.meta != nil { + localVarQueryParams.Add("meta", parameterToString(*r.meta, "")) + } + // to determine the Content-Type header + localVarHTTPContentTypes := []string{} + + // set Content-Type header + localVarHTTPContentType := selectHeaderContentType(localVarHTTPContentTypes) + if localVarHTTPContentType != "" { + localVarHeaderParams["Content-Type"] = localVarHTTPContentType + } + + // to determine the Accept header + localVarHTTPHeaderAccepts := []string{"application/json"} + + // set Accept header + localVarHTTPHeaderAccept := selectHeaderAccept(localVarHTTPHeaderAccepts) + if localVarHTTPHeaderAccept != "" { + localVarHeaderParams["Accept"] = localVarHTTPHeaderAccept + } + req, err := r.apiService.client.prepareRequest(r.ctx, localVarPath, localVarHTTPMethod, localVarPostBody, localVarHeaderParams, localVarQueryParams, localVarFormParams, localVarFormFileName, localVarFileName, localVarFileBytes) + if err != nil { + return localVarReturnValue, nil, err + } + + localVarHTTPResponse, err := r.apiService.client.callAPI(req) + if err != nil || localVarHTTPResponse == nil { + return localVarReturnValue, localVarHTTPResponse, err + } + + localVarBody, err := _io.ReadAll(localVarHTTPResponse.Body) + localVarHTTPResponse.Body.Close() + if err != nil { + return localVarReturnValue, localVarHTTPResponse, err + } + + if localVarHTTPResponse.StatusCode >= 300 { + newErr := GenericOpenAPIError{ + body: localVarBody, + error: localVarHTTPResponse.Status, + } + if localVarHTTPResponse.StatusCode >= 400 && localVarHTTPResponse.StatusCode <= 600 { + var v Failure + err = r.apiService.client.decode(&v, localVarBody, localVarHTTPResponse.Header.Get("Content-Type")) + if err != nil { + newErr.error = err.Error() + return localVarReturnValue, localVarHTTPResponse, newErr + } + newErr.model = v + return localVarReturnValue, localVarHTTPResponse, newErr + } + return localVarReturnValue, localVarHTTPResponse, newErr + } + + err = r.apiService.client.decode(&localVarReturnValue, localVarBody, localVarHTTPResponse.Header.Get("Content-Type")) + if err != nil { + newErr := GenericOpenAPIError{ + body: localVarBody, + error: err.Error(), + } + return localVarReturnValue, localVarHTTPResponse, newErr + } + + return localVarReturnValue, localVarHTTPResponse, nil +} + +type apiPinsPostRequest struct { + ctx _context.Context + apiService *PinsApiService + pin *Pin +} + +func (r apiPinsPostRequest) Pin(pin Pin) apiPinsPostRequest { + r.pin = &pin + return r +} + +/* +PinsPost Add pin object +Add a new pin object for the current access token + - @param ctx _context.Context - for authentication, logging, cancellation, deadlines, tracing, etc. Passed from http.Request or context.Background(). + +@return apiPinsPostRequest +*/ +func (a *PinsApiService) PinsPost(ctx _context.Context) apiPinsPostRequest { + return apiPinsPostRequest{ + apiService: a, + ctx: ctx, + } +} + +/* +Execute executes the request + + @return PinStatus +*/ +func (r apiPinsPostRequest) Execute() (PinStatus, *_nethttp.Response, error) { + var ( + localVarHTTPMethod = _nethttp.MethodPost + localVarPostBody interface{} + localVarFormFileName string + localVarFileName string + localVarFileBytes []byte + localVarReturnValue PinStatus + ) + + localBasePath, err := r.apiService.client.cfg.ServerURLWithContext(r.ctx, "PinsApiService.PinsPost") + if err != nil { + return localVarReturnValue, nil, GenericOpenAPIError{error: err.Error()} + } + + localVarPath := localBasePath + "/pins" + + localVarHeaderParams := make(map[string]string) + localVarQueryParams := _neturl.Values{} + localVarFormParams := _neturl.Values{} + + if r.pin == nil { + return localVarReturnValue, nil, reportError("pin is required and must be specified") + } + + // to determine the Content-Type header + localVarHTTPContentTypes := []string{"application/json"} + + // set Content-Type header + localVarHTTPContentType := selectHeaderContentType(localVarHTTPContentTypes) + if localVarHTTPContentType != "" { + localVarHeaderParams["Content-Type"] = localVarHTTPContentType + } + + // to determine the Accept header + localVarHTTPHeaderAccepts := []string{"application/json"} + + // set Accept header + localVarHTTPHeaderAccept := selectHeaderAccept(localVarHTTPHeaderAccepts) + if localVarHTTPHeaderAccept != "" { + localVarHeaderParams["Accept"] = localVarHTTPHeaderAccept + } + // body params + localVarPostBody = r.pin + req, err := r.apiService.client.prepareRequest(r.ctx, localVarPath, localVarHTTPMethod, localVarPostBody, localVarHeaderParams, localVarQueryParams, localVarFormParams, localVarFormFileName, localVarFileName, localVarFileBytes) + if err != nil { + return localVarReturnValue, nil, err + } + + localVarHTTPResponse, err := r.apiService.client.callAPI(req) + if err != nil || localVarHTTPResponse == nil { + return localVarReturnValue, localVarHTTPResponse, err + } + + localVarBody, err := _io.ReadAll(localVarHTTPResponse.Body) + localVarHTTPResponse.Body.Close() + if err != nil { + return localVarReturnValue, localVarHTTPResponse, err + } + + if localVarHTTPResponse.StatusCode >= 300 { + newErr := GenericOpenAPIError{ + body: localVarBody, + error: localVarHTTPResponse.Status, + } + if localVarHTTPResponse.StatusCode >= 400 && localVarHTTPResponse.StatusCode <= 600 { + var v Failure + err = r.apiService.client.decode(&v, localVarBody, localVarHTTPResponse.Header.Get("Content-Type")) + if err != nil { + newErr.error = err.Error() + return localVarReturnValue, localVarHTTPResponse, newErr + } + newErr.model = v + return localVarReturnValue, localVarHTTPResponse, newErr + } + return localVarReturnValue, localVarHTTPResponse, newErr + } + + err = r.apiService.client.decode(&localVarReturnValue, localVarBody, localVarHTTPResponse.Header.Get("Content-Type")) + if err != nil { + newErr := GenericOpenAPIError{ + body: localVarBody, + error: err.Error(), + } + return localVarReturnValue, localVarHTTPResponse, newErr + } + + return localVarReturnValue, localVarHTTPResponse, nil +} + +type apiPinsRequestidDeleteRequest struct { + ctx _context.Context + apiService *PinsApiService + requestid string +} + +/* +PinsRequestidDelete Remove pin object +Remove a pin object + - @param ctx _context.Context - for authentication, logging, cancellation, deadlines, tracing, etc. Passed from http.Request or context.Background(). + - @param requestid + +@return apiPinsRequestidDeleteRequest +*/ +func (a *PinsApiService) PinsRequestidDelete(ctx _context.Context, requestid string) apiPinsRequestidDeleteRequest { + return apiPinsRequestidDeleteRequest{ + apiService: a, + ctx: ctx, + requestid: requestid, + } +} + +/* +Execute executes the request +*/ +func (r apiPinsRequestidDeleteRequest) Execute() (*_nethttp.Response, error) { + var ( + localVarHTTPMethod = _nethttp.MethodDelete + localVarPostBody interface{} + localVarFormFileName string + localVarFileName string + localVarFileBytes []byte + ) + + localBasePath, err := r.apiService.client.cfg.ServerURLWithContext(r.ctx, "PinsApiService.PinsRequestidDelete") + if err != nil { + return nil, GenericOpenAPIError{error: err.Error()} + } + + localVarPath := localBasePath + "/pins/{requestid}" + localVarPath = strings.Replace(localVarPath, "{"+"requestid"+"}", _neturl.PathEscape(parameterToString(r.requestid, "")), -1) + + localVarHeaderParams := make(map[string]string) + localVarQueryParams := _neturl.Values{} + localVarFormParams := _neturl.Values{} + + // to determine the Content-Type header + localVarHTTPContentTypes := []string{} + + // set Content-Type header + localVarHTTPContentType := selectHeaderContentType(localVarHTTPContentTypes) + if localVarHTTPContentType != "" { + localVarHeaderParams["Content-Type"] = localVarHTTPContentType + } + + // to determine the Accept header + localVarHTTPHeaderAccepts := []string{"application/json"} + + // set Accept header + localVarHTTPHeaderAccept := selectHeaderAccept(localVarHTTPHeaderAccepts) + if localVarHTTPHeaderAccept != "" { + localVarHeaderParams["Accept"] = localVarHTTPHeaderAccept + } + req, err := r.apiService.client.prepareRequest(r.ctx, localVarPath, localVarHTTPMethod, localVarPostBody, localVarHeaderParams, localVarQueryParams, localVarFormParams, localVarFormFileName, localVarFileName, localVarFileBytes) + if err != nil { + return nil, err + } + + localVarHTTPResponse, err := r.apiService.client.callAPI(req) + if err != nil || localVarHTTPResponse == nil { + return localVarHTTPResponse, err + } + + localVarBody, err := _io.ReadAll(localVarHTTPResponse.Body) + localVarHTTPResponse.Body.Close() + if err != nil { + return localVarHTTPResponse, err + } + + if localVarHTTPResponse.StatusCode >= 300 { + newErr := GenericOpenAPIError{ + body: localVarBody, + error: localVarHTTPResponse.Status, + } + if localVarHTTPResponse.StatusCode >= 400 && localVarHTTPResponse.StatusCode <= 600 { + var v Failure + err = r.apiService.client.decode(&v, localVarBody, localVarHTTPResponse.Header.Get("Content-Type")) + if err != nil { + newErr.error = err.Error() + return localVarHTTPResponse, newErr + } + newErr.model = v + return localVarHTTPResponse, newErr + } + return localVarHTTPResponse, newErr + } + + return localVarHTTPResponse, nil +} + +type apiPinsRequestidGetRequest struct { + ctx _context.Context + apiService *PinsApiService + requestid string +} + +/* +PinsRequestidGet Get pin object +Get a pin object and its status + - @param ctx _context.Context - for authentication, logging, cancellation, deadlines, tracing, etc. Passed from http.Request or context.Background(). + - @param requestid + +@return apiPinsRequestidGetRequest +*/ +func (a *PinsApiService) PinsRequestidGet(ctx _context.Context, requestid string) apiPinsRequestidGetRequest { + return apiPinsRequestidGetRequest{ + apiService: a, + ctx: ctx, + requestid: requestid, + } +} + +/* +Execute executes the request + + @return PinStatus +*/ +func (r apiPinsRequestidGetRequest) Execute() (PinStatus, *_nethttp.Response, error) { + var ( + localVarHTTPMethod = _nethttp.MethodGet + localVarPostBody interface{} + localVarFormFileName string + localVarFileName string + localVarFileBytes []byte + localVarReturnValue PinStatus + ) + + localBasePath, err := r.apiService.client.cfg.ServerURLWithContext(r.ctx, "PinsApiService.PinsRequestidGet") + if err != nil { + return localVarReturnValue, nil, GenericOpenAPIError{error: err.Error()} + } + + localVarPath := localBasePath + "/pins/{requestid}" + localVarPath = strings.Replace(localVarPath, "{"+"requestid"+"}", _neturl.PathEscape(parameterToString(r.requestid, "")), -1) + + localVarHeaderParams := make(map[string]string) + localVarQueryParams := _neturl.Values{} + localVarFormParams := _neturl.Values{} + + // to determine the Content-Type header + localVarHTTPContentTypes := []string{} + + // set Content-Type header + localVarHTTPContentType := selectHeaderContentType(localVarHTTPContentTypes) + if localVarHTTPContentType != "" { + localVarHeaderParams["Content-Type"] = localVarHTTPContentType + } + + // to determine the Accept header + localVarHTTPHeaderAccepts := []string{"application/json"} + + // set Accept header + localVarHTTPHeaderAccept := selectHeaderAccept(localVarHTTPHeaderAccepts) + if localVarHTTPHeaderAccept != "" { + localVarHeaderParams["Accept"] = localVarHTTPHeaderAccept + } + req, err := r.apiService.client.prepareRequest(r.ctx, localVarPath, localVarHTTPMethod, localVarPostBody, localVarHeaderParams, localVarQueryParams, localVarFormParams, localVarFormFileName, localVarFileName, localVarFileBytes) + if err != nil { + return localVarReturnValue, nil, err + } + + localVarHTTPResponse, err := r.apiService.client.callAPI(req) + if err != nil || localVarHTTPResponse == nil { + return localVarReturnValue, localVarHTTPResponse, err + } + + localVarBody, err := _io.ReadAll(localVarHTTPResponse.Body) + localVarHTTPResponse.Body.Close() + if err != nil { + return localVarReturnValue, localVarHTTPResponse, err + } + + if localVarHTTPResponse.StatusCode >= 300 { + newErr := GenericOpenAPIError{ + body: localVarBody, + error: localVarHTTPResponse.Status, + } + if localVarHTTPResponse.StatusCode >= 400 && localVarHTTPResponse.StatusCode <= 600 { + var v Failure + err = r.apiService.client.decode(&v, localVarBody, localVarHTTPResponse.Header.Get("Content-Type")) + if err != nil { + newErr.error = err.Error() + return localVarReturnValue, localVarHTTPResponse, newErr + } + newErr.model = v + return localVarReturnValue, localVarHTTPResponse, newErr + } + return localVarReturnValue, localVarHTTPResponse, newErr + } + + err = r.apiService.client.decode(&localVarReturnValue, localVarBody, localVarHTTPResponse.Header.Get("Content-Type")) + if err != nil { + newErr := GenericOpenAPIError{ + body: localVarBody, + error: err.Error(), + } + return localVarReturnValue, localVarHTTPResponse, newErr + } + + return localVarReturnValue, localVarHTTPResponse, nil +} + +type apiPinsRequestidPostRequest struct { + ctx _context.Context + apiService *PinsApiService + requestid string + pin *Pin +} + +func (r apiPinsRequestidPostRequest) Pin(pin Pin) apiPinsRequestidPostRequest { + r.pin = &pin + return r +} + +/* +PinsRequestidPost Replace pin object +Replace an existing pin object (shortcut for executing remove and add operations in one step to avoid unnecessary garbage collection of blocks present in both recursive pins) + - @param ctx _context.Context - for authentication, logging, cancellation, deadlines, tracing, etc. Passed from http.Request or context.Background(). + - @param requestid + +@return apiPinsRequestidPostRequest +*/ +func (a *PinsApiService) PinsRequestidPost(ctx _context.Context, requestid string) apiPinsRequestidPostRequest { + return apiPinsRequestidPostRequest{ + apiService: a, + ctx: ctx, + requestid: requestid, + } +} + +/* +Execute executes the request + + @return PinStatus +*/ +func (r apiPinsRequestidPostRequest) Execute() (PinStatus, *_nethttp.Response, error) { + var ( + localVarHTTPMethod = _nethttp.MethodPost + localVarPostBody interface{} + localVarFormFileName string + localVarFileName string + localVarFileBytes []byte + localVarReturnValue PinStatus + ) + + localBasePath, err := r.apiService.client.cfg.ServerURLWithContext(r.ctx, "PinsApiService.PinsRequestidPost") + if err != nil { + return localVarReturnValue, nil, GenericOpenAPIError{error: err.Error()} + } + + localVarPath := localBasePath + "/pins/{requestid}" + localVarPath = strings.Replace(localVarPath, "{"+"requestid"+"}", _neturl.PathEscape(parameterToString(r.requestid, "")), -1) + + localVarHeaderParams := make(map[string]string) + localVarQueryParams := _neturl.Values{} + localVarFormParams := _neturl.Values{} + + if r.pin == nil { + return localVarReturnValue, nil, reportError("pin is required and must be specified") + } + + // to determine the Content-Type header + localVarHTTPContentTypes := []string{"application/json"} + + // set Content-Type header + localVarHTTPContentType := selectHeaderContentType(localVarHTTPContentTypes) + if localVarHTTPContentType != "" { + localVarHeaderParams["Content-Type"] = localVarHTTPContentType + } + + // to determine the Accept header + localVarHTTPHeaderAccepts := []string{"application/json"} + + // set Accept header + localVarHTTPHeaderAccept := selectHeaderAccept(localVarHTTPHeaderAccepts) + if localVarHTTPHeaderAccept != "" { + localVarHeaderParams["Accept"] = localVarHTTPHeaderAccept + } + // body params + localVarPostBody = r.pin + req, err := r.apiService.client.prepareRequest(r.ctx, localVarPath, localVarHTTPMethod, localVarPostBody, localVarHeaderParams, localVarQueryParams, localVarFormParams, localVarFormFileName, localVarFileName, localVarFileBytes) + if err != nil { + return localVarReturnValue, nil, err + } + + localVarHTTPResponse, err := r.apiService.client.callAPI(req) + if err != nil || localVarHTTPResponse == nil { + return localVarReturnValue, localVarHTTPResponse, err + } + + localVarBody, err := _io.ReadAll(localVarHTTPResponse.Body) + localVarHTTPResponse.Body.Close() + if err != nil { + return localVarReturnValue, localVarHTTPResponse, err + } + + if localVarHTTPResponse.StatusCode >= 300 { + newErr := GenericOpenAPIError{ + body: localVarBody, + error: localVarHTTPResponse.Status, + } + if localVarHTTPResponse.StatusCode >= 400 && localVarHTTPResponse.StatusCode <= 600 { + var v Failure + err = r.apiService.client.decode(&v, localVarBody, localVarHTTPResponse.Header.Get("Content-Type")) + if err != nil { + newErr.error = err.Error() + return localVarReturnValue, localVarHTTPResponse, newErr + } + newErr.model = v + return localVarReturnValue, localVarHTTPResponse, newErr + } + return localVarReturnValue, localVarHTTPResponse, newErr + } + + err = r.apiService.client.decode(&localVarReturnValue, localVarBody, localVarHTTPResponse.Header.Get("Content-Type")) + if err != nil { + newErr := GenericOpenAPIError{ + body: localVarBody, + error: err.Error(), + } + return localVarReturnValue, localVarHTTPResponse, newErr + } + + return localVarReturnValue, localVarHTTPResponse, nil +} diff --git a/pinning/remote/client/openapi/client.go b/pinning/remote/client/openapi/client.go new file mode 100644 index 0000000000..b3cea998b5 --- /dev/null +++ b/pinning/remote/client/openapi/client.go @@ -0,0 +1,498 @@ +/* + * IPFS Pinning Service API + * + * ## About this spec The IPFS Pinning Service API is intended to be an implementation-agnostic API: - For use and implementation by pinning service providers - For use in client mode by IPFS nodes and GUI-based applications > **Note**: while ready for implementation, this spec is still a work in progress! 🏗️ **Your input and feedback are welcome and valuable as we develop this API spec. Please join the design discussion at [github.com/ipfs/pinning-services-api-spec](https://github.com/ipfs/pinning-services-api-spec).** # Schemas This section describes the most important object types and conventions. A full list of fields and schemas can be found in the `schemas` section of the [YAML file](https://github.com/ipfs/pinning-services-api-spec/blob/master/ipfs-pinning-service.yaml). ## Identifiers ### cid [Content Identifier (CID)](https://docs.ipfs.io/concepts/content-addressing/) points at the root of a DAG that is pinned recursively. ### requestid Unique identifier of a pin request. When a pin is created, the service responds with unique `requestid` that can be later used for pin removal. When the same `cid` is pinned again, a different `requestid` is returned to differentiate between those pin requests. Service implementation should use UUID, `hash(accessToken,Pin,PinStatus.created)`, or any other opaque identifier that provides equally strong protection against race conditions. ## Objects ### Pin object ![pin object](https://bafybeideck2fchyxna4wqwc2mo67yriokehw3yujboc5redjdaajrk2fjq.ipfs.dweb.link/pin.png) The `Pin` object is a representation of a pin request. It includes the `cid` of data to be pinned, as well as optional metadata in `name`, `origins`, and `meta`. ### Pin status response ![pin status response object](https://bafybeideck2fchyxna4wqwc2mo67yriokehw3yujboc5redjdaajrk2fjq.ipfs.dweb.link/pinstatus.png) The `PinStatus` object is a representation of the current state of a pinning operation. It includes the original `pin` object, along with the current `status` and globally unique `requestid` of the entire pinning request, which can be used for future status checks and management. Addresses in the `delegates` array are peers delegated by the pinning service for facilitating direct file transfers (more details in the provider hints section). Any additional vendor-specific information is returned in optional `info`. ## The pin lifecycle ![pinning service objects and lifecycle](https://bafybeideck2fchyxna4wqwc2mo67yriokehw3yujboc5redjdaajrk2fjq.ipfs.dweb.link/lifecycle.png) ### Creating a new pin object The user sends a `Pin` object to `POST /pins` and receives a `PinStatus` response: - `requestid` in `PinStatus` is the identifier of the pin operation, which can can be used for checking status, and removing the pin in the future - `status` in `PinStatus` indicates the current state of a pin ### Checking status of in-progress pinning `status` (in `PinStatus`) may indicate a pending state (`queued` or `pinning`). This means the data behind `Pin.cid` was not found on the pinning service and is being fetched from the IPFS network at large, which may take time. In this case, the user can periodically check pinning progress via `GET /pins/{requestid}` until pinning is successful, or the user decides to remove the pending pin. ### Replacing an existing pin object The user can replace an existing pin object via `POST /pins/{requestid}`. This is a shortcut for removing a pin object identified by `requestid` and creating a new one in a single API call that protects against undesired garbage collection of blocks common to both pins. Useful when updating a pin representing a huge dataset where most of blocks did not change. The new pin object `requestid` is returned in the `PinStatus` response. The old pin object is deleted automatically. ### Removing a pin object A pin object can be removed via `DELETE /pins/{requestid}`. ## Provider hints Pinning of new data can be accelerated by providing a list of known data sources in `Pin.origins`, and connecting at least one of them to pinning service nodes at `PinStatus.delegates`. The most common scenario is a client putting its own IPFS node's multiaddrs in `Pin.origins`, and then directly connecting to every multiaddr returned by a pinning service in `PinStatus.delegates` to initiate transfer. This ensures data transfer starts immediately (without waiting for provider discovery over DHT), and direct dial from a client works around peer routing issues in restrictive network topologies such as NATs. ## Custom metadata Pinning services are encouraged to add support for additional features by leveraging the optional `Pin.meta` and `PinStatus.info` fields. While these attributes can be application- or vendor-specific, we encourage the community at large to leverage these attributes as a sandbox to come up with conventions that could become part of future revisions of this API. ### Pin metadata String keys and values passed in `Pin.meta` are persisted with the pin object. Potential uses: - `Pin.meta[app_id]`: Attaching a unique identifier to pins created by an app enables filtering pins per app via `?meta={\"app_id\":}` - `Pin.meta[vendor_policy]`: Vendor-specific policy (for example: which region to use, how many copies to keep) Note that it is OK for a client to omit or ignore these optional attributes; doing so should not impact the basic pinning functionality. ### Pin status info Additional `PinStatus.info` can be returned by pinning service. Potential uses: - `PinStatus.info[status_details]`: more info about the current status (queue position, percentage of transferred data, summary of where data is stored, etc); when `PinStatus.status=failed`, it could provide a reason why a pin operation failed (e.g. lack of funds, DAG too big, etc.) - `PinStatus.info[dag_size]`: the size of pinned data, along with DAG overhead - `PinStatus.info[raw_size]`: the size of data without DAG overhead (eg. unixfs) - `PinStatus.info[pinned_until]`: if vendor supports time-bound pins, this could indicate when the pin will expire # Pagination and filtering Pin objects can be listed by executing `GET /pins` with optional parameters: - When no filters are provided, the endpoint will return a small batch of the 10 most recently created items, from the latest to the oldest. - The number of returned items can be adjusted with the `limit` parameter (implicit default is 10). - If the value in `PinResults.count` is bigger than the length of `PinResults.results`, the client can infer there are more results that can be queried. - To read more items, pass the `before` filter with the timestamp from `PinStatus.created` found in the oldest item in the current batch of results. Repeat to read all results. - Returned results can be fine-tuned by applying optional `after`, `cid`, `name`, `status`, or `meta` filters. > **Note**: pagination by the `created` timestamp requires each value to be globally unique. Any future considerations to add support for bulk creation must account for this. + * + * API version: 0.1.1 + * Generated by: OpenAPI Generator (https://openapi-generator.tech) + */ + +package openapi + +import ( + "bytes" + "context" + "encoding/json" + "encoding/xml" + "errors" + "fmt" + "io" + "log" + "mime/multipart" + "net/http" + "net/http/httputil" + "net/url" + "os" + "path/filepath" + "reflect" + "regexp" + "strings" + "time" + + "golang.org/x/oauth2" +) + +var ( + jsonCheck = regexp.MustCompile(`(?i:(?:application|text)/(?:vnd\.[^;]+\+)?json)`) + xmlCheck = regexp.MustCompile(`(?i:(?:application|text)/xml)`) +) + +// APIClient manages communication with the IPFS Pinning Service API API v0.1.1 +// In most cases there should be only one, shared, APIClient. +type APIClient struct { + cfg *Configuration + common service // Reuse a single struct instead of allocating one for each service on the heap. + + // API Services + + PinsApi *PinsApiService +} + +type service struct { + client *APIClient +} + +// NewAPIClient creates a new API client. Requires a userAgent string describing your application. +// optionally a custom http.Client to allow for advanced features such as caching. +func NewAPIClient(cfg *Configuration) *APIClient { + if cfg.HTTPClient == nil { + cfg.HTTPClient = http.DefaultClient + } + + c := &APIClient{} + c.cfg = cfg + c.common.client = c + + // API Services + c.PinsApi = (*PinsApiService)(&c.common) + + return c +} + +// selectHeaderContentType select a content type from the available list. +func selectHeaderContentType(contentTypes []string) string { + if len(contentTypes) == 0 { + return "" + } + if contains(contentTypes, "application/json") { + return "application/json" + } + return contentTypes[0] // use the first content type specified in 'consumes' +} + +// selectHeaderAccept join all accept types and return +func selectHeaderAccept(accepts []string) string { + if len(accepts) == 0 { + return "" + } + + if contains(accepts, "application/json") { + return "application/json" + } + + return strings.Join(accepts, ",") +} + +// contains is a case insenstive match, finding needle in a haystack +func contains(haystack []string, needle string) bool { + for _, a := range haystack { + if strings.EqualFold(a, needle) { + return true + } + } + return false +} + +// parameterToString convert interface{} parameters to string, using a delimiter if format is provided. +func parameterToString(obj interface{}, collectionFormat string) string { + var delimiter string + + switch collectionFormat { + case "pipes": + delimiter = "|" + case "ssv": + delimiter = " " + case "tsv": + delimiter = "\t" + case "csv": + delimiter = "," + } + + if reflect.TypeOf(obj).Kind() == reflect.Slice { + return strings.Trim(strings.Replace(fmt.Sprint(obj), " ", delimiter, -1), "[]") + } else if t, ok := obj.(time.Time); ok { + return t.Format(time.RFC3339Nano) + } + + return fmt.Sprintf("%v", obj) +} + +// callAPI do the request. +func (c *APIClient) callAPI(request *http.Request) (*http.Response, error) { + if c.cfg.Debug { + dump, err := httputil.DumpRequestOut(request, true) + if err != nil { + return nil, err + } + log.Printf("\n%s\n", string(dump)) + } + + resp, err := c.cfg.HTTPClient.Do(request) + if err != nil { + return resp, err + } + + if c.cfg.Debug { + dump, err := httputil.DumpResponse(resp, true) + if err != nil { + return resp, err + } + log.Printf("\n%s\n", string(dump)) + } + return resp, err +} + +// Allow modification of underlying config for alternate implementations and testing +// Caution: modifying the configuration while live can cause data races and potentially unwanted behavior +func (c *APIClient) GetConfig() *Configuration { + return c.cfg +} + +// prepareRequest build the request +func (c *APIClient) prepareRequest( + ctx context.Context, + path string, method string, + postBody interface{}, + headerParams map[string]string, + queryParams url.Values, + formParams url.Values, + formFileName string, + fileName string, + fileBytes []byte) (localVarRequest *http.Request, err error) { + + var body *bytes.Buffer + + // Detect postBody type and post. + if postBody != nil { + contentType := headerParams["Content-Type"] + if contentType == "" { + contentType = detectContentType(postBody) + headerParams["Content-Type"] = contentType + } + + body, err = setBody(postBody, contentType) + if err != nil { + return nil, err + } + } + + // add form parameters and file if available. + if strings.HasPrefix(headerParams["Content-Type"], "multipart/form-data") && len(formParams) > 0 || (len(fileBytes) > 0 && fileName != "") { + if body != nil { + return nil, errors.New("cannot specify postBody and multipart form at the same time") + } + body = &bytes.Buffer{} + w := multipart.NewWriter(body) + + for k, v := range formParams { + for _, iv := range v { + if strings.HasPrefix(k, "@") { // file + err = addFile(w, k[1:], iv) + if err != nil { + return nil, err + } + } else { // form value + w.WriteField(k, iv) + } + } + } + if len(fileBytes) > 0 && fileName != "" { + w.Boundary() + //_, fileNm := filepath.Split(fileName) + part, err := w.CreateFormFile(formFileName, filepath.Base(fileName)) + if err != nil { + return nil, err + } + _, err = part.Write(fileBytes) + if err != nil { + return nil, err + } + } + + // Set the Boundary in the Content-Type + headerParams["Content-Type"] = w.FormDataContentType() + + // Set Content-Length + headerParams["Content-Length"] = fmt.Sprintf("%d", body.Len()) + w.Close() + } + + if strings.HasPrefix(headerParams["Content-Type"], "application/x-www-form-urlencoded") && len(formParams) > 0 { + if body != nil { + return nil, errors.New("cannot specify postBody and x-www-form-urlencoded form at the same time") + } + body = &bytes.Buffer{} + body.WriteString(formParams.Encode()) + // Set Content-Length + headerParams["Content-Length"] = fmt.Sprintf("%d", body.Len()) + } + + // Setup path and query parameters + url, err := url.Parse(path) + if err != nil { + return nil, err + } + + // Override request host, if applicable + if c.cfg.Host != "" { + url.Host = c.cfg.Host + } + + // Override request scheme, if applicable + if c.cfg.Scheme != "" { + url.Scheme = c.cfg.Scheme + } + + // Adding Query Param + query := url.Query() + for k, v := range queryParams { + for _, iv := range v { + query.Add(k, iv) + } + } + + // Encode the parameters. + url.RawQuery = query.Encode() + + // Generate a new request + if body != nil { + localVarRequest, err = http.NewRequest(method, url.String(), body) + } else { + localVarRequest, err = http.NewRequest(method, url.String(), nil) + } + if err != nil { + return nil, err + } + + // add header parameters, if any + if len(headerParams) > 0 { + headers := http.Header{} + for h, v := range headerParams { + headers.Set(h, v) + } + localVarRequest.Header = headers + } + + // Add the user agent to the request. + localVarRequest.Header.Add("User-Agent", c.cfg.UserAgent) + + if ctx != nil { + // add context to the request + localVarRequest = localVarRequest.WithContext(ctx) + + // Walk through any authentication. + + // OAuth2 authentication + if tok, ok := ctx.Value(ContextOAuth2).(oauth2.TokenSource); ok { + // We were able to grab an oauth2 token from the context + var latestToken *oauth2.Token + if latestToken, err = tok.Token(); err != nil { + return nil, err + } + + latestToken.SetAuthHeader(localVarRequest) + } + + // Basic HTTP Authentication + if auth, ok := ctx.Value(ContextBasicAuth).(BasicAuth); ok { + localVarRequest.SetBasicAuth(auth.UserName, auth.Password) + } + + // AccessToken Authentication + if auth, ok := ctx.Value(ContextAccessToken).(string); ok { + localVarRequest.Header.Add("Authorization", "Bearer "+auth) + } + } + + for header, value := range c.cfg.DefaultHeader { + localVarRequest.Header.Add(header, value) + } + return localVarRequest, nil +} + +func (c *APIClient) decode(v interface{}, b []byte, contentType string) (err error) { + if len(b) == 0 { + return nil + } + if s, ok := v.(*string); ok { + *s = string(b) + return nil + } + if xmlCheck.MatchString(contentType) { + if err = xml.Unmarshal(b, v); err != nil { + return err + } + return nil + } + if jsonCheck.MatchString(contentType) { + if actualObj, ok := v.(interface{ GetActualInstance() interface{} }); ok { // oneOf, anyOf schemas + if unmarshalObj, ok := actualObj.(interface{ UnmarshalJSON([]byte) error }); ok { // make sure it has UnmarshalJSON defined + if err = unmarshalObj.UnmarshalJSON(b); err != nil { + return err + } + } else { + return errors.New("unknown type with GetActualInstance but no unmarshalObj.UnmarshalJSON defined") + } + } else if err = json.Unmarshal(b, v); err != nil { // simple model + return err + } + return nil + } + return errors.New("undefined response type") +} + +// Add a file to the multipart request +func addFile(w *multipart.Writer, fieldName, path string) error { + file, err := os.Open(path) + if err != nil { + return err + } + defer file.Close() + + part, err := w.CreateFormFile(fieldName, filepath.Base(path)) + if err != nil { + return err + } + _, err = io.Copy(part, file) + + return err +} + +// Prevent trying to import "fmt" +func reportError(format string, a ...interface{}) error { + return fmt.Errorf(format, a...) +} + +// Set request body from an interface{} +func setBody(body interface{}, contentType string) (bodyBuf *bytes.Buffer, err error) { + if bodyBuf == nil { + bodyBuf = &bytes.Buffer{} + } + + if reader, ok := body.(io.Reader); ok { + _, err = bodyBuf.ReadFrom(reader) + } else if b, ok := body.([]byte); ok { + _, err = bodyBuf.Write(b) + } else if s, ok := body.(string); ok { + _, err = bodyBuf.WriteString(s) + } else if s, ok := body.(*string); ok { + _, err = bodyBuf.WriteString(*s) + } else if jsonCheck.MatchString(contentType) { + err = json.NewEncoder(bodyBuf).Encode(body) + } else if xmlCheck.MatchString(contentType) { + err = xml.NewEncoder(bodyBuf).Encode(body) + } + + if err != nil { + return nil, err + } + + if bodyBuf.Len() == 0 { + err = fmt.Errorf("invalid body type %s", contentType) + return nil, err + } + return bodyBuf, nil +} + +// detectContentType method is used to figure out `Request.Body` content type for request header +func detectContentType(body interface{}) string { + contentType := "text/plain; charset=utf-8" + kind := reflect.TypeOf(body).Kind() + + switch kind { + case reflect.Struct, reflect.Map, reflect.Ptr: + contentType = "application/json; charset=utf-8" + case reflect.String: + contentType = "text/plain; charset=utf-8" + default: + if b, ok := body.([]byte); ok { + contentType = http.DetectContentType(b) + } else if kind == reflect.Slice { + contentType = "application/json; charset=utf-8" + } + } + + return contentType +} + +// Ripped from https://github.com/gregjones/httpcache/blob/master/httpcache.go +type cacheControl map[string]string + +func parseCacheControl(headers http.Header) cacheControl { + cc := cacheControl{} + ccHeader := headers.Get("Cache-Control") + for _, part := range strings.Split(ccHeader, ",") { + part = strings.Trim(part, " ") + if part == "" { + continue + } + if strings.ContainsRune(part, '=') { + keyval := strings.Split(part, "=") + cc[strings.Trim(keyval[0], " ")] = strings.Trim(keyval[1], ",") + } else { + cc[part] = "" + } + } + return cc +} + +// CacheExpires helper function to determine remaining time before repeating a request. +func CacheExpires(r *http.Response) time.Time { + // Figure out when the cache expires. + var expires time.Time + now, err := time.Parse(time.RFC1123, r.Header.Get("date")) + if err != nil { + return time.Now() + } + respCacheControl := parseCacheControl(r.Header) + + if maxAge, ok := respCacheControl["max-age"]; ok { + lifetime, err := time.ParseDuration(maxAge + "s") + if err != nil { + expires = now + } else { + expires = now.Add(lifetime) + } + } else { + expiresHeader := r.Header.Get("Expires") + if expiresHeader != "" { + expires, err = time.Parse(time.RFC1123, expiresHeader) + if err != nil { + expires = now + } + } + } + return expires +} + +// GenericOpenAPIError Provides access to the body, error and model on returned errors. +type GenericOpenAPIError struct { + body []byte + error string + model interface{} +} + +// Error returns non-empty string if there was an error. +func (e GenericOpenAPIError) Error() string { + return e.error +} + +// Body returns the raw bytes of the response +func (e GenericOpenAPIError) Body() []byte { + return e.body +} + +// Model returns the unpacked model of the error +func (e GenericOpenAPIError) Model() interface{} { + return e.model +} diff --git a/pinning/remote/client/openapi/configuration.go b/pinning/remote/client/openapi/configuration.go new file mode 100644 index 0000000000..bb0a8e5073 --- /dev/null +++ b/pinning/remote/client/openapi/configuration.go @@ -0,0 +1,228 @@ +/* + * IPFS Pinning Service API + * + * ## About this spec The IPFS Pinning Service API is intended to be an implementation-agnostic API: - For use and implementation by pinning service providers - For use in client mode by IPFS nodes and GUI-based applications > **Note**: while ready for implementation, this spec is still a work in progress! 🏗️ **Your input and feedback are welcome and valuable as we develop this API spec. Please join the design discussion at [github.com/ipfs/pinning-services-api-spec](https://github.com/ipfs/pinning-services-api-spec).** # Schemas This section describes the most important object types and conventions. A full list of fields and schemas can be found in the `schemas` section of the [YAML file](https://github.com/ipfs/pinning-services-api-spec/blob/master/ipfs-pinning-service.yaml). ## Identifiers ### cid [Content Identifier (CID)](https://docs.ipfs.io/concepts/content-addressing/) points at the root of a DAG that is pinned recursively. ### requestid Unique identifier of a pin request. When a pin is created, the service responds with unique `requestid` that can be later used for pin removal. When the same `cid` is pinned again, a different `requestid` is returned to differentiate between those pin requests. Service implementation should use UUID, `hash(accessToken,Pin,PinStatus.created)`, or any other opaque identifier that provides equally strong protection against race conditions. ## Objects ### Pin object ![pin object](https://bafybeideck2fchyxna4wqwc2mo67yriokehw3yujboc5redjdaajrk2fjq.ipfs.dweb.link/pin.png) The `Pin` object is a representation of a pin request. It includes the `cid` of data to be pinned, as well as optional metadata in `name`, `origins`, and `meta`. ### Pin status response ![pin status response object](https://bafybeideck2fchyxna4wqwc2mo67yriokehw3yujboc5redjdaajrk2fjq.ipfs.dweb.link/pinstatus.png) The `PinStatus` object is a representation of the current state of a pinning operation. It includes the original `pin` object, along with the current `status` and globally unique `requestid` of the entire pinning request, which can be used for future status checks and management. Addresses in the `delegates` array are peers delegated by the pinning service for facilitating direct file transfers (more details in the provider hints section). Any additional vendor-specific information is returned in optional `info`. ## The pin lifecycle ![pinning service objects and lifecycle](https://bafybeideck2fchyxna4wqwc2mo67yriokehw3yujboc5redjdaajrk2fjq.ipfs.dweb.link/lifecycle.png) ### Creating a new pin object The user sends a `Pin` object to `POST /pins` and receives a `PinStatus` response: - `requestid` in `PinStatus` is the identifier of the pin operation, which can can be used for checking status, and removing the pin in the future - `status` in `PinStatus` indicates the current state of a pin ### Checking status of in-progress pinning `status` (in `PinStatus`) may indicate a pending state (`queued` or `pinning`). This means the data behind `Pin.cid` was not found on the pinning service and is being fetched from the IPFS network at large, which may take time. In this case, the user can periodically check pinning progress via `GET /pins/{requestid}` until pinning is successful, or the user decides to remove the pending pin. ### Replacing an existing pin object The user can replace an existing pin object via `POST /pins/{requestid}`. This is a shortcut for removing a pin object identified by `requestid` and creating a new one in a single API call that protects against undesired garbage collection of blocks common to both pins. Useful when updating a pin representing a huge dataset where most of blocks did not change. The new pin object `requestid` is returned in the `PinStatus` response. The old pin object is deleted automatically. ### Removing a pin object A pin object can be removed via `DELETE /pins/{requestid}`. ## Provider hints Pinning of new data can be accelerated by providing a list of known data sources in `Pin.origins`, and connecting at least one of them to pinning service nodes at `PinStatus.delegates`. The most common scenario is a client putting its own IPFS node's multiaddrs in `Pin.origins`, and then directly connecting to every multiaddr returned by a pinning service in `PinStatus.delegates` to initiate transfer. This ensures data transfer starts immediately (without waiting for provider discovery over DHT), and direct dial from a client works around peer routing issues in restrictive network topologies such as NATs. ## Custom metadata Pinning services are encouraged to add support for additional features by leveraging the optional `Pin.meta` and `PinStatus.info` fields. While these attributes can be application- or vendor-specific, we encourage the community at large to leverage these attributes as a sandbox to come up with conventions that could become part of future revisions of this API. ### Pin metadata String keys and values passed in `Pin.meta` are persisted with the pin object. Potential uses: - `Pin.meta[app_id]`: Attaching a unique identifier to pins created by an app enables filtering pins per app via `?meta={\"app_id\":}` - `Pin.meta[vendor_policy]`: Vendor-specific policy (for example: which region to use, how many copies to keep) Note that it is OK for a client to omit or ignore these optional attributes; doing so should not impact the basic pinning functionality. ### Pin status info Additional `PinStatus.info` can be returned by pinning service. Potential uses: - `PinStatus.info[status_details]`: more info about the current status (queue position, percentage of transferred data, summary of where data is stored, etc); when `PinStatus.status=failed`, it could provide a reason why a pin operation failed (e.g. lack of funds, DAG too big, etc.) - `PinStatus.info[dag_size]`: the size of pinned data, along with DAG overhead - `PinStatus.info[raw_size]`: the size of data without DAG overhead (eg. unixfs) - `PinStatus.info[pinned_until]`: if vendor supports time-bound pins, this could indicate when the pin will expire # Pagination and filtering Pin objects can be listed by executing `GET /pins` with optional parameters: - When no filters are provided, the endpoint will return a small batch of the 10 most recently created items, from the latest to the oldest. - The number of returned items can be adjusted with the `limit` parameter (implicit default is 10). - If the value in `PinResults.count` is bigger than the length of `PinResults.results`, the client can infer there are more results that can be queried. - To read more items, pass the `before` filter with the timestamp from `PinStatus.created` found in the oldest item in the current batch of results. Repeat to read all results. - Returned results can be fine-tuned by applying optional `after`, `cid`, `name`, `status`, or `meta` filters. > **Note**: pagination by the `created` timestamp requires each value to be globally unique. Any future considerations to add support for bulk creation must account for this. + * + * API version: 0.1.1 + * Generated by: OpenAPI Generator (https://openapi-generator.tech) + */ + +package openapi + +import ( + "context" + "fmt" + "net/http" + "strings" +) + +// contextKeys are used to identify the type of value in the context. +// Since these are string, it is possible to get a short description of the +// context key for logging and debugging using key.String(). + +type contextKey string + +func (c contextKey) String() string { + return "auth " + string(c) +} + +var ( + // ContextOAuth2 takes an oauth2.TokenSource as authentication for the request. + ContextOAuth2 = contextKey("token") + + // ContextBasicAuth takes BasicAuth as authentication for the request. + ContextBasicAuth = contextKey("basic") + + // ContextAccessToken takes a string oauth2 access token as authentication for the request. + ContextAccessToken = contextKey("accesstoken") + + // ContextAPIKeys takes a string apikey as authentication for the request + ContextAPIKeys = contextKey("apiKeys") + + // ContextHttpSignatureAuth takes HttpSignatureAuth as authentication for the request. + ContextHttpSignatureAuth = contextKey("httpsignature") + + // ContextServerIndex uses a server configuration from the index. + ContextServerIndex = contextKey("serverIndex") + + // ContextOperationServerIndices uses a server configuration from the index mapping. + ContextOperationServerIndices = contextKey("serverOperationIndices") + + // ContextServerVariables overrides a server configuration variables. + ContextServerVariables = contextKey("serverVariables") + + // ContextOperationServerVariables overrides a server configuration variables using operation specific values. + ContextOperationServerVariables = contextKey("serverOperationVariables") +) + +// BasicAuth provides basic http authentication to a request passed via context using ContextBasicAuth +type BasicAuth struct { + UserName string `json:"userName,omitempty"` + Password string `json:"password,omitempty"` +} + +// APIKey provides API key based authentication to a request passed via context using ContextAPIKey +type APIKey struct { + Key string + Prefix string +} + +// ServerVariable stores the information about a server variable +type ServerVariable struct { + Description string + DefaultValue string + EnumValues []string +} + +// ServerConfiguration stores the information about a server +type ServerConfiguration struct { + URL string + Description string + Variables map[string]ServerVariable +} + +// ServerConfigurations stores multiple ServerConfiguration items +type ServerConfigurations []ServerConfiguration + +// Configuration stores the configuration of the API client +type Configuration struct { + Host string `json:"host,omitempty"` + Scheme string `json:"scheme,omitempty"` + DefaultHeader map[string]string `json:"defaultHeader,omitempty"` + UserAgent string `json:"userAgent,omitempty"` + Debug bool `json:"debug,omitempty"` + Servers ServerConfigurations + OperationServers map[string]ServerConfigurations + HTTPClient *http.Client +} + +// NewConfiguration returns a new Configuration object +func NewConfiguration() *Configuration { + cfg := &Configuration{ + DefaultHeader: make(map[string]string), + UserAgent: "OpenAPI-Generator/1.0.0/go", + Debug: false, + Servers: ServerConfigurations{ + { + URL: "https://pinning-service.example.com", + Description: "No description provided", + }, + }, + OperationServers: map[string]ServerConfigurations{}, + } + return cfg +} + +// AddDefaultHeader adds a new HTTP header to the default header in the request +func (c *Configuration) AddDefaultHeader(key string, value string) { + c.DefaultHeader[key] = value +} + +// URL formats template on a index using given variables +func (sc ServerConfigurations) URL(index int, variables map[string]string) (string, error) { + if index < 0 || len(sc) <= index { + return "", fmt.Errorf("index %v out of range %v", index, len(sc)-1) + } + server := sc[index] + url := server.URL + + // go through variables and replace placeholders + for name, variable := range server.Variables { + if value, ok := variables[name]; ok { + found := bool(len(variable.EnumValues) == 0) + for _, enumValue := range variable.EnumValues { + if value == enumValue { + found = true + } + } + if !found { + return "", fmt.Errorf("the variable %s in the server URL has invalid value %v. Must be %v", name, value, variable.EnumValues) + } + url = strings.Replace(url, "{"+name+"}", value, -1) + } else { + url = strings.Replace(url, "{"+name+"}", variable.DefaultValue, -1) + } + } + return url, nil +} + +// ServerURL returns URL based on server settings +func (c *Configuration) ServerURL(index int, variables map[string]string) (string, error) { + return c.Servers.URL(index, variables) +} + +func getServerIndex(ctx context.Context) (int, error) { + si := ctx.Value(ContextServerIndex) + if si != nil { + if index, ok := si.(int); ok { + return index, nil + } + return 0, reportError("Invalid type %T should be int", si) + } + return 0, nil +} + +func getServerOperationIndex(ctx context.Context, endpoint string) (int, error) { + osi := ctx.Value(ContextOperationServerIndices) + if osi != nil { + if operationIndices, ok := osi.(map[string]int); !ok { + return 0, reportError("Invalid type %T should be map[string]int", osi) + } else { + index, ok := operationIndices[endpoint] + if ok { + return index, nil + } + } + } + return getServerIndex(ctx) +} + +func getServerVariables(ctx context.Context) (map[string]string, error) { + sv := ctx.Value(ContextServerVariables) + if sv != nil { + if variables, ok := sv.(map[string]string); ok { + return variables, nil + } + return nil, reportError("ctx value of ContextServerVariables has invalid type %T should be map[string]string", sv) + } + return nil, nil +} + +func getServerOperationVariables(ctx context.Context, endpoint string) (map[string]string, error) { + osv := ctx.Value(ContextOperationServerVariables) + if osv != nil { + if operationVariables, ok := osv.(map[string]map[string]string); !ok { + return nil, reportError("ctx value of ContextOperationServerVariables has invalid type %T should be map[string]map[string]string", osv) + } else { + variables, ok := operationVariables[endpoint] + if ok { + return variables, nil + } + } + } + return getServerVariables(ctx) +} + +// ServerURLWithContext returns a new server URL given an endpoint +func (c *Configuration) ServerURLWithContext(ctx context.Context, endpoint string) (string, error) { + sc, ok := c.OperationServers[endpoint] + if !ok { + sc = c.Servers + } + + if ctx == nil { + return sc.URL(0, nil) + } + + index, err := getServerOperationIndex(ctx, endpoint) + if err != nil { + return "", err + } + + variables, err := getServerOperationVariables(ctx, endpoint) + if err != nil { + return "", err + } + + return sc.URL(index, variables) +} diff --git a/pinning/remote/client/openapi/docs/Failure.md b/pinning/remote/client/openapi/docs/Failure.md new file mode 100644 index 0000000000..c899f7138a --- /dev/null +++ b/pinning/remote/client/openapi/docs/Failure.md @@ -0,0 +1,51 @@ +# Failure + +## Properties + +Name | Type | Description | Notes +------------ | ------------- | ------------- | ------------- +**Error** | [**FailureError**](Failure_error.md) | | + +## Methods + +### NewFailure + +`func NewFailure(error_ FailureError, ) *Failure` + +NewFailure instantiates a new Failure object +This constructor will assign default values to properties that have it defined, +and makes sure properties required by API are set, but the set of arguments +will change when the set of required properties is changed + +### NewFailureWithDefaults + +`func NewFailureWithDefaults() *Failure` + +NewFailureWithDefaults instantiates a new Failure object +This constructor will only assign default values to properties that have it defined, +but it doesn't guarantee that properties required by API are set + +### GetError + +`func (o *Failure) GetError() FailureError` + +GetError returns the Error field if non-nil, zero value otherwise. + +### GetErrorOk + +`func (o *Failure) GetErrorOk() (*FailureError, bool)` + +GetErrorOk returns a tuple with the Error field if it's non-nil, zero value otherwise +and a boolean to check if the value has been set. + +### SetError + +`func (o *Failure) SetError(v FailureError)` + +SetError sets Error field to given value. + + + +[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) + + diff --git a/pinning/remote/client/openapi/docs/FailureError.md b/pinning/remote/client/openapi/docs/FailureError.md new file mode 100644 index 0000000000..478f1b942a --- /dev/null +++ b/pinning/remote/client/openapi/docs/FailureError.md @@ -0,0 +1,77 @@ +# FailureError + +## Properties + +Name | Type | Description | Notes +------------ | ------------- | ------------- | ------------- +**Reason** | **string** | Mandatory string identifying the type of error | +**Details** | Pointer to **string** | Optional, longer description of the error; may include UUID of transaction for support, links to documentation etc | [optional] + +## Methods + +### NewFailureError + +`func NewFailureError(reason string, ) *FailureError` + +NewFailureError instantiates a new FailureError object +This constructor will assign default values to properties that have it defined, +and makes sure properties required by API are set, but the set of arguments +will change when the set of required properties is changed + +### NewFailureErrorWithDefaults + +`func NewFailureErrorWithDefaults() *FailureError` + +NewFailureErrorWithDefaults instantiates a new FailureError object +This constructor will only assign default values to properties that have it defined, +but it doesn't guarantee that properties required by API are set + +### GetReason + +`func (o *FailureError) GetReason() string` + +GetReason returns the Reason field if non-nil, zero value otherwise. + +### GetReasonOk + +`func (o *FailureError) GetReasonOk() (*string, bool)` + +GetReasonOk returns a tuple with the Reason field if it's non-nil, zero value otherwise +and a boolean to check if the value has been set. + +### SetReason + +`func (o *FailureError) SetReason(v string)` + +SetReason sets Reason field to given value. + + +### GetDetails + +`func (o *FailureError) GetDetails() string` + +GetDetails returns the Details field if non-nil, zero value otherwise. + +### GetDetailsOk + +`func (o *FailureError) GetDetailsOk() (*string, bool)` + +GetDetailsOk returns a tuple with the Details field if it's non-nil, zero value otherwise +and a boolean to check if the value has been set. + +### SetDetails + +`func (o *FailureError) SetDetails(v string)` + +SetDetails sets Details field to given value. + +### HasDetails + +`func (o *FailureError) HasDetails() bool` + +HasDetails returns a boolean if a field has been set. + + +[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) + + diff --git a/pinning/remote/client/openapi/docs/Pin.md b/pinning/remote/client/openapi/docs/Pin.md new file mode 100644 index 0000000000..e5d3e0f18f --- /dev/null +++ b/pinning/remote/client/openapi/docs/Pin.md @@ -0,0 +1,129 @@ +# Pin + +## Properties + +Name | Type | Description | Notes +------------ | ------------- | ------------- | ------------- +**Cid** | **string** | Content Identifier (CID) to be pinned recursively | +**Name** | Pointer to **string** | Optional name for pinned data; can be used for lookups later | [optional] +**Origins** | Pointer to **[]string** | Optional list of multiaddrs known to provide the data | [optional] +**Meta** | Pointer to **map[string]string** | Optional metadata for pin object | [optional] + +## Methods + +### NewPin + +`func NewPin(cid string, ) *Pin` + +NewPin instantiates a new Pin object +This constructor will assign default values to properties that have it defined, +and makes sure properties required by API are set, but the set of arguments +will change when the set of required properties is changed + +### NewPinWithDefaults + +`func NewPinWithDefaults() *Pin` + +NewPinWithDefaults instantiates a new Pin object +This constructor will only assign default values to properties that have it defined, +but it doesn't guarantee that properties required by API are set + +### GetCid + +`func (o *Pin) GetCid() string` + +GetCid returns the Cid field if non-nil, zero value otherwise. + +### GetCidOk + +`func (o *Pin) GetCidOk() (*string, bool)` + +GetCidOk returns a tuple with the Cid field if it's non-nil, zero value otherwise +and a boolean to check if the value has been set. + +### SetCid + +`func (o *Pin) SetCid(v string)` + +SetCid sets Cid field to given value. + + +### GetName + +`func (o *Pin) GetName() string` + +GetName returns the Name field if non-nil, zero value otherwise. + +### GetNameOk + +`func (o *Pin) GetNameOk() (*string, bool)` + +GetNameOk returns a tuple with the Name field if it's non-nil, zero value otherwise +and a boolean to check if the value has been set. + +### SetName + +`func (o *Pin) SetName(v string)` + +SetName sets Name field to given value. + +### HasName + +`func (o *Pin) HasName() bool` + +HasName returns a boolean if a field has been set. + +### GetOrigins + +`func (o *Pin) GetOrigins() []string` + +GetOrigins returns the Origins field if non-nil, zero value otherwise. + +### GetOriginsOk + +`func (o *Pin) GetOriginsOk() (*[]string, bool)` + +GetOriginsOk returns a tuple with the Origins field if it's non-nil, zero value otherwise +and a boolean to check if the value has been set. + +### SetOrigins + +`func (o *Pin) SetOrigins(v []string)` + +SetOrigins sets Origins field to given value. + +### HasOrigins + +`func (o *Pin) HasOrigins() bool` + +HasOrigins returns a boolean if a field has been set. + +### GetMeta + +`func (o *Pin) GetMeta() map[string]string` + +GetMeta returns the Meta field if non-nil, zero value otherwise. + +### GetMetaOk + +`func (o *Pin) GetMetaOk() (*map[string]string, bool)` + +GetMetaOk returns a tuple with the Meta field if it's non-nil, zero value otherwise +and a boolean to check if the value has been set. + +### SetMeta + +`func (o *Pin) SetMeta(v map[string]string)` + +SetMeta sets Meta field to given value. + +### HasMeta + +`func (o *Pin) HasMeta() bool` + +HasMeta returns a boolean if a field has been set. + + +[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) + + diff --git a/pinning/remote/client/openapi/docs/PinResults.md b/pinning/remote/client/openapi/docs/PinResults.md new file mode 100644 index 0000000000..1982bfddb0 --- /dev/null +++ b/pinning/remote/client/openapi/docs/PinResults.md @@ -0,0 +1,72 @@ +# PinResults + +## Properties + +Name | Type | Description | Notes +------------ | ------------- | ------------- | ------------- +**Count** | **int32** | The total number of pin objects that exist for passed query filters | +**Results** | [**[]PinStatus**](PinStatus.md) | An array of PinStatus results | + +## Methods + +### NewPinResults + +`func NewPinResults(count int32, results []PinStatus, ) *PinResults` + +NewPinResults instantiates a new PinResults object +This constructor will assign default values to properties that have it defined, +and makes sure properties required by API are set, but the set of arguments +will change when the set of required properties is changed + +### NewPinResultsWithDefaults + +`func NewPinResultsWithDefaults() *PinResults` + +NewPinResultsWithDefaults instantiates a new PinResults object +This constructor will only assign default values to properties that have it defined, +but it doesn't guarantee that properties required by API are set + +### GetCount + +`func (o *PinResults) GetCount() int32` + +GetCount returns the Count field if non-nil, zero value otherwise. + +### GetCountOk + +`func (o *PinResults) GetCountOk() (*int32, bool)` + +GetCountOk returns a tuple with the Count field if it's non-nil, zero value otherwise +and a boolean to check if the value has been set. + +### SetCount + +`func (o *PinResults) SetCount(v int32)` + +SetCount sets Count field to given value. + + +### GetResults + +`func (o *PinResults) GetResults() []PinStatus` + +GetResults returns the Results field if non-nil, zero value otherwise. + +### GetResultsOk + +`func (o *PinResults) GetResultsOk() (*[]PinStatus, bool)` + +GetResultsOk returns a tuple with the Results field if it's non-nil, zero value otherwise +and a boolean to check if the value has been set. + +### SetResults + +`func (o *PinResults) SetResults(v []PinStatus)` + +SetResults sets Results field to given value. + + + +[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) + + diff --git a/pinning/remote/client/openapi/docs/PinStatus.md b/pinning/remote/client/openapi/docs/PinStatus.md new file mode 100644 index 0000000000..40ae992ab0 --- /dev/null +++ b/pinning/remote/client/openapi/docs/PinStatus.md @@ -0,0 +1,161 @@ +# PinStatus + +## Properties + +Name | Type | Description | Notes +------------ | ------------- | ------------- | ------------- +**Requestid** | **string** | Globally unique identifier of the pin request; can be used to check the status of ongoing pinning, or pin removal | +**Status** | [**Status**](Status.md) | | +**Created** | [**time.Time**](time.Time.md) | Immutable timestamp indicating when a pin request entered a pinning service; can be used for filtering results and pagination | +**Pin** | [**Pin**](Pin.md) | | +**Delegates** | **[]string** | List of multiaddrs designated by pinning service for transferring any new data from external peers | +**Info** | Pointer to **map[string]string** | Optional info for PinStatus response | [optional] + +## Methods + +### NewPinStatus + +`func NewPinStatus(requestid string, status Status, created time.Time, pin Pin, delegates []string, ) *PinStatus` + +NewPinStatus instantiates a new PinStatus object +This constructor will assign default values to properties that have it defined, +and makes sure properties required by API are set, but the set of arguments +will change when the set of required properties is changed + +### NewPinStatusWithDefaults + +`func NewPinStatusWithDefaults() *PinStatus` + +NewPinStatusWithDefaults instantiates a new PinStatus object +This constructor will only assign default values to properties that have it defined, +but it doesn't guarantee that properties required by API are set + +### GetRequestid + +`func (o *PinStatus) GetRequestid() string` + +GetRequestid returns the Requestid field if non-nil, zero value otherwise. + +### GetRequestidOk + +`func (o *PinStatus) GetRequestidOk() (*string, bool)` + +GetRequestidOk returns a tuple with the Requestid field if it's non-nil, zero value otherwise +and a boolean to check if the value has been set. + +### SetRequestid + +`func (o *PinStatus) SetRequestid(v string)` + +SetRequestid sets Requestid field to given value. + + +### GetStatus + +`func (o *PinStatus) GetStatus() Status` + +GetStatus returns the Status field if non-nil, zero value otherwise. + +### GetStatusOk + +`func (o *PinStatus) GetStatusOk() (*Status, bool)` + +GetStatusOk returns a tuple with the Status field if it's non-nil, zero value otherwise +and a boolean to check if the value has been set. + +### SetStatus + +`func (o *PinStatus) SetStatus(v Status)` + +SetStatus sets Status field to given value. + + +### GetCreated + +`func (o *PinStatus) GetCreated() time.Time` + +GetCreated returns the Created field if non-nil, zero value otherwise. + +### GetCreatedOk + +`func (o *PinStatus) GetCreatedOk() (*time.Time, bool)` + +GetCreatedOk returns a tuple with the Created field if it's non-nil, zero value otherwise +and a boolean to check if the value has been set. + +### SetCreated + +`func (o *PinStatus) SetCreated(v time.Time)` + +SetCreated sets Created field to given value. + + +### GetPin + +`func (o *PinStatus) GetPin() Pin` + +GetPin returns the Pin field if non-nil, zero value otherwise. + +### GetPinOk + +`func (o *PinStatus) GetPinOk() (*Pin, bool)` + +GetPinOk returns a tuple with the Pin field if it's non-nil, zero value otherwise +and a boolean to check if the value has been set. + +### SetPin + +`func (o *PinStatus) SetPin(v Pin)` + +SetPin sets Pin field to given value. + + +### GetDelegates + +`func (o *PinStatus) GetDelegates() []string` + +GetDelegates returns the Delegates field if non-nil, zero value otherwise. + +### GetDelegatesOk + +`func (o *PinStatus) GetDelegatesOk() (*[]string, bool)` + +GetDelegatesOk returns a tuple with the Delegates field if it's non-nil, zero value otherwise +and a boolean to check if the value has been set. + +### SetDelegates + +`func (o *PinStatus) SetDelegates(v []string)` + +SetDelegates sets Delegates field to given value. + + +### GetInfo + +`func (o *PinStatus) GetInfo() map[string]string` + +GetInfo returns the Info field if non-nil, zero value otherwise. + +### GetInfoOk + +`func (o *PinStatus) GetInfoOk() (*map[string]string, bool)` + +GetInfoOk returns a tuple with the Info field if it's non-nil, zero value otherwise +and a boolean to check if the value has been set. + +### SetInfo + +`func (o *PinStatus) SetInfo(v map[string]string)` + +SetInfo sets Info field to given value. + +### HasInfo + +`func (o *PinStatus) HasInfo() bool` + +HasInfo returns a boolean if a field has been set. + + +[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) + + diff --git a/pinning/remote/client/openapi/docs/PinsApi.md b/pinning/remote/client/openapi/docs/PinsApi.md new file mode 100644 index 0000000000..00cff667b6 --- /dev/null +++ b/pinning/remote/client/openapi/docs/PinsApi.md @@ -0,0 +1,367 @@ +# \PinsApi + +All URIs are relative to *https://pinning-service.example.com* + +Method | HTTP request | Description +------------- | ------------- | ------------- +[**PinsGet**](PinsApi.md#PinsGet) | **Get** /pins | List pin objects +[**PinsPost**](PinsApi.md#PinsPost) | **Post** /pins | Add pin object +[**PinsRequestidDelete**](PinsApi.md#PinsRequestidDelete) | **Delete** /pins/{requestid} | Remove pin object +[**PinsRequestidGet**](PinsApi.md#PinsRequestidGet) | **Get** /pins/{requestid} | Get pin object +[**PinsRequestidPost**](PinsApi.md#PinsRequestidPost) | **Post** /pins/{requestid} | Replace pin object + + + +## PinsGet + +> PinResults PinsGet(ctx).Cid(cid).Name(name).Status(status).Before(before).After(after).Limit(limit).Meta(meta).Execute() + +List pin objects + + + +### Example + +```go +package main + +import ( + "context" + "fmt" + "os" + openapiclient "./openapi" +) + +func main() { + cid := []string{"Inner_example"} // []string | Return pin objects responsible for pinning the specified CID(s); be aware that using longer hash functions introduces further constraints on the number of CIDs that will fit under the limit of 2000 characters per URL in browser contexts (optional) + name := "name_example" // string | Return pin objects with names that contain provided value (case-insensitive, partial or full match) (optional) + status := []Status{openapiclient.Status{}} // []Status | Return pin objects for pins with the specified status (optional) + before := Get-Date // time.Time | Return results created (queued) before provided timestamp (optional) + after := Get-Date // time.Time | Return results created (queued) after provided timestamp (optional) + limit := 987 // int32 | Max records to return (optional) (default to 10) + meta := map[string]string{ "Key" = "Value" } // map[string]string | Return pin objects that match specified metadata (optional) + + configuration := openapiclient.NewConfiguration() + api_client := openapiclient.NewAPIClient(configuration) + resp, r, err := api_client.PinsApi.PinsGet(context.Background(), ).Cid(cid).Name(name).Status(status).Before(before).After(after).Limit(limit).Meta(meta).Execute() + if err != nil { + fmt.Fprintf(os.Stderr, "Error when calling `PinsApi.PinsGet``: %v\n", err) + fmt.Fprintf(os.Stderr, "Full HTTP response: %v\n", r) + } + // response from `PinsGet`: PinResults + fmt.Fprintf(os.Stdout, "Response from `PinsApi.PinsGet`: %v\n", resp) +} +``` + +### Path Parameters + + + +### Other Parameters + +Other parameters are passed through a pointer to a apiPinsGetRequest struct via the builder pattern + + +Name | Type | Description | Notes +------------- | ------------- | ------------- | ------------- + **cid** | [**[]string**](string.md) | Return pin objects responsible for pinning the specified CID(s); be aware that using longer hash functions introduces further constraints on the number of CIDs that will fit under the limit of 2000 characters per URL in browser contexts | + **name** | **string** | Return pin objects with names that contain provided value (case-insensitive, partial or full match) | + **status** | [**[]Status**](Status.md) | Return pin objects for pins with the specified status | + **before** | **time.Time** | Return results created (queued) before provided timestamp | + **after** | **time.Time** | Return results created (queued) after provided timestamp | + **limit** | **int32** | Max records to return | [default to 10] + **meta** | [**map[string]string**](string.md) | Return pin objects that match specified metadata | + +### Return type + +[**PinResults**](PinResults.md) + +### Authorization + +[accessToken](../README.md#accessToken) + +### HTTP request headers + +- **Content-Type**: Not defined +- **Accept**: application/json + +[[Back to top]](#) [[Back to API list]](../README.md#documentation-for-api-endpoints) +[[Back to Model list]](../README.md#documentation-for-models) +[[Back to README]](../README.md) + + +## PinsPost + +> PinStatus PinsPost(ctx).Pin(pin).Execute() + +Add pin object + + + +### Example + +```go +package main + +import ( + "context" + "fmt" + "os" + openapiclient "./openapi" +) + +func main() { + pin := openapiclient.Pin{Cid: "Cid_example", Name: "Name_example", Origins: []string{"Origins_example"), Meta: map[string]string{ "Key" = "Value" }} // Pin | + + configuration := openapiclient.NewConfiguration() + api_client := openapiclient.NewAPIClient(configuration) + resp, r, err := api_client.PinsApi.PinsPost(context.Background(), pin).Execute() + if err != nil { + fmt.Fprintf(os.Stderr, "Error when calling `PinsApi.PinsPost``: %v\n", err) + fmt.Fprintf(os.Stderr, "Full HTTP response: %v\n", r) + } + // response from `PinsPost`: PinStatus + fmt.Fprintf(os.Stdout, "Response from `PinsApi.PinsPost`: %v\n", resp) +} +``` + +### Path Parameters + + + +### Other Parameters + +Other parameters are passed through a pointer to a apiPinsPostRequest struct via the builder pattern + + +Name | Type | Description | Notes +------------- | ------------- | ------------- | ------------- + **pin** | [**Pin**](Pin.md) | | + +### Return type + +[**PinStatus**](PinStatus.md) + +### Authorization + +[accessToken](../README.md#accessToken) + +### HTTP request headers + +- **Content-Type**: application/json +- **Accept**: application/json + +[[Back to top]](#) [[Back to API list]](../README.md#documentation-for-api-endpoints) +[[Back to Model list]](../README.md#documentation-for-models) +[[Back to README]](../README.md) + + +## PinsRequestidDelete + +> PinsRequestidDelete(ctx, requestid).Execute() + +Remove pin object + + + +### Example + +```go +package main + +import ( + "context" + "fmt" + "os" + openapiclient "./openapi" +) + +func main() { + requestid := "requestid_example" // string | + + configuration := openapiclient.NewConfiguration() + api_client := openapiclient.NewAPIClient(configuration) + resp, r, err := api_client.PinsApi.PinsRequestidDelete(context.Background(), requestid).Execute() + if err != nil { + fmt.Fprintf(os.Stderr, "Error when calling `PinsApi.PinsRequestidDelete``: %v\n", err) + fmt.Fprintf(os.Stderr, "Full HTTP response: %v\n", r) + } +} +``` + +### Path Parameters + + +Name | Type | Description | Notes +------------- | ------------- | ------------- | ------------- +**ctx** | **context.Context** | context for authentication, logging, cancellation, deadlines, tracing, etc. +**requestid** | **string** | | + +### Other Parameters + +Other parameters are passed through a pointer to a apiPinsRequestidDeleteRequest struct via the builder pattern + + +Name | Type | Description | Notes +------------- | ------------- | ------------- | ------------- + + +### Return type + + (empty response body) + +### Authorization + +[accessToken](../README.md#accessToken) + +### HTTP request headers + +- **Content-Type**: Not defined +- **Accept**: application/json + +[[Back to top]](#) [[Back to API list]](../README.md#documentation-for-api-endpoints) +[[Back to Model list]](../README.md#documentation-for-models) +[[Back to README]](../README.md) + + +## PinsRequestidGet + +> PinStatus PinsRequestidGet(ctx, requestid).Execute() + +Get pin object + + + +### Example + +```go +package main + +import ( + "context" + "fmt" + "os" + openapiclient "./openapi" +) + +func main() { + requestid := "requestid_example" // string | + + configuration := openapiclient.NewConfiguration() + api_client := openapiclient.NewAPIClient(configuration) + resp, r, err := api_client.PinsApi.PinsRequestidGet(context.Background(), requestid).Execute() + if err != nil { + fmt.Fprintf(os.Stderr, "Error when calling `PinsApi.PinsRequestidGet``: %v\n", err) + fmt.Fprintf(os.Stderr, "Full HTTP response: %v\n", r) + } + // response from `PinsRequestidGet`: PinStatus + fmt.Fprintf(os.Stdout, "Response from `PinsApi.PinsRequestidGet`: %v\n", resp) +} +``` + +### Path Parameters + + +Name | Type | Description | Notes +------------- | ------------- | ------------- | ------------- +**ctx** | **context.Context** | context for authentication, logging, cancellation, deadlines, tracing, etc. +**requestid** | **string** | | + +### Other Parameters + +Other parameters are passed through a pointer to a apiPinsRequestidGetRequest struct via the builder pattern + + +Name | Type | Description | Notes +------------- | ------------- | ------------- | ------------- + + +### Return type + +[**PinStatus**](PinStatus.md) + +### Authorization + +[accessToken](../README.md#accessToken) + +### HTTP request headers + +- **Content-Type**: Not defined +- **Accept**: application/json + +[[Back to top]](#) [[Back to API list]](../README.md#documentation-for-api-endpoints) +[[Back to Model list]](../README.md#documentation-for-models) +[[Back to README]](../README.md) + + +## PinsRequestidPost + +> PinStatus PinsRequestidPost(ctx, requestid).Pin(pin).Execute() + +Replace pin object + + + +### Example + +```go +package main + +import ( + "context" + "fmt" + "os" + openapiclient "./openapi" +) + +func main() { + requestid := "requestid_example" // string | + pin := openapiclient.Pin{Cid: "Cid_example", Name: "Name_example", Origins: []string{"Origins_example"), Meta: map[string]string{ "Key" = "Value" }} // Pin | + + configuration := openapiclient.NewConfiguration() + api_client := openapiclient.NewAPIClient(configuration) + resp, r, err := api_client.PinsApi.PinsRequestidPost(context.Background(), requestid, pin).Execute() + if err != nil { + fmt.Fprintf(os.Stderr, "Error when calling `PinsApi.PinsRequestidPost``: %v\n", err) + fmt.Fprintf(os.Stderr, "Full HTTP response: %v\n", r) + } + // response from `PinsRequestidPost`: PinStatus + fmt.Fprintf(os.Stdout, "Response from `PinsApi.PinsRequestidPost`: %v\n", resp) +} +``` + +### Path Parameters + + +Name | Type | Description | Notes +------------- | ------------- | ------------- | ------------- +**ctx** | **context.Context** | context for authentication, logging, cancellation, deadlines, tracing, etc. +**requestid** | **string** | | + +### Other Parameters + +Other parameters are passed through a pointer to a apiPinsRequestidPostRequest struct via the builder pattern + + +Name | Type | Description | Notes +------------- | ------------- | ------------- | ------------- + + **pin** | [**Pin**](Pin.md) | | + +### Return type + +[**PinStatus**](PinStatus.md) + +### Authorization + +[accessToken](../README.md#accessToken) + +### HTTP request headers + +- **Content-Type**: application/json +- **Accept**: application/json + +[[Back to top]](#) [[Back to API list]](../README.md#documentation-for-api-endpoints) +[[Back to Model list]](../README.md#documentation-for-models) +[[Back to README]](../README.md) + diff --git a/pinning/remote/client/openapi/docs/Status.md b/pinning/remote/client/openapi/docs/Status.md new file mode 100644 index 0000000000..01176af11e --- /dev/null +++ b/pinning/remote/client/openapi/docs/Status.md @@ -0,0 +1,11 @@ +# Status + +## Properties + +Name | Type | Description | Notes +------------ | ------------- | ------------- | ------------- + + +[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) + + diff --git a/pinning/remote/client/openapi/model_failure.go b/pinning/remote/client/openapi/model_failure.go new file mode 100644 index 0000000000..ec1fc8268a --- /dev/null +++ b/pinning/remote/client/openapi/model_failure.go @@ -0,0 +1,105 @@ +/* + * IPFS Pinning Service API + * + * ## About this spec The IPFS Pinning Service API is intended to be an implementation-agnostic API: - For use and implementation by pinning service providers - For use in client mode by IPFS nodes and GUI-based applications > **Note**: while ready for implementation, this spec is still a work in progress! 🏗️ **Your input and feedback are welcome and valuable as we develop this API spec. Please join the design discussion at [github.com/ipfs/pinning-services-api-spec](https://github.com/ipfs/pinning-services-api-spec).** # Schemas This section describes the most important object types and conventions. A full list of fields and schemas can be found in the `schemas` section of the [YAML file](https://github.com/ipfs/pinning-services-api-spec/blob/master/ipfs-pinning-service.yaml). ## Identifiers ### cid [Content Identifier (CID)](https://docs.ipfs.io/concepts/content-addressing/) points at the root of a DAG that is pinned recursively. ### requestid Unique identifier of a pin request. When a pin is created, the service responds with unique `requestid` that can be later used for pin removal. When the same `cid` is pinned again, a different `requestid` is returned to differentiate between those pin requests. Service implementation should use UUID, `hash(accessToken,Pin,PinStatus.created)`, or any other opaque identifier that provides equally strong protection against race conditions. ## Objects ### Pin object ![pin object](https://bafybeideck2fchyxna4wqwc2mo67yriokehw3yujboc5redjdaajrk2fjq.ipfs.dweb.link/pin.png) The `Pin` object is a representation of a pin request. It includes the `cid` of data to be pinned, as well as optional metadata in `name`, `origins`, and `meta`. ### Pin status response ![pin status response object](https://bafybeideck2fchyxna4wqwc2mo67yriokehw3yujboc5redjdaajrk2fjq.ipfs.dweb.link/pinstatus.png) The `PinStatus` object is a representation of the current state of a pinning operation. It includes the original `pin` object, along with the current `status` and globally unique `requestid` of the entire pinning request, which can be used for future status checks and management. Addresses in the `delegates` array are peers delegated by the pinning service for facilitating direct file transfers (more details in the provider hints section). Any additional vendor-specific information is returned in optional `info`. ## The pin lifecycle ![pinning service objects and lifecycle](https://bafybeideck2fchyxna4wqwc2mo67yriokehw3yujboc5redjdaajrk2fjq.ipfs.dweb.link/lifecycle.png) ### Creating a new pin object The user sends a `Pin` object to `POST /pins` and receives a `PinStatus` response: - `requestid` in `PinStatus` is the identifier of the pin operation, which can can be used for checking status, and removing the pin in the future - `status` in `PinStatus` indicates the current state of a pin ### Checking status of in-progress pinning `status` (in `PinStatus`) may indicate a pending state (`queued` or `pinning`). This means the data behind `Pin.cid` was not found on the pinning service and is being fetched from the IPFS network at large, which may take time. In this case, the user can periodically check pinning progress via `GET /pins/{requestid}` until pinning is successful, or the user decides to remove the pending pin. ### Replacing an existing pin object The user can replace an existing pin object via `POST /pins/{requestid}`. This is a shortcut for removing a pin object identified by `requestid` and creating a new one in a single API call that protects against undesired garbage collection of blocks common to both pins. Useful when updating a pin representing a huge dataset where most of blocks did not change. The new pin object `requestid` is returned in the `PinStatus` response. The old pin object is deleted automatically. ### Removing a pin object A pin object can be removed via `DELETE /pins/{requestid}`. ## Provider hints Pinning of new data can be accelerated by providing a list of known data sources in `Pin.origins`, and connecting at least one of them to pinning service nodes at `PinStatus.delegates`. The most common scenario is a client putting its own IPFS node's multiaddrs in `Pin.origins`, and then directly connecting to every multiaddr returned by a pinning service in `PinStatus.delegates` to initiate transfer. This ensures data transfer starts immediately (without waiting for provider discovery over DHT), and direct dial from a client works around peer routing issues in restrictive network topologies such as NATs. ## Custom metadata Pinning services are encouraged to add support for additional features by leveraging the optional `Pin.meta` and `PinStatus.info` fields. While these attributes can be application- or vendor-specific, we encourage the community at large to leverage these attributes as a sandbox to come up with conventions that could become part of future revisions of this API. ### Pin metadata String keys and values passed in `Pin.meta` are persisted with the pin object. Potential uses: - `Pin.meta[app_id]`: Attaching a unique identifier to pins created by an app enables filtering pins per app via `?meta={\"app_id\":}` - `Pin.meta[vendor_policy]`: Vendor-specific policy (for example: which region to use, how many copies to keep) Note that it is OK for a client to omit or ignore these optional attributes; doing so should not impact the basic pinning functionality. ### Pin status info Additional `PinStatus.info` can be returned by pinning service. Potential uses: - `PinStatus.info[status_details]`: more info about the current status (queue position, percentage of transferred data, summary of where data is stored, etc); when `PinStatus.status=failed`, it could provide a reason why a pin operation failed (e.g. lack of funds, DAG too big, etc.) - `PinStatus.info[dag_size]`: the size of pinned data, along with DAG overhead - `PinStatus.info[raw_size]`: the size of data without DAG overhead (eg. unixfs) - `PinStatus.info[pinned_until]`: if vendor supports time-bound pins, this could indicate when the pin will expire # Pagination and filtering Pin objects can be listed by executing `GET /pins` with optional parameters: - When no filters are provided, the endpoint will return a small batch of the 10 most recently created items, from the latest to the oldest. - The number of returned items can be adjusted with the `limit` parameter (implicit default is 10). - If the value in `PinResults.count` is bigger than the length of `PinResults.results`, the client can infer there are more results that can be queried. - To read more items, pass the `before` filter with the timestamp from `PinStatus.created` found in the oldest item in the current batch of results. Repeat to read all results. - Returned results can be fine-tuned by applying optional `after`, `cid`, `name`, `status`, or `meta` filters. > **Note**: pagination by the `created` timestamp requires each value to be globally unique. Any future considerations to add support for bulk creation must account for this. + * + * API version: 0.1.1 + * Generated by: OpenAPI Generator (https://openapi-generator.tech) + */ + +package openapi + +import ( + "encoding/json" +) + +// Failure Response for a failed request +type Failure struct { + Error FailureError `json:"error"` +} + +// NewFailure instantiates a new Failure object +// This constructor will assign default values to properties that have it defined, +// and makes sure properties required by API are set, but the set of arguments +// will change when the set of required properties is changed +func NewFailure(error_ FailureError) *Failure { + this := Failure{} + this.Error = error_ + return &this +} + +// NewFailureWithDefaults instantiates a new Failure object +// This constructor will only assign default values to properties that have it defined, +// but it doesn't guarantee that properties required by API are set +func NewFailureWithDefaults() *Failure { + this := Failure{} + return &this +} + +// GetError returns the Error field value +func (o *Failure) GetError() FailureError { + if o == nil { + var ret FailureError + return ret + } + + return o.Error +} + +// GetErrorOk returns a tuple with the Error field value +// and a boolean to check if the value has been set. +func (o *Failure) GetErrorOk() (*FailureError, bool) { + if o == nil { + return nil, false + } + return &o.Error, true +} + +// SetError sets field value +func (o *Failure) SetError(v FailureError) { + o.Error = v +} + +func (o Failure) MarshalJSON() ([]byte, error) { + toSerialize := map[string]interface{}{} + if true { + toSerialize["error"] = o.Error + } + return json.Marshal(toSerialize) +} + +type NullableFailure struct { + value *Failure + isSet bool +} + +func (v NullableFailure) Get() *Failure { + return v.value +} + +func (v *NullableFailure) Set(val *Failure) { + v.value = val + v.isSet = true +} + +func (v NullableFailure) IsSet() bool { + return v.isSet +} + +func (v *NullableFailure) Unset() { + v.value = nil + v.isSet = false +} + +func NewNullableFailure(val *Failure) *NullableFailure { + return &NullableFailure{value: val, isSet: true} +} + +func (v NullableFailure) MarshalJSON() ([]byte, error) { + return json.Marshal(v.value) +} + +func (v *NullableFailure) UnmarshalJSON(src []byte) error { + v.isSet = true + return json.Unmarshal(src, &v.value) +} diff --git a/pinning/remote/client/openapi/model_failure_error.go b/pinning/remote/client/openapi/model_failure_error.go new file mode 100644 index 0000000000..4f38acdf96 --- /dev/null +++ b/pinning/remote/client/openapi/model_failure_error.go @@ -0,0 +1,143 @@ +/* + * IPFS Pinning Service API + * + * ## About this spec The IPFS Pinning Service API is intended to be an implementation-agnostic API: - For use and implementation by pinning service providers - For use in client mode by IPFS nodes and GUI-based applications > **Note**: while ready for implementation, this spec is still a work in progress! 🏗️ **Your input and feedback are welcome and valuable as we develop this API spec. Please join the design discussion at [github.com/ipfs/pinning-services-api-spec](https://github.com/ipfs/pinning-services-api-spec).** # Schemas This section describes the most important object types and conventions. A full list of fields and schemas can be found in the `schemas` section of the [YAML file](https://github.com/ipfs/pinning-services-api-spec/blob/master/ipfs-pinning-service.yaml). ## Identifiers ### cid [Content Identifier (CID)](https://docs.ipfs.io/concepts/content-addressing/) points at the root of a DAG that is pinned recursively. ### requestid Unique identifier of a pin request. When a pin is created, the service responds with unique `requestid` that can be later used for pin removal. When the same `cid` is pinned again, a different `requestid` is returned to differentiate between those pin requests. Service implementation should use UUID, `hash(accessToken,Pin,PinStatus.created)`, or any other opaque identifier that provides equally strong protection against race conditions. ## Objects ### Pin object ![pin object](https://bafybeideck2fchyxna4wqwc2mo67yriokehw3yujboc5redjdaajrk2fjq.ipfs.dweb.link/pin.png) The `Pin` object is a representation of a pin request. It includes the `cid` of data to be pinned, as well as optional metadata in `name`, `origins`, and `meta`. ### Pin status response ![pin status response object](https://bafybeideck2fchyxna4wqwc2mo67yriokehw3yujboc5redjdaajrk2fjq.ipfs.dweb.link/pinstatus.png) The `PinStatus` object is a representation of the current state of a pinning operation. It includes the original `pin` object, along with the current `status` and globally unique `requestid` of the entire pinning request, which can be used for future status checks and management. Addresses in the `delegates` array are peers delegated by the pinning service for facilitating direct file transfers (more details in the provider hints section). Any additional vendor-specific information is returned in optional `info`. ## The pin lifecycle ![pinning service objects and lifecycle](https://bafybeideck2fchyxna4wqwc2mo67yriokehw3yujboc5redjdaajrk2fjq.ipfs.dweb.link/lifecycle.png) ### Creating a new pin object The user sends a `Pin` object to `POST /pins` and receives a `PinStatus` response: - `requestid` in `PinStatus` is the identifier of the pin operation, which can can be used for checking status, and removing the pin in the future - `status` in `PinStatus` indicates the current state of a pin ### Checking status of in-progress pinning `status` (in `PinStatus`) may indicate a pending state (`queued` or `pinning`). This means the data behind `Pin.cid` was not found on the pinning service and is being fetched from the IPFS network at large, which may take time. In this case, the user can periodically check pinning progress via `GET /pins/{requestid}` until pinning is successful, or the user decides to remove the pending pin. ### Replacing an existing pin object The user can replace an existing pin object via `POST /pins/{requestid}`. This is a shortcut for removing a pin object identified by `requestid` and creating a new one in a single API call that protects against undesired garbage collection of blocks common to both pins. Useful when updating a pin representing a huge dataset where most of blocks did not change. The new pin object `requestid` is returned in the `PinStatus` response. The old pin object is deleted automatically. ### Removing a pin object A pin object can be removed via `DELETE /pins/{requestid}`. ## Provider hints Pinning of new data can be accelerated by providing a list of known data sources in `Pin.origins`, and connecting at least one of them to pinning service nodes at `PinStatus.delegates`. The most common scenario is a client putting its own IPFS node's multiaddrs in `Pin.origins`, and then directly connecting to every multiaddr returned by a pinning service in `PinStatus.delegates` to initiate transfer. This ensures data transfer starts immediately (without waiting for provider discovery over DHT), and direct dial from a client works around peer routing issues in restrictive network topologies such as NATs. ## Custom metadata Pinning services are encouraged to add support for additional features by leveraging the optional `Pin.meta` and `PinStatus.info` fields. While these attributes can be application- or vendor-specific, we encourage the community at large to leverage these attributes as a sandbox to come up with conventions that could become part of future revisions of this API. ### Pin metadata String keys and values passed in `Pin.meta` are persisted with the pin object. Potential uses: - `Pin.meta[app_id]`: Attaching a unique identifier to pins created by an app enables filtering pins per app via `?meta={\"app_id\":}` - `Pin.meta[vendor_policy]`: Vendor-specific policy (for example: which region to use, how many copies to keep) Note that it is OK for a client to omit or ignore these optional attributes; doing so should not impact the basic pinning functionality. ### Pin status info Additional `PinStatus.info` can be returned by pinning service. Potential uses: - `PinStatus.info[status_details]`: more info about the current status (queue position, percentage of transferred data, summary of where data is stored, etc); when `PinStatus.status=failed`, it could provide a reason why a pin operation failed (e.g. lack of funds, DAG too big, etc.) - `PinStatus.info[dag_size]`: the size of pinned data, along with DAG overhead - `PinStatus.info[raw_size]`: the size of data without DAG overhead (eg. unixfs) - `PinStatus.info[pinned_until]`: if vendor supports time-bound pins, this could indicate when the pin will expire # Pagination and filtering Pin objects can be listed by executing `GET /pins` with optional parameters: - When no filters are provided, the endpoint will return a small batch of the 10 most recently created items, from the latest to the oldest. - The number of returned items can be adjusted with the `limit` parameter (implicit default is 10). - If the value in `PinResults.count` is bigger than the length of `PinResults.results`, the client can infer there are more results that can be queried. - To read more items, pass the `before` filter with the timestamp from `PinStatus.created` found in the oldest item in the current batch of results. Repeat to read all results. - Returned results can be fine-tuned by applying optional `after`, `cid`, `name`, `status`, or `meta` filters. > **Note**: pagination by the `created` timestamp requires each value to be globally unique. Any future considerations to add support for bulk creation must account for this. + * + * API version: 0.1.1 + * Generated by: OpenAPI Generator (https://openapi-generator.tech) + */ + +package openapi + +import ( + "encoding/json" +) + +// FailureError struct for FailureError +type FailureError struct { + // Mandatory string identifying the type of error + Reason string `json:"reason"` + // Optional, longer description of the error; may include UUID of transaction for support, links to documentation etc + Details *string `json:"details,omitempty"` +} + +// NewFailureError instantiates a new FailureError object +// This constructor will assign default values to properties that have it defined, +// and makes sure properties required by API are set, but the set of arguments +// will change when the set of required properties is changed +func NewFailureError(reason string) *FailureError { + this := FailureError{} + this.Reason = reason + return &this +} + +// NewFailureErrorWithDefaults instantiates a new FailureError object +// This constructor will only assign default values to properties that have it defined, +// but it doesn't guarantee that properties required by API are set +func NewFailureErrorWithDefaults() *FailureError { + this := FailureError{} + return &this +} + +// GetReason returns the Reason field value +func (o *FailureError) GetReason() string { + if o == nil { + var ret string + return ret + } + + return o.Reason +} + +// GetReasonOk returns a tuple with the Reason field value +// and a boolean to check if the value has been set. +func (o *FailureError) GetReasonOk() (*string, bool) { + if o == nil { + return nil, false + } + return &o.Reason, true +} + +// SetReason sets field value +func (o *FailureError) SetReason(v string) { + o.Reason = v +} + +// GetDetails returns the Details field value if set, zero value otherwise. +func (o *FailureError) GetDetails() string { + if o == nil || o.Details == nil { + var ret string + return ret + } + return *o.Details +} + +// GetDetailsOk returns a tuple with the Details field value if set, nil otherwise +// and a boolean to check if the value has been set. +func (o *FailureError) GetDetailsOk() (*string, bool) { + if o == nil || o.Details == nil { + return nil, false + } + return o.Details, true +} + +// HasDetails returns a boolean if a field has been set. +func (o *FailureError) HasDetails() bool { + if o != nil && o.Details != nil { + return true + } + + return false +} + +// SetDetails gets a reference to the given string and assigns it to the Details field. +func (o *FailureError) SetDetails(v string) { + o.Details = &v +} + +func (o FailureError) MarshalJSON() ([]byte, error) { + toSerialize := map[string]interface{}{} + if true { + toSerialize["reason"] = o.Reason + } + if o.Details != nil { + toSerialize["details"] = o.Details + } + return json.Marshal(toSerialize) +} + +type NullableFailureError struct { + value *FailureError + isSet bool +} + +func (v NullableFailureError) Get() *FailureError { + return v.value +} + +func (v *NullableFailureError) Set(val *FailureError) { + v.value = val + v.isSet = true +} + +func (v NullableFailureError) IsSet() bool { + return v.isSet +} + +func (v *NullableFailureError) Unset() { + v.value = nil + v.isSet = false +} + +func NewNullableFailureError(val *FailureError) *NullableFailureError { + return &NullableFailureError{value: val, isSet: true} +} + +func (v NullableFailureError) MarshalJSON() ([]byte, error) { + return json.Marshal(v.value) +} + +func (v *NullableFailureError) UnmarshalJSON(src []byte) error { + v.isSet = true + return json.Unmarshal(src, &v.value) +} diff --git a/pinning/remote/client/openapi/model_pin.go b/pinning/remote/client/openapi/model_pin.go new file mode 100644 index 0000000000..0152d43bbf --- /dev/null +++ b/pinning/remote/client/openapi/model_pin.go @@ -0,0 +1,217 @@ +/* + * IPFS Pinning Service API + * + * ## About this spec The IPFS Pinning Service API is intended to be an implementation-agnostic API: - For use and implementation by pinning service providers - For use in client mode by IPFS nodes and GUI-based applications > **Note**: while ready for implementation, this spec is still a work in progress! 🏗️ **Your input and feedback are welcome and valuable as we develop this API spec. Please join the design discussion at [github.com/ipfs/pinning-services-api-spec](https://github.com/ipfs/pinning-services-api-spec).** # Schemas This section describes the most important object types and conventions. A full list of fields and schemas can be found in the `schemas` section of the [YAML file](https://github.com/ipfs/pinning-services-api-spec/blob/master/ipfs-pinning-service.yaml). ## Identifiers ### cid [Content Identifier (CID)](https://docs.ipfs.io/concepts/content-addressing/) points at the root of a DAG that is pinned recursively. ### requestid Unique identifier of a pin request. When a pin is created, the service responds with unique `requestid` that can be later used for pin removal. When the same `cid` is pinned again, a different `requestid` is returned to differentiate between those pin requests. Service implementation should use UUID, `hash(accessToken,Pin,PinStatus.created)`, or any other opaque identifier that provides equally strong protection against race conditions. ## Objects ### Pin object ![pin object](https://bafybeideck2fchyxna4wqwc2mo67yriokehw3yujboc5redjdaajrk2fjq.ipfs.dweb.link/pin.png) The `Pin` object is a representation of a pin request. It includes the `cid` of data to be pinned, as well as optional metadata in `name`, `origins`, and `meta`. ### Pin status response ![pin status response object](https://bafybeideck2fchyxna4wqwc2mo67yriokehw3yujboc5redjdaajrk2fjq.ipfs.dweb.link/pinstatus.png) The `PinStatus` object is a representation of the current state of a pinning operation. It includes the original `pin` object, along with the current `status` and globally unique `requestid` of the entire pinning request, which can be used for future status checks and management. Addresses in the `delegates` array are peers delegated by the pinning service for facilitating direct file transfers (more details in the provider hints section). Any additional vendor-specific information is returned in optional `info`. ## The pin lifecycle ![pinning service objects and lifecycle](https://bafybeideck2fchyxna4wqwc2mo67yriokehw3yujboc5redjdaajrk2fjq.ipfs.dweb.link/lifecycle.png) ### Creating a new pin object The user sends a `Pin` object to `POST /pins` and receives a `PinStatus` response: - `requestid` in `PinStatus` is the identifier of the pin operation, which can can be used for checking status, and removing the pin in the future - `status` in `PinStatus` indicates the current state of a pin ### Checking status of in-progress pinning `status` (in `PinStatus`) may indicate a pending state (`queued` or `pinning`). This means the data behind `Pin.cid` was not found on the pinning service and is being fetched from the IPFS network at large, which may take time. In this case, the user can periodically check pinning progress via `GET /pins/{requestid}` until pinning is successful, or the user decides to remove the pending pin. ### Replacing an existing pin object The user can replace an existing pin object via `POST /pins/{requestid}`. This is a shortcut for removing a pin object identified by `requestid` and creating a new one in a single API call that protects against undesired garbage collection of blocks common to both pins. Useful when updating a pin representing a huge dataset where most of blocks did not change. The new pin object `requestid` is returned in the `PinStatus` response. The old pin object is deleted automatically. ### Removing a pin object A pin object can be removed via `DELETE /pins/{requestid}`. ## Provider hints Pinning of new data can be accelerated by providing a list of known data sources in `Pin.origins`, and connecting at least one of them to pinning service nodes at `PinStatus.delegates`. The most common scenario is a client putting its own IPFS node's multiaddrs in `Pin.origins`, and then directly connecting to every multiaddr returned by a pinning service in `PinStatus.delegates` to initiate transfer. This ensures data transfer starts immediately (without waiting for provider discovery over DHT), and direct dial from a client works around peer routing issues in restrictive network topologies such as NATs. ## Custom metadata Pinning services are encouraged to add support for additional features by leveraging the optional `Pin.meta` and `PinStatus.info` fields. While these attributes can be application- or vendor-specific, we encourage the community at large to leverage these attributes as a sandbox to come up with conventions that could become part of future revisions of this API. ### Pin metadata String keys and values passed in `Pin.meta` are persisted with the pin object. Potential uses: - `Pin.meta[app_id]`: Attaching a unique identifier to pins created by an app enables filtering pins per app via `?meta={\"app_id\":}` - `Pin.meta[vendor_policy]`: Vendor-specific policy (for example: which region to use, how many copies to keep) Note that it is OK for a client to omit or ignore these optional attributes; doing so should not impact the basic pinning functionality. ### Pin status info Additional `PinStatus.info` can be returned by pinning service. Potential uses: - `PinStatus.info[status_details]`: more info about the current status (queue position, percentage of transferred data, summary of where data is stored, etc); when `PinStatus.status=failed`, it could provide a reason why a pin operation failed (e.g. lack of funds, DAG too big, etc.) - `PinStatus.info[dag_size]`: the size of pinned data, along with DAG overhead - `PinStatus.info[raw_size]`: the size of data without DAG overhead (eg. unixfs) - `PinStatus.info[pinned_until]`: if vendor supports time-bound pins, this could indicate when the pin will expire # Pagination and filtering Pin objects can be listed by executing `GET /pins` with optional parameters: - When no filters are provided, the endpoint will return a small batch of the 10 most recently created items, from the latest to the oldest. - The number of returned items can be adjusted with the `limit` parameter (implicit default is 10). - If the value in `PinResults.count` is bigger than the length of `PinResults.results`, the client can infer there are more results that can be queried. - To read more items, pass the `before` filter with the timestamp from `PinStatus.created` found in the oldest item in the current batch of results. Repeat to read all results. - Returned results can be fine-tuned by applying optional `after`, `cid`, `name`, `status`, or `meta` filters. > **Note**: pagination by the `created` timestamp requires each value to be globally unique. Any future considerations to add support for bulk creation must account for this. + * + * API version: 0.1.1 + * Generated by: OpenAPI Generator (https://openapi-generator.tech) + */ + +package openapi + +import ( + "encoding/json" +) + +// Pin Pin object +type Pin struct { + // Content Identifier (CID) to be pinned recursively + Cid string `json:"cid"` + // Optional name for pinned data; can be used for lookups later + Name *string `json:"name,omitempty"` + // Optional list of multiaddrs known to provide the data + Origins *[]string `json:"origins,omitempty"` + // Optional metadata for pin object + Meta *map[string]string `json:"meta,omitempty"` +} + +// NewPin instantiates a new Pin object +// This constructor will assign default values to properties that have it defined, +// and makes sure properties required by API are set, but the set of arguments +// will change when the set of required properties is changed +func NewPin(cid string) *Pin { + this := Pin{} + this.Cid = cid + return &this +} + +// NewPinWithDefaults instantiates a new Pin object +// This constructor will only assign default values to properties that have it defined, +// but it doesn't guarantee that properties required by API are set +func NewPinWithDefaults() *Pin { + this := Pin{} + return &this +} + +// GetCid returns the Cid field value +func (o *Pin) GetCid() string { + if o == nil { + var ret string + return ret + } + + return o.Cid +} + +// GetCidOk returns a tuple with the Cid field value +// and a boolean to check if the value has been set. +func (o *Pin) GetCidOk() (*string, bool) { + if o == nil { + return nil, false + } + return &o.Cid, true +} + +// SetCid sets field value +func (o *Pin) SetCid(v string) { + o.Cid = v +} + +// GetName returns the Name field value if set, zero value otherwise. +func (o *Pin) GetName() string { + if o == nil || o.Name == nil { + var ret string + return ret + } + return *o.Name +} + +// GetNameOk returns a tuple with the Name field value if set, nil otherwise +// and a boolean to check if the value has been set. +func (o *Pin) GetNameOk() (*string, bool) { + if o == nil || o.Name == nil { + return nil, false + } + return o.Name, true +} + +// HasName returns a boolean if a field has been set. +func (o *Pin) HasName() bool { + if o != nil && o.Name != nil { + return true + } + + return false +} + +// SetName gets a reference to the given string and assigns it to the Name field. +func (o *Pin) SetName(v string) { + o.Name = &v +} + +// GetOrigins returns the Origins field value if set, zero value otherwise. +func (o *Pin) GetOrigins() []string { + if o == nil || o.Origins == nil { + var ret []string + return ret + } + return *o.Origins +} + +// GetOriginsOk returns a tuple with the Origins field value if set, nil otherwise +// and a boolean to check if the value has been set. +func (o *Pin) GetOriginsOk() (*[]string, bool) { + if o == nil || o.Origins == nil { + return nil, false + } + return o.Origins, true +} + +// HasOrigins returns a boolean if a field has been set. +func (o *Pin) HasOrigins() bool { + if o != nil && o.Origins != nil { + return true + } + + return false +} + +// SetOrigins gets a reference to the given []string and assigns it to the Origins field. +func (o *Pin) SetOrigins(v []string) { + o.Origins = &v +} + +// GetMeta returns the Meta field value if set, zero value otherwise. +func (o *Pin) GetMeta() map[string]string { + if o == nil || o.Meta == nil { + var ret map[string]string + return ret + } + return *o.Meta +} + +// GetMetaOk returns a tuple with the Meta field value if set, nil otherwise +// and a boolean to check if the value has been set. +func (o *Pin) GetMetaOk() (*map[string]string, bool) { + if o == nil || o.Meta == nil { + return nil, false + } + return o.Meta, true +} + +// HasMeta returns a boolean if a field has been set. +func (o *Pin) HasMeta() bool { + if o != nil && o.Meta != nil { + return true + } + + return false +} + +// SetMeta gets a reference to the given map[string]string and assigns it to the Meta field. +func (o *Pin) SetMeta(v map[string]string) { + o.Meta = &v +} + +func (o Pin) MarshalJSON() ([]byte, error) { + toSerialize := map[string]interface{}{} + if true { + toSerialize["cid"] = o.Cid + } + if o.Name != nil { + toSerialize["name"] = o.Name + } + if o.Origins != nil { + toSerialize["origins"] = o.Origins + } + if o.Meta != nil { + toSerialize["meta"] = o.Meta + } + return json.Marshal(toSerialize) +} + +type NullablePin struct { + value *Pin + isSet bool +} + +func (v NullablePin) Get() *Pin { + return v.value +} + +func (v *NullablePin) Set(val *Pin) { + v.value = val + v.isSet = true +} + +func (v NullablePin) IsSet() bool { + return v.isSet +} + +func (v *NullablePin) Unset() { + v.value = nil + v.isSet = false +} + +func NewNullablePin(val *Pin) *NullablePin { + return &NullablePin{value: val, isSet: true} +} + +func (v NullablePin) MarshalJSON() ([]byte, error) { + return json.Marshal(v.value) +} + +func (v *NullablePin) UnmarshalJSON(src []byte) error { + v.isSet = true + return json.Unmarshal(src, &v.value) +} diff --git a/pinning/remote/client/openapi/model_pin_results.go b/pinning/remote/client/openapi/model_pin_results.go new file mode 100644 index 0000000000..eacb5e0212 --- /dev/null +++ b/pinning/remote/client/openapi/model_pin_results.go @@ -0,0 +1,136 @@ +/* + * IPFS Pinning Service API + * + * ## About this spec The IPFS Pinning Service API is intended to be an implementation-agnostic API: - For use and implementation by pinning service providers - For use in client mode by IPFS nodes and GUI-based applications > **Note**: while ready for implementation, this spec is still a work in progress! 🏗️ **Your input and feedback are welcome and valuable as we develop this API spec. Please join the design discussion at [github.com/ipfs/pinning-services-api-spec](https://github.com/ipfs/pinning-services-api-spec).** # Schemas This section describes the most important object types and conventions. A full list of fields and schemas can be found in the `schemas` section of the [YAML file](https://github.com/ipfs/pinning-services-api-spec/blob/master/ipfs-pinning-service.yaml). ## Identifiers ### cid [Content Identifier (CID)](https://docs.ipfs.io/concepts/content-addressing/) points at the root of a DAG that is pinned recursively. ### requestid Unique identifier of a pin request. When a pin is created, the service responds with unique `requestid` that can be later used for pin removal. When the same `cid` is pinned again, a different `requestid` is returned to differentiate between those pin requests. Service implementation should use UUID, `hash(accessToken,Pin,PinStatus.created)`, or any other opaque identifier that provides equally strong protection against race conditions. ## Objects ### Pin object ![pin object](https://bafybeideck2fchyxna4wqwc2mo67yriokehw3yujboc5redjdaajrk2fjq.ipfs.dweb.link/pin.png) The `Pin` object is a representation of a pin request. It includes the `cid` of data to be pinned, as well as optional metadata in `name`, `origins`, and `meta`. ### Pin status response ![pin status response object](https://bafybeideck2fchyxna4wqwc2mo67yriokehw3yujboc5redjdaajrk2fjq.ipfs.dweb.link/pinstatus.png) The `PinStatus` object is a representation of the current state of a pinning operation. It includes the original `pin` object, along with the current `status` and globally unique `requestid` of the entire pinning request, which can be used for future status checks and management. Addresses in the `delegates` array are peers delegated by the pinning service for facilitating direct file transfers (more details in the provider hints section). Any additional vendor-specific information is returned in optional `info`. ## The pin lifecycle ![pinning service objects and lifecycle](https://bafybeideck2fchyxna4wqwc2mo67yriokehw3yujboc5redjdaajrk2fjq.ipfs.dweb.link/lifecycle.png) ### Creating a new pin object The user sends a `Pin` object to `POST /pins` and receives a `PinStatus` response: - `requestid` in `PinStatus` is the identifier of the pin operation, which can can be used for checking status, and removing the pin in the future - `status` in `PinStatus` indicates the current state of a pin ### Checking status of in-progress pinning `status` (in `PinStatus`) may indicate a pending state (`queued` or `pinning`). This means the data behind `Pin.cid` was not found on the pinning service and is being fetched from the IPFS network at large, which may take time. In this case, the user can periodically check pinning progress via `GET /pins/{requestid}` until pinning is successful, or the user decides to remove the pending pin. ### Replacing an existing pin object The user can replace an existing pin object via `POST /pins/{requestid}`. This is a shortcut for removing a pin object identified by `requestid` and creating a new one in a single API call that protects against undesired garbage collection of blocks common to both pins. Useful when updating a pin representing a huge dataset where most of blocks did not change. The new pin object `requestid` is returned in the `PinStatus` response. The old pin object is deleted automatically. ### Removing a pin object A pin object can be removed via `DELETE /pins/{requestid}`. ## Provider hints Pinning of new data can be accelerated by providing a list of known data sources in `Pin.origins`, and connecting at least one of them to pinning service nodes at `PinStatus.delegates`. The most common scenario is a client putting its own IPFS node's multiaddrs in `Pin.origins`, and then directly connecting to every multiaddr returned by a pinning service in `PinStatus.delegates` to initiate transfer. This ensures data transfer starts immediately (without waiting for provider discovery over DHT), and direct dial from a client works around peer routing issues in restrictive network topologies such as NATs. ## Custom metadata Pinning services are encouraged to add support for additional features by leveraging the optional `Pin.meta` and `PinStatus.info` fields. While these attributes can be application- or vendor-specific, we encourage the community at large to leverage these attributes as a sandbox to come up with conventions that could become part of future revisions of this API. ### Pin metadata String keys and values passed in `Pin.meta` are persisted with the pin object. Potential uses: - `Pin.meta[app_id]`: Attaching a unique identifier to pins created by an app enables filtering pins per app via `?meta={\"app_id\":}` - `Pin.meta[vendor_policy]`: Vendor-specific policy (for example: which region to use, how many copies to keep) Note that it is OK for a client to omit or ignore these optional attributes; doing so should not impact the basic pinning functionality. ### Pin status info Additional `PinStatus.info` can be returned by pinning service. Potential uses: - `PinStatus.info[status_details]`: more info about the current status (queue position, percentage of transferred data, summary of where data is stored, etc); when `PinStatus.status=failed`, it could provide a reason why a pin operation failed (e.g. lack of funds, DAG too big, etc.) - `PinStatus.info[dag_size]`: the size of pinned data, along with DAG overhead - `PinStatus.info[raw_size]`: the size of data without DAG overhead (eg. unixfs) - `PinStatus.info[pinned_until]`: if vendor supports time-bound pins, this could indicate when the pin will expire # Pagination and filtering Pin objects can be listed by executing `GET /pins` with optional parameters: - When no filters are provided, the endpoint will return a small batch of the 10 most recently created items, from the latest to the oldest. - The number of returned items can be adjusted with the `limit` parameter (implicit default is 10). - If the value in `PinResults.count` is bigger than the length of `PinResults.results`, the client can infer there are more results that can be queried. - To read more items, pass the `before` filter with the timestamp from `PinStatus.created` found in the oldest item in the current batch of results. Repeat to read all results. - Returned results can be fine-tuned by applying optional `after`, `cid`, `name`, `status`, or `meta` filters. > **Note**: pagination by the `created` timestamp requires each value to be globally unique. Any future considerations to add support for bulk creation must account for this. + * + * API version: 0.1.1 + * Generated by: OpenAPI Generator (https://openapi-generator.tech) + */ + +package openapi + +import ( + "encoding/json" +) + +// PinResults Response used for listing pin objects matching request +type PinResults struct { + // The total number of pin objects that exist for passed query filters + Count int32 `json:"count"` + // An array of PinStatus results + Results []PinStatus `json:"results"` +} + +// NewPinResults instantiates a new PinResults object +// This constructor will assign default values to properties that have it defined, +// and makes sure properties required by API are set, but the set of arguments +// will change when the set of required properties is changed +func NewPinResults(count int32, results []PinStatus) *PinResults { + this := PinResults{} + this.Count = count + this.Results = results + return &this +} + +// NewPinResultsWithDefaults instantiates a new PinResults object +// This constructor will only assign default values to properties that have it defined, +// but it doesn't guarantee that properties required by API are set +func NewPinResultsWithDefaults() *PinResults { + this := PinResults{} + return &this +} + +// GetCount returns the Count field value +func (o *PinResults) GetCount() int32 { + if o == nil { + var ret int32 + return ret + } + + return o.Count +} + +// GetCountOk returns a tuple with the Count field value +// and a boolean to check if the value has been set. +func (o *PinResults) GetCountOk() (*int32, bool) { + if o == nil { + return nil, false + } + return &o.Count, true +} + +// SetCount sets field value +func (o *PinResults) SetCount(v int32) { + o.Count = v +} + +// GetResults returns the Results field value +func (o *PinResults) GetResults() []PinStatus { + if o == nil { + var ret []PinStatus + return ret + } + + return o.Results +} + +// GetResultsOk returns a tuple with the Results field value +// and a boolean to check if the value has been set. +func (o *PinResults) GetResultsOk() (*[]PinStatus, bool) { + if o == nil { + return nil, false + } + return &o.Results, true +} + +// SetResults sets field value +func (o *PinResults) SetResults(v []PinStatus) { + o.Results = v +} + +func (o PinResults) MarshalJSON() ([]byte, error) { + toSerialize := map[string]interface{}{} + if true { + toSerialize["count"] = o.Count + } + if true { + toSerialize["results"] = o.Results + } + return json.Marshal(toSerialize) +} + +type NullablePinResults struct { + value *PinResults + isSet bool +} + +func (v NullablePinResults) Get() *PinResults { + return v.value +} + +func (v *NullablePinResults) Set(val *PinResults) { + v.value = val + v.isSet = true +} + +func (v NullablePinResults) IsSet() bool { + return v.isSet +} + +func (v *NullablePinResults) Unset() { + v.value = nil + v.isSet = false +} + +func NewNullablePinResults(val *PinResults) *NullablePinResults { + return &NullablePinResults{value: val, isSet: true} +} + +func (v NullablePinResults) MarshalJSON() ([]byte, error) { + return json.Marshal(v.value) +} + +func (v *NullablePinResults) UnmarshalJSON(src []byte) error { + v.isSet = true + return json.Unmarshal(src, &v.value) +} diff --git a/pinning/remote/client/openapi/model_pin_status.go b/pinning/remote/client/openapi/model_pin_status.go new file mode 100644 index 0000000000..0f44e62c2d --- /dev/null +++ b/pinning/remote/client/openapi/model_pin_status.go @@ -0,0 +1,262 @@ +/* + * IPFS Pinning Service API + * + * ## About this spec The IPFS Pinning Service API is intended to be an implementation-agnostic API: - For use and implementation by pinning service providers - For use in client mode by IPFS nodes and GUI-based applications > **Note**: while ready for implementation, this spec is still a work in progress! 🏗️ **Your input and feedback are welcome and valuable as we develop this API spec. Please join the design discussion at [github.com/ipfs/pinning-services-api-spec](https://github.com/ipfs/pinning-services-api-spec).** # Schemas This section describes the most important object types and conventions. A full list of fields and schemas can be found in the `schemas` section of the [YAML file](https://github.com/ipfs/pinning-services-api-spec/blob/master/ipfs-pinning-service.yaml). ## Identifiers ### cid [Content Identifier (CID)](https://docs.ipfs.io/concepts/content-addressing/) points at the root of a DAG that is pinned recursively. ### requestid Unique identifier of a pin request. When a pin is created, the service responds with unique `requestid` that can be later used for pin removal. When the same `cid` is pinned again, a different `requestid` is returned to differentiate between those pin requests. Service implementation should use UUID, `hash(accessToken,Pin,PinStatus.created)`, or any other opaque identifier that provides equally strong protection against race conditions. ## Objects ### Pin object ![pin object](https://bafybeideck2fchyxna4wqwc2mo67yriokehw3yujboc5redjdaajrk2fjq.ipfs.dweb.link/pin.png) The `Pin` object is a representation of a pin request. It includes the `cid` of data to be pinned, as well as optional metadata in `name`, `origins`, and `meta`. ### Pin status response ![pin status response object](https://bafybeideck2fchyxna4wqwc2mo67yriokehw3yujboc5redjdaajrk2fjq.ipfs.dweb.link/pinstatus.png) The `PinStatus` object is a representation of the current state of a pinning operation. It includes the original `pin` object, along with the current `status` and globally unique `requestid` of the entire pinning request, which can be used for future status checks and management. Addresses in the `delegates` array are peers delegated by the pinning service for facilitating direct file transfers (more details in the provider hints section). Any additional vendor-specific information is returned in optional `info`. ## The pin lifecycle ![pinning service objects and lifecycle](https://bafybeideck2fchyxna4wqwc2mo67yriokehw3yujboc5redjdaajrk2fjq.ipfs.dweb.link/lifecycle.png) ### Creating a new pin object The user sends a `Pin` object to `POST /pins` and receives a `PinStatus` response: - `requestid` in `PinStatus` is the identifier of the pin operation, which can can be used for checking status, and removing the pin in the future - `status` in `PinStatus` indicates the current state of a pin ### Checking status of in-progress pinning `status` (in `PinStatus`) may indicate a pending state (`queued` or `pinning`). This means the data behind `Pin.cid` was not found on the pinning service and is being fetched from the IPFS network at large, which may take time. In this case, the user can periodically check pinning progress via `GET /pins/{requestid}` until pinning is successful, or the user decides to remove the pending pin. ### Replacing an existing pin object The user can replace an existing pin object via `POST /pins/{requestid}`. This is a shortcut for removing a pin object identified by `requestid` and creating a new one in a single API call that protects against undesired garbage collection of blocks common to both pins. Useful when updating a pin representing a huge dataset where most of blocks did not change. The new pin object `requestid` is returned in the `PinStatus` response. The old pin object is deleted automatically. ### Removing a pin object A pin object can be removed via `DELETE /pins/{requestid}`. ## Provider hints Pinning of new data can be accelerated by providing a list of known data sources in `Pin.origins`, and connecting at least one of them to pinning service nodes at `PinStatus.delegates`. The most common scenario is a client putting its own IPFS node's multiaddrs in `Pin.origins`, and then directly connecting to every multiaddr returned by a pinning service in `PinStatus.delegates` to initiate transfer. This ensures data transfer starts immediately (without waiting for provider discovery over DHT), and direct dial from a client works around peer routing issues in restrictive network topologies such as NATs. ## Custom metadata Pinning services are encouraged to add support for additional features by leveraging the optional `Pin.meta` and `PinStatus.info` fields. While these attributes can be application- or vendor-specific, we encourage the community at large to leverage these attributes as a sandbox to come up with conventions that could become part of future revisions of this API. ### Pin metadata String keys and values passed in `Pin.meta` are persisted with the pin object. Potential uses: - `Pin.meta[app_id]`: Attaching a unique identifier to pins created by an app enables filtering pins per app via `?meta={\"app_id\":}` - `Pin.meta[vendor_policy]`: Vendor-specific policy (for example: which region to use, how many copies to keep) Note that it is OK for a client to omit or ignore these optional attributes; doing so should not impact the basic pinning functionality. ### Pin status info Additional `PinStatus.info` can be returned by pinning service. Potential uses: - `PinStatus.info[status_details]`: more info about the current status (queue position, percentage of transferred data, summary of where data is stored, etc); when `PinStatus.status=failed`, it could provide a reason why a pin operation failed (e.g. lack of funds, DAG too big, etc.) - `PinStatus.info[dag_size]`: the size of pinned data, along with DAG overhead - `PinStatus.info[raw_size]`: the size of data without DAG overhead (eg. unixfs) - `PinStatus.info[pinned_until]`: if vendor supports time-bound pins, this could indicate when the pin will expire # Pagination and filtering Pin objects can be listed by executing `GET /pins` with optional parameters: - When no filters are provided, the endpoint will return a small batch of the 10 most recently created items, from the latest to the oldest. - The number of returned items can be adjusted with the `limit` parameter (implicit default is 10). - If the value in `PinResults.count` is bigger than the length of `PinResults.results`, the client can infer there are more results that can be queried. - To read more items, pass the `before` filter with the timestamp from `PinStatus.created` found in the oldest item in the current batch of results. Repeat to read all results. - Returned results can be fine-tuned by applying optional `after`, `cid`, `name`, `status`, or `meta` filters. > **Note**: pagination by the `created` timestamp requires each value to be globally unique. Any future considerations to add support for bulk creation must account for this. + * + * API version: 0.1.1 + * Generated by: OpenAPI Generator (https://openapi-generator.tech) + */ + +package openapi + +import ( + "encoding/json" + "time" +) + +// PinStatus Pin object with status +type PinStatus struct { + // Globally unique identifier of the pin request; can be used to check the status of ongoing pinning, or pin removal + Requestid string `json:"requestid"` + Status Status `json:"status"` + // Immutable timestamp indicating when a pin request entered a pinning service; can be used for filtering results and pagination + Created time.Time `json:"created"` + Pin Pin `json:"pin"` + // List of multiaddrs designated by pinning service for transferring any new data from external peers + Delegates []string `json:"delegates"` + // Optional info for PinStatus response + Info *map[string]string `json:"info,omitempty"` +} + +// NewPinStatus instantiates a new PinStatus object +// This constructor will assign default values to properties that have it defined, +// and makes sure properties required by API are set, but the set of arguments +// will change when the set of required properties is changed +func NewPinStatus(requestid string, status Status, created time.Time, pin Pin, delegates []string) *PinStatus { + this := PinStatus{} + this.Requestid = requestid + this.Status = status + this.Created = created + this.Pin = pin + this.Delegates = delegates + return &this +} + +// NewPinStatusWithDefaults instantiates a new PinStatus object +// This constructor will only assign default values to properties that have it defined, +// but it doesn't guarantee that properties required by API are set +func NewPinStatusWithDefaults() *PinStatus { + this := PinStatus{} + return &this +} + +// GetRequestid returns the Requestid field value +func (o *PinStatus) GetRequestid() string { + if o == nil { + var ret string + return ret + } + + return o.Requestid +} + +// GetRequestidOk returns a tuple with the Requestid field value +// and a boolean to check if the value has been set. +func (o *PinStatus) GetRequestidOk() (*string, bool) { + if o == nil { + return nil, false + } + return &o.Requestid, true +} + +// SetRequestid sets field value +func (o *PinStatus) SetRequestid(v string) { + o.Requestid = v +} + +// GetStatus returns the Status field value +func (o *PinStatus) GetStatus() Status { + if o == nil { + var ret Status + return ret + } + + return o.Status +} + +// GetStatusOk returns a tuple with the Status field value +// and a boolean to check if the value has been set. +func (o *PinStatus) GetStatusOk() (*Status, bool) { + if o == nil { + return nil, false + } + return &o.Status, true +} + +// SetStatus sets field value +func (o *PinStatus) SetStatus(v Status) { + o.Status = v +} + +// GetCreated returns the Created field value +func (o *PinStatus) GetCreated() time.Time { + if o == nil { + var ret time.Time + return ret + } + + return o.Created +} + +// GetCreatedOk returns a tuple with the Created field value +// and a boolean to check if the value has been set. +func (o *PinStatus) GetCreatedOk() (*time.Time, bool) { + if o == nil { + return nil, false + } + return &o.Created, true +} + +// SetCreated sets field value +func (o *PinStatus) SetCreated(v time.Time) { + o.Created = v +} + +// GetPin returns the Pin field value +func (o *PinStatus) GetPin() Pin { + if o == nil { + var ret Pin + return ret + } + + return o.Pin +} + +// GetPinOk returns a tuple with the Pin field value +// and a boolean to check if the value has been set. +func (o *PinStatus) GetPinOk() (*Pin, bool) { + if o == nil { + return nil, false + } + return &o.Pin, true +} + +// SetPin sets field value +func (o *PinStatus) SetPin(v Pin) { + o.Pin = v +} + +// GetDelegates returns the Delegates field value +func (o *PinStatus) GetDelegates() []string { + if o == nil { + var ret []string + return ret + } + + return o.Delegates +} + +// GetDelegatesOk returns a tuple with the Delegates field value +// and a boolean to check if the value has been set. +func (o *PinStatus) GetDelegatesOk() (*[]string, bool) { + if o == nil { + return nil, false + } + return &o.Delegates, true +} + +// SetDelegates sets field value +func (o *PinStatus) SetDelegates(v []string) { + o.Delegates = v +} + +// GetInfo returns the Info field value if set, zero value otherwise. +func (o *PinStatus) GetInfo() map[string]string { + if o == nil || o.Info == nil { + var ret map[string]string + return ret + } + return *o.Info +} + +// GetInfoOk returns a tuple with the Info field value if set, nil otherwise +// and a boolean to check if the value has been set. +func (o *PinStatus) GetInfoOk() (*map[string]string, bool) { + if o == nil || o.Info == nil { + return nil, false + } + return o.Info, true +} + +// HasInfo returns a boolean if a field has been set. +func (o *PinStatus) HasInfo() bool { + if o != nil && o.Info != nil { + return true + } + + return false +} + +// SetInfo gets a reference to the given map[string]string and assigns it to the Info field. +func (o *PinStatus) SetInfo(v map[string]string) { + o.Info = &v +} + +func (o PinStatus) MarshalJSON() ([]byte, error) { + toSerialize := map[string]interface{}{} + if true { + toSerialize["requestid"] = o.Requestid + } + if true { + toSerialize["status"] = o.Status + } + if true { + toSerialize["created"] = o.Created + } + if true { + toSerialize["pin"] = o.Pin + } + if true { + toSerialize["delegates"] = o.Delegates + } + if o.Info != nil { + toSerialize["info"] = o.Info + } + return json.Marshal(toSerialize) +} + +type NullablePinStatus struct { + value *PinStatus + isSet bool +} + +func (v NullablePinStatus) Get() *PinStatus { + return v.value +} + +func (v *NullablePinStatus) Set(val *PinStatus) { + v.value = val + v.isSet = true +} + +func (v NullablePinStatus) IsSet() bool { + return v.isSet +} + +func (v *NullablePinStatus) Unset() { + v.value = nil + v.isSet = false +} + +func NewNullablePinStatus(val *PinStatus) *NullablePinStatus { + return &NullablePinStatus{value: val, isSet: true} +} + +func (v NullablePinStatus) MarshalJSON() ([]byte, error) { + return json.Marshal(v.value) +} + +func (v *NullablePinStatus) UnmarshalJSON(src []byte) error { + v.isSet = true + return json.Unmarshal(src, &v.value) +} diff --git a/pinning/remote/client/openapi/model_status.go b/pinning/remote/client/openapi/model_status.go new file mode 100644 index 0000000000..56944819fa --- /dev/null +++ b/pinning/remote/client/openapi/model_status.go @@ -0,0 +1,84 @@ +/* + * IPFS Pinning Service API + * + * ## About this spec The IPFS Pinning Service API is intended to be an implementation-agnostic API: - For use and implementation by pinning service providers - For use in client mode by IPFS nodes and GUI-based applications > **Note**: while ready for implementation, this spec is still a work in progress! 🏗️ **Your input and feedback are welcome and valuable as we develop this API spec. Please join the design discussion at [github.com/ipfs/pinning-services-api-spec](https://github.com/ipfs/pinning-services-api-spec).** # Schemas This section describes the most important object types and conventions. A full list of fields and schemas can be found in the `schemas` section of the [YAML file](https://github.com/ipfs/pinning-services-api-spec/blob/master/ipfs-pinning-service.yaml). ## Identifiers ### cid [Content Identifier (CID)](https://docs.ipfs.io/concepts/content-addressing/) points at the root of a DAG that is pinned recursively. ### requestid Unique identifier of a pin request. When a pin is created, the service responds with unique `requestid` that can be later used for pin removal. When the same `cid` is pinned again, a different `requestid` is returned to differentiate between those pin requests. Service implementation should use UUID, `hash(accessToken,Pin,PinStatus.created)`, or any other opaque identifier that provides equally strong protection against race conditions. ## Objects ### Pin object ![pin object](https://bafybeideck2fchyxna4wqwc2mo67yriokehw3yujboc5redjdaajrk2fjq.ipfs.dweb.link/pin.png) The `Pin` object is a representation of a pin request. It includes the `cid` of data to be pinned, as well as optional metadata in `name`, `origins`, and `meta`. ### Pin status response ![pin status response object](https://bafybeideck2fchyxna4wqwc2mo67yriokehw3yujboc5redjdaajrk2fjq.ipfs.dweb.link/pinstatus.png) The `PinStatus` object is a representation of the current state of a pinning operation. It includes the original `pin` object, along with the current `status` and globally unique `requestid` of the entire pinning request, which can be used for future status checks and management. Addresses in the `delegates` array are peers delegated by the pinning service for facilitating direct file transfers (more details in the provider hints section). Any additional vendor-specific information is returned in optional `info`. ## The pin lifecycle ![pinning service objects and lifecycle](https://bafybeideck2fchyxna4wqwc2mo67yriokehw3yujboc5redjdaajrk2fjq.ipfs.dweb.link/lifecycle.png) ### Creating a new pin object The user sends a `Pin` object to `POST /pins` and receives a `PinStatus` response: - `requestid` in `PinStatus` is the identifier of the pin operation, which can can be used for checking status, and removing the pin in the future - `status` in `PinStatus` indicates the current state of a pin ### Checking status of in-progress pinning `status` (in `PinStatus`) may indicate a pending state (`queued` or `pinning`). This means the data behind `Pin.cid` was not found on the pinning service and is being fetched from the IPFS network at large, which may take time. In this case, the user can periodically check pinning progress via `GET /pins/{requestid}` until pinning is successful, or the user decides to remove the pending pin. ### Replacing an existing pin object The user can replace an existing pin object via `POST /pins/{requestid}`. This is a shortcut for removing a pin object identified by `requestid` and creating a new one in a single API call that protects against undesired garbage collection of blocks common to both pins. Useful when updating a pin representing a huge dataset where most of blocks did not change. The new pin object `requestid` is returned in the `PinStatus` response. The old pin object is deleted automatically. ### Removing a pin object A pin object can be removed via `DELETE /pins/{requestid}`. ## Provider hints Pinning of new data can be accelerated by providing a list of known data sources in `Pin.origins`, and connecting at least one of them to pinning service nodes at `PinStatus.delegates`. The most common scenario is a client putting its own IPFS node's multiaddrs in `Pin.origins`, and then directly connecting to every multiaddr returned by a pinning service in `PinStatus.delegates` to initiate transfer. This ensures data transfer starts immediately (without waiting for provider discovery over DHT), and direct dial from a client works around peer routing issues in restrictive network topologies such as NATs. ## Custom metadata Pinning services are encouraged to add support for additional features by leveraging the optional `Pin.meta` and `PinStatus.info` fields. While these attributes can be application- or vendor-specific, we encourage the community at large to leverage these attributes as a sandbox to come up with conventions that could become part of future revisions of this API. ### Pin metadata String keys and values passed in `Pin.meta` are persisted with the pin object. Potential uses: - `Pin.meta[app_id]`: Attaching a unique identifier to pins created by an app enables filtering pins per app via `?meta={\"app_id\":}` - `Pin.meta[vendor_policy]`: Vendor-specific policy (for example: which region to use, how many copies to keep) Note that it is OK for a client to omit or ignore these optional attributes; doing so should not impact the basic pinning functionality. ### Pin status info Additional `PinStatus.info` can be returned by pinning service. Potential uses: - `PinStatus.info[status_details]`: more info about the current status (queue position, percentage of transferred data, summary of where data is stored, etc); when `PinStatus.status=failed`, it could provide a reason why a pin operation failed (e.g. lack of funds, DAG too big, etc.) - `PinStatus.info[dag_size]`: the size of pinned data, along with DAG overhead - `PinStatus.info[raw_size]`: the size of data without DAG overhead (eg. unixfs) - `PinStatus.info[pinned_until]`: if vendor supports time-bound pins, this could indicate when the pin will expire # Pagination and filtering Pin objects can be listed by executing `GET /pins` with optional parameters: - When no filters are provided, the endpoint will return a small batch of the 10 most recently created items, from the latest to the oldest. - The number of returned items can be adjusted with the `limit` parameter (implicit default is 10). - If the value in `PinResults.count` is bigger than the length of `PinResults.results`, the client can infer there are more results that can be queried. - To read more items, pass the `before` filter with the timestamp from `PinStatus.created` found in the oldest item in the current batch of results. Repeat to read all results. - Returned results can be fine-tuned by applying optional `after`, `cid`, `name`, `status`, or `meta` filters. > **Note**: pagination by the `created` timestamp requires each value to be globally unique. Any future considerations to add support for bulk creation must account for this. + * + * API version: 0.1.1 + * Generated by: OpenAPI Generator (https://openapi-generator.tech) + */ + +package openapi + +import ( + "encoding/json" + "fmt" +) + +// Status Status a pin object can have at a pinning service +type Status string + +// List of Status +const ( + QUEUED Status = "queued" + PINNING Status = "pinning" + PINNED Status = "pinned" + FAILED Status = "failed" +) + +func (v *Status) UnmarshalJSON(src []byte) error { + var value string + err := json.Unmarshal(src, &value) + if err != nil { + return err + } + enumTypeValue := Status(value) + for _, existing := range []Status{"queued", "pinning", "pinned", "failed"} { + if existing == enumTypeValue { + *v = enumTypeValue + return nil + } + } + + return fmt.Errorf("%+v is not a valid Status", value) +} + +// Ptr returns reference to Status value +func (v Status) Ptr() *Status { + return &v +} + +type NullableStatus struct { + value *Status + isSet bool +} + +func (v NullableStatus) Get() *Status { + return v.value +} + +func (v *NullableStatus) Set(val *Status) { + v.value = val + v.isSet = true +} + +func (v NullableStatus) IsSet() bool { + return v.isSet +} + +func (v *NullableStatus) Unset() { + v.value = nil + v.isSet = false +} + +func NewNullableStatus(val *Status) *NullableStatus { + return &NullableStatus{value: val, isSet: true} +} + +func (v NullableStatus) MarshalJSON() ([]byte, error) { + return json.Marshal(v.value) +} + +func (v *NullableStatus) UnmarshalJSON(src []byte) error { + v.isSet = true + return json.Unmarshal(src, &v.value) +} diff --git a/pinning/remote/client/openapi/response.go b/pinning/remote/client/openapi/response.go new file mode 100644 index 0000000000..8f9fb0b081 --- /dev/null +++ b/pinning/remote/client/openapi/response.go @@ -0,0 +1,46 @@ +/* + * IPFS Pinning Service API + * + * ## About this spec The IPFS Pinning Service API is intended to be an implementation-agnostic API: - For use and implementation by pinning service providers - For use in client mode by IPFS nodes and GUI-based applications > **Note**: while ready for implementation, this spec is still a work in progress! 🏗️ **Your input and feedback are welcome and valuable as we develop this API spec. Please join the design discussion at [github.com/ipfs/pinning-services-api-spec](https://github.com/ipfs/pinning-services-api-spec).** # Schemas This section describes the most important object types and conventions. A full list of fields and schemas can be found in the `schemas` section of the [YAML file](https://github.com/ipfs/pinning-services-api-spec/blob/master/ipfs-pinning-service.yaml). ## Identifiers ### cid [Content Identifier (CID)](https://docs.ipfs.io/concepts/content-addressing/) points at the root of a DAG that is pinned recursively. ### requestid Unique identifier of a pin request. When a pin is created, the service responds with unique `requestid` that can be later used for pin removal. When the same `cid` is pinned again, a different `requestid` is returned to differentiate between those pin requests. Service implementation should use UUID, `hash(accessToken,Pin,PinStatus.created)`, or any other opaque identifier that provides equally strong protection against race conditions. ## Objects ### Pin object ![pin object](https://bafybeideck2fchyxna4wqwc2mo67yriokehw3yujboc5redjdaajrk2fjq.ipfs.dweb.link/pin.png) The `Pin` object is a representation of a pin request. It includes the `cid` of data to be pinned, as well as optional metadata in `name`, `origins`, and `meta`. ### Pin status response ![pin status response object](https://bafybeideck2fchyxna4wqwc2mo67yriokehw3yujboc5redjdaajrk2fjq.ipfs.dweb.link/pinstatus.png) The `PinStatus` object is a representation of the current state of a pinning operation. It includes the original `pin` object, along with the current `status` and globally unique `requestid` of the entire pinning request, which can be used for future status checks and management. Addresses in the `delegates` array are peers delegated by the pinning service for facilitating direct file transfers (more details in the provider hints section). Any additional vendor-specific information is returned in optional `info`. ## The pin lifecycle ![pinning service objects and lifecycle](https://bafybeideck2fchyxna4wqwc2mo67yriokehw3yujboc5redjdaajrk2fjq.ipfs.dweb.link/lifecycle.png) ### Creating a new pin object The user sends a `Pin` object to `POST /pins` and receives a `PinStatus` response: - `requestid` in `PinStatus` is the identifier of the pin operation, which can can be used for checking status, and removing the pin in the future - `status` in `PinStatus` indicates the current state of a pin ### Checking status of in-progress pinning `status` (in `PinStatus`) may indicate a pending state (`queued` or `pinning`). This means the data behind `Pin.cid` was not found on the pinning service and is being fetched from the IPFS network at large, which may take time. In this case, the user can periodically check pinning progress via `GET /pins/{requestid}` until pinning is successful, or the user decides to remove the pending pin. ### Replacing an existing pin object The user can replace an existing pin object via `POST /pins/{requestid}`. This is a shortcut for removing a pin object identified by `requestid` and creating a new one in a single API call that protects against undesired garbage collection of blocks common to both pins. Useful when updating a pin representing a huge dataset where most of blocks did not change. The new pin object `requestid` is returned in the `PinStatus` response. The old pin object is deleted automatically. ### Removing a pin object A pin object can be removed via `DELETE /pins/{requestid}`. ## Provider hints Pinning of new data can be accelerated by providing a list of known data sources in `Pin.origins`, and connecting at least one of them to pinning service nodes at `PinStatus.delegates`. The most common scenario is a client putting its own IPFS node's multiaddrs in `Pin.origins`, and then directly connecting to every multiaddr returned by a pinning service in `PinStatus.delegates` to initiate transfer. This ensures data transfer starts immediately (without waiting for provider discovery over DHT), and direct dial from a client works around peer routing issues in restrictive network topologies such as NATs. ## Custom metadata Pinning services are encouraged to add support for additional features by leveraging the optional `Pin.meta` and `PinStatus.info` fields. While these attributes can be application- or vendor-specific, we encourage the community at large to leverage these attributes as a sandbox to come up with conventions that could become part of future revisions of this API. ### Pin metadata String keys and values passed in `Pin.meta` are persisted with the pin object. Potential uses: - `Pin.meta[app_id]`: Attaching a unique identifier to pins created by an app enables filtering pins per app via `?meta={\"app_id\":}` - `Pin.meta[vendor_policy]`: Vendor-specific policy (for example: which region to use, how many copies to keep) Note that it is OK for a client to omit or ignore these optional attributes; doing so should not impact the basic pinning functionality. ### Pin status info Additional `PinStatus.info` can be returned by pinning service. Potential uses: - `PinStatus.info[status_details]`: more info about the current status (queue position, percentage of transferred data, summary of where data is stored, etc); when `PinStatus.status=failed`, it could provide a reason why a pin operation failed (e.g. lack of funds, DAG too big, etc.) - `PinStatus.info[dag_size]`: the size of pinned data, along with DAG overhead - `PinStatus.info[raw_size]`: the size of data without DAG overhead (eg. unixfs) - `PinStatus.info[pinned_until]`: if vendor supports time-bound pins, this could indicate when the pin will expire # Pagination and filtering Pin objects can be listed by executing `GET /pins` with optional parameters: - When no filters are provided, the endpoint will return a small batch of the 10 most recently created items, from the latest to the oldest. - The number of returned items can be adjusted with the `limit` parameter (implicit default is 10). - If the value in `PinResults.count` is bigger than the length of `PinResults.results`, the client can infer there are more results that can be queried. - To read more items, pass the `before` filter with the timestamp from `PinStatus.created` found in the oldest item in the current batch of results. Repeat to read all results. - Returned results can be fine-tuned by applying optional `after`, `cid`, `name`, `status`, or `meta` filters. > **Note**: pagination by the `created` timestamp requires each value to be globally unique. Any future considerations to add support for bulk creation must account for this. + * + * API version: 0.1.1 + * Generated by: OpenAPI Generator (https://openapi-generator.tech) + */ + +package openapi + +import ( + "net/http" +) + +// APIResponse stores the API response returned by the server. +type APIResponse struct { + *http.Response `json:"-"` + Message string `json:"message,omitempty"` + // Operation is the name of the OpenAPI operation. + Operation string `json:"operation,omitempty"` + // RequestURL is the request URL. This value is always available, even if the + // embedded *http.Response is nil. + RequestURL string `json:"url,omitempty"` + // Method is the HTTP method used for the request. This value is always + // available, even if the embedded *http.Response is nil. + Method string `json:"method,omitempty"` + // Payload holds the contents of the response body (which may be nil or empty). + // This is provided here as the raw response.Body() reader will have already + // been drained. + Payload []byte `json:"-"` +} + +// NewAPIResponse returns a new APIResonse object. +func NewAPIResponse(r *http.Response) *APIResponse { + + response := &APIResponse{Response: r} + return response +} + +// NewAPIResponseWithError returns a new APIResponse object with the provided error message. +func NewAPIResponseWithError(errorMessage string) *APIResponse { + + response := &APIResponse{Message: errorMessage} + return response +} diff --git a/pinning/remote/client/openapi/utils.go b/pinning/remote/client/openapi/utils.go new file mode 100644 index 0000000000..25d36f11b6 --- /dev/null +++ b/pinning/remote/client/openapi/utils.go @@ -0,0 +1,327 @@ +/* + * IPFS Pinning Service API + * + * ## About this spec The IPFS Pinning Service API is intended to be an implementation-agnostic API: - For use and implementation by pinning service providers - For use in client mode by IPFS nodes and GUI-based applications > **Note**: while ready for implementation, this spec is still a work in progress! 🏗️ **Your input and feedback are welcome and valuable as we develop this API spec. Please join the design discussion at [github.com/ipfs/pinning-services-api-spec](https://github.com/ipfs/pinning-services-api-spec).** # Schemas This section describes the most important object types and conventions. A full list of fields and schemas can be found in the `schemas` section of the [YAML file](https://github.com/ipfs/pinning-services-api-spec/blob/master/ipfs-pinning-service.yaml). ## Identifiers ### cid [Content Identifier (CID)](https://docs.ipfs.io/concepts/content-addressing/) points at the root of a DAG that is pinned recursively. ### requestid Unique identifier of a pin request. When a pin is created, the service responds with unique `requestid` that can be later used for pin removal. When the same `cid` is pinned again, a different `requestid` is returned to differentiate between those pin requests. Service implementation should use UUID, `hash(accessToken,Pin,PinStatus.created)`, or any other opaque identifier that provides equally strong protection against race conditions. ## Objects ### Pin object ![pin object](https://bafybeideck2fchyxna4wqwc2mo67yriokehw3yujboc5redjdaajrk2fjq.ipfs.dweb.link/pin.png) The `Pin` object is a representation of a pin request. It includes the `cid` of data to be pinned, as well as optional metadata in `name`, `origins`, and `meta`. ### Pin status response ![pin status response object](https://bafybeideck2fchyxna4wqwc2mo67yriokehw3yujboc5redjdaajrk2fjq.ipfs.dweb.link/pinstatus.png) The `PinStatus` object is a representation of the current state of a pinning operation. It includes the original `pin` object, along with the current `status` and globally unique `requestid` of the entire pinning request, which can be used for future status checks and management. Addresses in the `delegates` array are peers delegated by the pinning service for facilitating direct file transfers (more details in the provider hints section). Any additional vendor-specific information is returned in optional `info`. ## The pin lifecycle ![pinning service objects and lifecycle](https://bafybeideck2fchyxna4wqwc2mo67yriokehw3yujboc5redjdaajrk2fjq.ipfs.dweb.link/lifecycle.png) ### Creating a new pin object The user sends a `Pin` object to `POST /pins` and receives a `PinStatus` response: - `requestid` in `PinStatus` is the identifier of the pin operation, which can can be used for checking status, and removing the pin in the future - `status` in `PinStatus` indicates the current state of a pin ### Checking status of in-progress pinning `status` (in `PinStatus`) may indicate a pending state (`queued` or `pinning`). This means the data behind `Pin.cid` was not found on the pinning service and is being fetched from the IPFS network at large, which may take time. In this case, the user can periodically check pinning progress via `GET /pins/{requestid}` until pinning is successful, or the user decides to remove the pending pin. ### Replacing an existing pin object The user can replace an existing pin object via `POST /pins/{requestid}`. This is a shortcut for removing a pin object identified by `requestid` and creating a new one in a single API call that protects against undesired garbage collection of blocks common to both pins. Useful when updating a pin representing a huge dataset where most of blocks did not change. The new pin object `requestid` is returned in the `PinStatus` response. The old pin object is deleted automatically. ### Removing a pin object A pin object can be removed via `DELETE /pins/{requestid}`. ## Provider hints Pinning of new data can be accelerated by providing a list of known data sources in `Pin.origins`, and connecting at least one of them to pinning service nodes at `PinStatus.delegates`. The most common scenario is a client putting its own IPFS node's multiaddrs in `Pin.origins`, and then directly connecting to every multiaddr returned by a pinning service in `PinStatus.delegates` to initiate transfer. This ensures data transfer starts immediately (without waiting for provider discovery over DHT), and direct dial from a client works around peer routing issues in restrictive network topologies such as NATs. ## Custom metadata Pinning services are encouraged to add support for additional features by leveraging the optional `Pin.meta` and `PinStatus.info` fields. While these attributes can be application- or vendor-specific, we encourage the community at large to leverage these attributes as a sandbox to come up with conventions that could become part of future revisions of this API. ### Pin metadata String keys and values passed in `Pin.meta` are persisted with the pin object. Potential uses: - `Pin.meta[app_id]`: Attaching a unique identifier to pins created by an app enables filtering pins per app via `?meta={\"app_id\":}` - `Pin.meta[vendor_policy]`: Vendor-specific policy (for example: which region to use, how many copies to keep) Note that it is OK for a client to omit or ignore these optional attributes; doing so should not impact the basic pinning functionality. ### Pin status info Additional `PinStatus.info` can be returned by pinning service. Potential uses: - `PinStatus.info[status_details]`: more info about the current status (queue position, percentage of transferred data, summary of where data is stored, etc); when `PinStatus.status=failed`, it could provide a reason why a pin operation failed (e.g. lack of funds, DAG too big, etc.) - `PinStatus.info[dag_size]`: the size of pinned data, along with DAG overhead - `PinStatus.info[raw_size]`: the size of data without DAG overhead (eg. unixfs) - `PinStatus.info[pinned_until]`: if vendor supports time-bound pins, this could indicate when the pin will expire # Pagination and filtering Pin objects can be listed by executing `GET /pins` with optional parameters: - When no filters are provided, the endpoint will return a small batch of the 10 most recently created items, from the latest to the oldest. - The number of returned items can be adjusted with the `limit` parameter (implicit default is 10). - If the value in `PinResults.count` is bigger than the length of `PinResults.results`, the client can infer there are more results that can be queried. - To read more items, pass the `before` filter with the timestamp from `PinStatus.created` found in the oldest item in the current batch of results. Repeat to read all results. - Returned results can be fine-tuned by applying optional `after`, `cid`, `name`, `status`, or `meta` filters. > **Note**: pagination by the `created` timestamp requires each value to be globally unique. Any future considerations to add support for bulk creation must account for this. + * + * API version: 0.1.1 + * Generated by: OpenAPI Generator (https://openapi-generator.tech) + */ + +package openapi + +import ( + "encoding/json" + "time" +) + +// PtrBool is a helper routine that returns a pointer to given integer value. +func PtrBool(v bool) *bool { return &v } + +// PtrInt is a helper routine that returns a pointer to given integer value. +func PtrInt(v int) *int { return &v } + +// PtrInt32 is a helper routine that returns a pointer to given integer value. +func PtrInt32(v int32) *int32 { return &v } + +// PtrInt64 is a helper routine that returns a pointer to given integer value. +func PtrInt64(v int64) *int64 { return &v } + +// PtrFloat32 is a helper routine that returns a pointer to given float value. +func PtrFloat32(v float32) *float32 { return &v } + +// PtrFloat64 is a helper routine that returns a pointer to given float value. +func PtrFloat64(v float64) *float64 { return &v } + +// PtrString is a helper routine that returns a pointer to given string value. +func PtrString(v string) *string { return &v } + +// PtrTime is helper routine that returns a pointer to given Time value. +func PtrTime(v time.Time) *time.Time { return &v } + +type NullableBool struct { + value *bool + isSet bool +} + +func (v NullableBool) Get() *bool { + return v.value +} + +func (v *NullableBool) Set(val *bool) { + v.value = val + v.isSet = true +} + +func (v NullableBool) IsSet() bool { + return v.isSet +} + +func (v *NullableBool) Unset() { + v.value = nil + v.isSet = false +} + +func NewNullableBool(val *bool) *NullableBool { + return &NullableBool{value: val, isSet: true} +} + +func (v NullableBool) MarshalJSON() ([]byte, error) { + return json.Marshal(v.value) +} + +func (v *NullableBool) UnmarshalJSON(src []byte) error { + v.isSet = true + return json.Unmarshal(src, &v.value) +} + +type NullableInt struct { + value *int + isSet bool +} + +func (v NullableInt) Get() *int { + return v.value +} + +func (v *NullableInt) Set(val *int) { + v.value = val + v.isSet = true +} + +func (v NullableInt) IsSet() bool { + return v.isSet +} + +func (v *NullableInt) Unset() { + v.value = nil + v.isSet = false +} + +func NewNullableInt(val *int) *NullableInt { + return &NullableInt{value: val, isSet: true} +} + +func (v NullableInt) MarshalJSON() ([]byte, error) { + return json.Marshal(v.value) +} + +func (v *NullableInt) UnmarshalJSON(src []byte) error { + v.isSet = true + return json.Unmarshal(src, &v.value) +} + +type NullableInt32 struct { + value *int32 + isSet bool +} + +func (v NullableInt32) Get() *int32 { + return v.value +} + +func (v *NullableInt32) Set(val *int32) { + v.value = val + v.isSet = true +} + +func (v NullableInt32) IsSet() bool { + return v.isSet +} + +func (v *NullableInt32) Unset() { + v.value = nil + v.isSet = false +} + +func NewNullableInt32(val *int32) *NullableInt32 { + return &NullableInt32{value: val, isSet: true} +} + +func (v NullableInt32) MarshalJSON() ([]byte, error) { + return json.Marshal(v.value) +} + +func (v *NullableInt32) UnmarshalJSON(src []byte) error { + v.isSet = true + return json.Unmarshal(src, &v.value) +} + +type NullableInt64 struct { + value *int64 + isSet bool +} + +func (v NullableInt64) Get() *int64 { + return v.value +} + +func (v *NullableInt64) Set(val *int64) { + v.value = val + v.isSet = true +} + +func (v NullableInt64) IsSet() bool { + return v.isSet +} + +func (v *NullableInt64) Unset() { + v.value = nil + v.isSet = false +} + +func NewNullableInt64(val *int64) *NullableInt64 { + return &NullableInt64{value: val, isSet: true} +} + +func (v NullableInt64) MarshalJSON() ([]byte, error) { + return json.Marshal(v.value) +} + +func (v *NullableInt64) UnmarshalJSON(src []byte) error { + v.isSet = true + return json.Unmarshal(src, &v.value) +} + +type NullableFloat32 struct { + value *float32 + isSet bool +} + +func (v NullableFloat32) Get() *float32 { + return v.value +} + +func (v *NullableFloat32) Set(val *float32) { + v.value = val + v.isSet = true +} + +func (v NullableFloat32) IsSet() bool { + return v.isSet +} + +func (v *NullableFloat32) Unset() { + v.value = nil + v.isSet = false +} + +func NewNullableFloat32(val *float32) *NullableFloat32 { + return &NullableFloat32{value: val, isSet: true} +} + +func (v NullableFloat32) MarshalJSON() ([]byte, error) { + return json.Marshal(v.value) +} + +func (v *NullableFloat32) UnmarshalJSON(src []byte) error { + v.isSet = true + return json.Unmarshal(src, &v.value) +} + +type NullableFloat64 struct { + value *float64 + isSet bool +} + +func (v NullableFloat64) Get() *float64 { + return v.value +} + +func (v *NullableFloat64) Set(val *float64) { + v.value = val + v.isSet = true +} + +func (v NullableFloat64) IsSet() bool { + return v.isSet +} + +func (v *NullableFloat64) Unset() { + v.value = nil + v.isSet = false +} + +func NewNullableFloat64(val *float64) *NullableFloat64 { + return &NullableFloat64{value: val, isSet: true} +} + +func (v NullableFloat64) MarshalJSON() ([]byte, error) { + return json.Marshal(v.value) +} + +func (v *NullableFloat64) UnmarshalJSON(src []byte) error { + v.isSet = true + return json.Unmarshal(src, &v.value) +} + +type NullableString struct { + value *string + isSet bool +} + +func (v NullableString) Get() *string { + return v.value +} + +func (v *NullableString) Set(val *string) { + v.value = val + v.isSet = true +} + +func (v NullableString) IsSet() bool { + return v.isSet +} + +func (v *NullableString) Unset() { + v.value = nil + v.isSet = false +} + +func NewNullableString(val *string) *NullableString { + return &NullableString{value: val, isSet: true} +} + +func (v NullableString) MarshalJSON() ([]byte, error) { + return json.Marshal(v.value) +} + +func (v *NullableString) UnmarshalJSON(src []byte) error { + v.isSet = true + return json.Unmarshal(src, &v.value) +} + +type NullableTime struct { + value *time.Time + isSet bool +} + +func (v NullableTime) Get() *time.Time { + return v.value +} + +func (v *NullableTime) Set(val *time.Time) { + v.value = val + v.isSet = true +} + +func (v NullableTime) IsSet() bool { + return v.isSet +} + +func (v *NullableTime) Unset() { + v.value = nil + v.isSet = false +} + +func NewNullableTime(val *time.Time) *NullableTime { + return &NullableTime{value: val, isSet: true} +} + +func (v NullableTime) MarshalJSON() ([]byte, error) { + return v.value.MarshalJSON() +} + +func (v *NullableTime) UnmarshalJSON(src []byte) error { + v.isSet = true + return json.Unmarshal(src, &v.value) +} diff --git a/provider/README.md b/provider/README.md new file mode 100644 index 0000000000..ad5e103245 --- /dev/null +++ b/provider/README.md @@ -0,0 +1,30 @@ +## Usage + +Here's how you create, start, interact with, and stop the provider system: + +```golang +import ( + "context" + "time" + + "github.com/ipfs/go-libipfs/provider" + "github.com/ipfs/go-libipfs/provider/queue" + "github.com/ipfs/go-libipfs/provider/simple" +) + +rsys := (your routing system here) +dstore := (your datastore here) +cid := (your cid to provide here) + +q := queue.NewQueue(context.Background(), "example", dstore) + +reprov := simple.NewReprovider(context.Background(), time.Hour * 12, rsys, simple.NewBlockstoreProvider(dstore)) +prov := simple.NewProvider(context.Background(), q, rsys) +sys := provider.NewSystem(prov, reprov) + +sys.Run() + +sys.Provide(cid) + +sys.Close() +``` diff --git a/provider/batched/system.go b/provider/batched/system.go new file mode 100644 index 0000000000..35e4e6fcbc --- /dev/null +++ b/provider/batched/system.go @@ -0,0 +1,420 @@ +package batched + +import ( + "context" + "errors" + "fmt" + "strconv" + "sync" + "time" + + "github.com/ipfs/go-cid" + "github.com/ipfs/go-datastore" + provider "github.com/ipfs/go-libipfs/provider" + "github.com/ipfs/go-libipfs/provider/queue" + "github.com/ipfs/go-libipfs/provider/simple" + "github.com/ipfs/go-libipfs/verifcid" + logging "github.com/ipfs/go-log" + "github.com/multiformats/go-multihash" +) + +var log = logging.Logger("provider.batched") + +type BatchProvidingSystem struct { + ctx context.Context + close context.CancelFunc + closewg sync.WaitGroup + + reprovideInterval time.Duration + initalReprovideDelay time.Duration + initialReprovideDelaySet bool + + rsys provideMany + keyProvider simple.KeyChanFunc + + q *queue.Queue + ds datastore.Batching + + reprovideCh chan cid.Cid + + totalProvides, lastReprovideBatchSize int + avgProvideDuration, lastReprovideDuration time.Duration +} + +var _ provider.System = (*BatchProvidingSystem)(nil) + +type provideMany interface { + ProvideMany(ctx context.Context, keys []multihash.Multihash) error + Ready() bool +} + +// Option defines the functional option type that can be used to configure +// BatchProvidingSystem instances +type Option func(system *BatchProvidingSystem) error + +var lastReprovideKey = datastore.NewKey("/provider/reprovide/lastreprovide") + +func New(provider provideMany, q *queue.Queue, opts ...Option) (*BatchProvidingSystem, error) { + s := &BatchProvidingSystem{ + reprovideInterval: time.Hour * 24, + rsys: provider, + keyProvider: nil, + q: q, + ds: datastore.NewMapDatastore(), + reprovideCh: make(chan cid.Cid), + } + + for _, o := range opts { + if err := o(s); err != nil { + return nil, err + } + } + + // Setup default behavior for the initial reprovide delay + // + // If the reprovide ticker is larger than a minute (likely), + // provide once after we've been up a minute. + // + // Don't provide _immediately_ as we might be just about to stop. + if !s.initialReprovideDelaySet && s.reprovideInterval > time.Minute { + s.initalReprovideDelay = time.Minute + s.initialReprovideDelaySet = true + } + + if s.keyProvider == nil { + s.keyProvider = func(ctx context.Context) (<-chan cid.Cid, error) { + ch := make(chan cid.Cid) + close(ch) + return ch, nil + } + } + + // This is after the options processing so we do not have to worry about leaking a context if there is an + // initialization error processing the options + ctx, cancel := context.WithCancel(context.Background()) + s.ctx = ctx + s.close = cancel + + return s, nil +} + +func Datastore(batching datastore.Batching) Option { + return func(system *BatchProvidingSystem) error { + system.ds = batching + return nil + } +} + +func ReproviderInterval(duration time.Duration) Option { + return func(system *BatchProvidingSystem) error { + system.reprovideInterval = duration + return nil + } +} + +func KeyProvider(fn simple.KeyChanFunc) Option { + return func(system *BatchProvidingSystem) error { + system.keyProvider = fn + return nil + } +} + +func initialReprovideDelay(duration time.Duration) Option { + return func(system *BatchProvidingSystem) error { + system.initialReprovideDelaySet = true + system.initalReprovideDelay = duration + return nil + } +} + +func (s *BatchProvidingSystem) Run() { + // how long we wait between the first provider we hear about and batching up the provides to send out + const pauseDetectionThreshold = time.Millisecond * 500 + // how long we are willing to collect providers for the batch after we receive the first one + const maxCollectionDuration = time.Minute * 10 + + provCh := s.q.Dequeue() + + s.closewg.Add(1) + go func() { + defer s.closewg.Done() + + m := make(map[cid.Cid]struct{}) + + // setup stopped timers + maxCollectionDurationTimer := time.NewTimer(time.Hour) + pauseDetectTimer := time.NewTimer(time.Hour) + stopAndEmptyTimer(maxCollectionDurationTimer) + stopAndEmptyTimer(pauseDetectTimer) + + // make sure timers are cleaned up + defer maxCollectionDurationTimer.Stop() + defer pauseDetectTimer.Stop() + + resetTimersAfterReceivingProvide := func() { + firstProvide := len(m) == 0 + if firstProvide { + // after receiving the first provider start up the timers + maxCollectionDurationTimer.Reset(maxCollectionDuration) + pauseDetectTimer.Reset(pauseDetectionThreshold) + } else { + // otherwise just do a full restart of the pause timer + stopAndEmptyTimer(pauseDetectTimer) + pauseDetectTimer.Reset(pauseDetectionThreshold) + } + } + + for { + performedReprovide := false + + // at the start of every loop the maxCollectionDurationTimer and pauseDetectTimer should be already be + // stopped and have empty channels + loop: + for { + select { + case <-maxCollectionDurationTimer.C: + // if this timer has fired then the pause timer has started so let's stop and empty it + stopAndEmptyTimer(pauseDetectTimer) + break loop + default: + } + + select { + case c := <-provCh: + resetTimersAfterReceivingProvide() + m[c] = struct{}{} + continue + default: + } + + select { + case c := <-provCh: + resetTimersAfterReceivingProvide() + m[c] = struct{}{} + case c := <-s.reprovideCh: + resetTimersAfterReceivingProvide() + m[c] = struct{}{} + performedReprovide = true + case <-pauseDetectTimer.C: + // if this timer has fired then the max collection timer has started so let's stop and empty it + stopAndEmptyTimer(maxCollectionDurationTimer) + break loop + case <-maxCollectionDurationTimer.C: + // if this timer has fired then the pause timer has started so let's stop and empty it + stopAndEmptyTimer(pauseDetectTimer) + break loop + case <-s.ctx.Done(): + return + } + } + + if len(m) == 0 { + continue + } + + keys := make([]multihash.Multihash, 0, len(m)) + for c := range m { + delete(m, c) + + // hash security + if err := verifcid.ValidateCid(c); err != nil { + log.Errorf("insecure hash in reprovider, %s (%s)", c, err) + continue + } + + keys = append(keys, c.Hash()) + } + + // in case after removing all the invalid CIDs there are no valid ones left + if len(keys) == 0 { + continue + } + + for !s.rsys.Ready() { + log.Debugf("reprovider system not ready") + select { + case <-time.After(time.Minute): + case <-s.ctx.Done(): + return + } + } + + log.Debugf("starting provide of %d keys", len(keys)) + start := time.Now() + err := s.rsys.ProvideMany(s.ctx, keys) + if err != nil { + log.Debugf("providing failed %v", err) + continue + } + dur := time.Since(start) + + totalProvideTime := int64(s.totalProvides) * int64(s.avgProvideDuration) + recentAvgProvideDuration := time.Duration(int64(dur) / int64(len(keys))) + s.avgProvideDuration = time.Duration((totalProvideTime + int64(dur)) / int64(s.totalProvides+len(keys))) + s.totalProvides += len(keys) + + log.Debugf("finished providing of %d keys. It took %v with an average of %v per provide", len(keys), dur, recentAvgProvideDuration) + + if performedReprovide { + s.lastReprovideBatchSize = len(keys) + s.lastReprovideDuration = dur + + if err := s.ds.Put(s.ctx, lastReprovideKey, storeTime(time.Now())); err != nil { + log.Errorf("could not store last reprovide time: %v", err) + } + if err := s.ds.Sync(s.ctx, lastReprovideKey); err != nil { + log.Errorf("could not perform sync of last reprovide time: %v", err) + } + } + } + }() + + s.closewg.Add(1) + go func() { + defer s.closewg.Done() + + var initialReprovideCh, reprovideCh <-chan time.Time + + // If reproviding is enabled (non-zero) + if s.reprovideInterval > 0 { + reprovideTicker := time.NewTicker(s.reprovideInterval) + defer reprovideTicker.Stop() + reprovideCh = reprovideTicker.C + + // if there is a non-zero initial reprovide time that was set in the initializer or if the fallback has been + if s.initialReprovideDelaySet { + initialReprovideTimer := time.NewTimer(s.initalReprovideDelay) + defer initialReprovideTimer.Stop() + + initialReprovideCh = initialReprovideTimer.C + } + } + + for s.ctx.Err() == nil { + select { + case <-initialReprovideCh: + case <-reprovideCh: + case <-s.ctx.Done(): + return + } + + err := s.reprovide(s.ctx, false) + + // only log if we've hit an actual error, otherwise just tell the client we're shutting down + if s.ctx.Err() == nil && err != nil { + log.Errorf("failed to reprovide: %s", err) + } + } + }() +} + +func stopAndEmptyTimer(t *time.Timer) { + if !t.Stop() { + <-t.C + } +} + +func storeTime(t time.Time) []byte { + val := []byte(fmt.Sprintf("%d", t.UnixNano())) + return val +} + +func parseTime(b []byte) (time.Time, error) { + tns, err := strconv.ParseInt(string(b), 10, 64) + if err != nil { + return time.Time{}, err + } + return time.Unix(0, tns), nil +} + +func (s *BatchProvidingSystem) Close() error { + s.close() + err := s.q.Close() + s.closewg.Wait() + return err +} + +func (s *BatchProvidingSystem) Provide(cid cid.Cid) error { + return s.q.Enqueue(cid) +} + +func (s *BatchProvidingSystem) Reprovide(ctx context.Context) error { + return s.reprovide(ctx, true) +} + +func (s *BatchProvidingSystem) reprovide(ctx context.Context, force bool) error { + if !s.shouldReprovide() && !force { + return nil + } + + kch, err := s.keyProvider(ctx) + if err != nil { + return err + } + +reprovideCidLoop: + for { + select { + case c, ok := <-kch: + if !ok { + break reprovideCidLoop + } + + select { + case s.reprovideCh <- c: + case <-ctx.Done(): + return ctx.Err() + } + case <-ctx.Done(): + return ctx.Err() + } + } + + return nil +} + +func (s *BatchProvidingSystem) getLastReprovideTime() (time.Time, error) { + val, err := s.ds.Get(s.ctx, lastReprovideKey) + if errors.Is(err, datastore.ErrNotFound) { + return time.Time{}, nil + } + if err != nil { + return time.Time{}, fmt.Errorf("could not get last reprovide time") + } + + t, err := parseTime(val) + if err != nil { + return time.Time{}, fmt.Errorf("could not decode last reprovide time, got %q", string(val)) + } + + return t, nil +} + +func (s *BatchProvidingSystem) shouldReprovide() bool { + t, err := s.getLastReprovideTime() + if err != nil { + log.Debugf("getting last reprovide time failed: %s", err) + return false + } + + if time.Since(t) < time.Duration(float64(s.reprovideInterval)*0.5) { + return false + } + return true +} + +type BatchedProviderStats struct { + TotalProvides, LastReprovideBatchSize int + AvgProvideDuration, LastReprovideDuration time.Duration +} + +// Stat returns various stats about this provider system +func (s *BatchProvidingSystem) Stat(ctx context.Context) (BatchedProviderStats, error) { + // TODO: Does it matter that there is no locking around the total+average values? + return BatchedProviderStats{ + TotalProvides: s.totalProvides, + LastReprovideBatchSize: s.lastReprovideBatchSize, + AvgProvideDuration: s.avgProvideDuration, + LastReprovideDuration: s.lastReprovideDuration, + }, nil +} diff --git a/provider/batched/system_test.go b/provider/batched/system_test.go new file mode 100644 index 0000000000..0b42fb17a0 --- /dev/null +++ b/provider/batched/system_test.go @@ -0,0 +1,117 @@ +package batched + +import ( + "context" + "strconv" + "sync" + "testing" + "time" + + "github.com/ipfs/go-cid" + "github.com/ipfs/go-datastore" + dssync "github.com/ipfs/go-datastore/sync" + mh "github.com/multiformats/go-multihash" + + q "github.com/ipfs/go-libipfs/provider/queue" +) + +type mockProvideMany struct { + lk sync.Mutex + keys []mh.Multihash +} + +func (m *mockProvideMany) ProvideMany(ctx context.Context, keys []mh.Multihash) error { + m.lk.Lock() + defer m.lk.Unlock() + m.keys = keys + return nil +} + +func (m *mockProvideMany) Ready() bool { + return true +} + +func (m *mockProvideMany) GetKeys() []mh.Multihash { + m.lk.Lock() + defer m.lk.Unlock() + return m.keys[:] +} + +var _ provideMany = (*mockProvideMany)(nil) + +func TestBatched(t *testing.T) { + ctx := context.Background() + defer ctx.Done() + + ds := dssync.MutexWrap(datastore.NewMapDatastore()) + queue, err := q.NewQueue(ctx, "test", ds) + if err != nil { + t.Fatal(err) + } + + provider := &mockProvideMany{} + + ctx, cancel := context.WithTimeout(ctx, time.Second*10) + defer cancel() + + const numProvides = 100 + keysToProvide := make(map[cid.Cid]int) + for i := 0; i < numProvides; i++ { + h, err := mh.Sum([]byte(strconv.Itoa(i)), mh.SHA2_256, -1) + if err != nil { + panic(err) + } + c := cid.NewCidV1(cid.Raw, h) + keysToProvide[c] = i + } + + batchSystem, err := New(provider, queue, KeyProvider(func(ctx context.Context) (<-chan cid.Cid, error) { + ch := make(chan cid.Cid) + go func() { + for k := range keysToProvide { + select { + case ch <- k: + case <-ctx.Done(): + return + } + } + }() + return ch, nil + }), initialReprovideDelay(0)) + if err != nil { + t.Fatal(err) + } + + batchSystem.Run() + + var keys []mh.Multihash + for { + if ctx.Err() != nil { + t.Fatal("test hung") + } + keys = provider.GetKeys() + if len(keys) != 0 { + break + } + time.Sleep(time.Millisecond * 100) + } + + if len(keys) != numProvides { + t.Fatalf("expected %d provider keys, got %d", numProvides, len(keys)) + } + + provMap := make(map[string]struct{}) + for _, k := range keys { + provMap[string(k)] = struct{}{} + } + + for i := 0; i < numProvides; i++ { + h, err := mh.Sum([]byte(strconv.Itoa(i)), mh.SHA2_256, -1) + if err != nil { + panic(err) + } + if _, found := provMap[string(h)]; !found { + t.Fatalf("could not find provider with value %d", i) + } + } +} diff --git a/provider/offline.go b/provider/offline.go new file mode 100644 index 0000000000..030a70ab13 --- /dev/null +++ b/provider/offline.go @@ -0,0 +1,29 @@ +package provider + +import ( + "context" + + "github.com/ipfs/go-cid" +) + +type offlineProvider struct{} + +// NewOfflineProvider creates a ProviderSystem that does nothing +func NewOfflineProvider() System { + return &offlineProvider{} +} + +func (op *offlineProvider) Run() { +} + +func (op *offlineProvider) Close() error { + return nil +} + +func (op *offlineProvider) Provide(cid.Cid) error { + return nil +} + +func (op *offlineProvider) Reprovide(context.Context) error { + return nil +} diff --git a/provider/provider.go b/provider/provider.go new file mode 100644 index 0000000000..3b9c6ba3eb --- /dev/null +++ b/provider/provider.go @@ -0,0 +1,27 @@ +package provider + +import ( + "context" + + "github.com/ipfs/go-cid" +) + +// Provider announces blocks to the network +type Provider interface { + // Run is used to begin processing the provider work + Run() + // Provide takes a cid and makes an attempt to announce it to the network + Provide(cid.Cid) error + // Close stops the provider + Close() error +} + +// Reprovider reannounces blocks to the network +type Reprovider interface { + // Run is used to begin processing the reprovider work and waiting for reprovide triggers + Run() + // Trigger a reprovide + Trigger(context.Context) error + // Close stops the reprovider + Close() error +} diff --git a/provider/queue/queue.go b/provider/queue/queue.go new file mode 100644 index 0000000000..618256bbee --- /dev/null +++ b/provider/queue/queue.go @@ -0,0 +1,156 @@ +package queue + +import ( + "context" + "fmt" + cid "github.com/ipfs/go-cid" + datastore "github.com/ipfs/go-datastore" + namespace "github.com/ipfs/go-datastore/namespace" + query "github.com/ipfs/go-datastore/query" + logging "github.com/ipfs/go-log" +) + +var log = logging.Logger("provider.queue") + +// Queue provides a best-effort durability, FIFO interface to the datastore for storing cids +// +// Best-effort durability just means that cids in the process of being provided when a +// crash or shutdown occurs may be in the queue when the node is brought back online +// depending on whether the underlying datastore has synchronous or asynchronous writes. +type Queue struct { + // used to differentiate queues in datastore + // e.g. provider vs reprovider + name string + ctx context.Context + ds datastore.Datastore // Must be threadsafe + dequeue chan cid.Cid + enqueue chan cid.Cid + close context.CancelFunc + closed chan struct{} + + counter uint64 +} + +// NewQueue creates a queue for cids +func NewQueue(ctx context.Context, name string, ds datastore.Datastore) (*Queue, error) { + namespaced := namespace.Wrap(ds, datastore.NewKey("/"+name+"/queue/")) + cancelCtx, cancel := context.WithCancel(ctx) + q := &Queue{ + name: name, + ctx: cancelCtx, + ds: namespaced, + dequeue: make(chan cid.Cid), + enqueue: make(chan cid.Cid), + close: cancel, + closed: make(chan struct{}, 1), + } + q.work() + return q, nil +} + +// Close stops the queue +func (q *Queue) Close() error { + q.close() + <-q.closed + return nil +} + +// Enqueue puts a cid in the queue +func (q *Queue) Enqueue(cid cid.Cid) error { + select { + case q.enqueue <- cid: + return nil + case <-q.ctx.Done(): + return fmt.Errorf("failed to enqueue CID: shutting down") + } +} + +// Dequeue returns a channel that if listened to will remove entries from the queue +func (q *Queue) Dequeue() <-chan cid.Cid { + return q.dequeue +} + +// Run dequeues and enqueues when available. +func (q *Queue) work() { + go func() { + var k datastore.Key = datastore.Key{} + var c cid.Cid = cid.Undef + + defer func() { + // also cancels any in-progess enqueue tasks. + q.close() + // unblocks anyone waiting + close(q.dequeue) + // unblocks the close call + close(q.closed) + }() + + for { + if c == cid.Undef { + head, err := q.getQueueHead() + + if err != nil { + log.Errorf("error querying for head of queue: %s, stopping provider", err) + return + } else if head != nil { + k = datastore.NewKey(head.Key) + c, err = cid.Parse(head.Value) + if err != nil { + log.Warnf("error parsing queue entry cid with key (%s), removing it from queue: %s", head.Key, err) + err = q.ds.Delete(q.ctx, k) + if err != nil { + log.Errorf("error deleting queue entry with key (%s), due to error (%s), stopping provider", head.Key, err) + return + } + continue + } + } else { + c = cid.Undef + } + } + + // If c != cid.Undef set dequeue and attempt write, otherwise wait for enqueue + var dequeue chan cid.Cid + if c != cid.Undef { + dequeue = q.dequeue + } + + select { + case toQueue := <-q.enqueue: + keyPath := fmt.Sprintf("%020d/%s", q.counter, c.String()) + q.counter++ + nextKey := datastore.NewKey(keyPath) + + if err := q.ds.Put(q.ctx, nextKey, toQueue.Bytes()); err != nil { + log.Errorf("Failed to enqueue cid: %s", err) + continue + } + case dequeue <- c: + err := q.ds.Delete(q.ctx, k) + + if err != nil { + log.Errorf("Failed to delete queued cid %s with key %s: %s", c, k, err) + continue + } + c = cid.Undef + case <-q.ctx.Done(): + return + } + } + }() +} + +func (q *Queue) getQueueHead() (*query.Entry, error) { + qry := query.Query{Orders: []query.Order{query.OrderByKey{}}, Limit: 1} + results, err := q.ds.Query(q.ctx, qry) + if err != nil { + return nil, err + } + defer results.Close() + r, ok := results.NextSync() + if !ok { + return nil, nil + } + + return &r.Entry, r.Error +} diff --git a/provider/queue/queue_test.go b/provider/queue/queue_test.go new file mode 100644 index 0000000000..a0fa36c3ad --- /dev/null +++ b/provider/queue/queue_test.go @@ -0,0 +1,133 @@ +package queue + +import ( + "context" + "testing" + "time" + + "github.com/ipfs/go-cid" + "github.com/ipfs/go-datastore" + "github.com/ipfs/go-datastore/sync" + blocksutil "github.com/ipfs/go-ipfs-blocksutil" +) + +var blockGenerator = blocksutil.NewBlockGenerator() + +func makeCids(n int) []cid.Cid { + cids := make([]cid.Cid, 0, n) + for i := 0; i < n; i++ { + c := blockGenerator.Next().Cid() + cids = append(cids, c) + } + return cids +} + +func assertOrdered(cids []cid.Cid, q *Queue, t *testing.T) { + for _, c := range cids { + select { + case dequeued := <-q.dequeue: + if c != dequeued { + t.Fatalf("Error in ordering of CIDs retrieved from queue. Expected: %s, got: %s", c, dequeued) + } + + case <-time.After(time.Second * 1): + t.Fatal("Timeout waiting for cids to be provided.") + } + } +} + +func TestBasicOperation(t *testing.T) { + ctx := context.Background() + defer ctx.Done() + + ds := sync.MutexWrap(datastore.NewMapDatastore()) + queue, err := NewQueue(ctx, "test", ds) + if err != nil { + t.Fatal(err) + } + + cids := makeCids(10) + + for _, c := range cids { + queue.Enqueue(c) + } + + assertOrdered(cids, queue, t) +} + +func TestMangledData(t *testing.T) { + ctx := context.Background() + defer ctx.Done() + + ds := sync.MutexWrap(datastore.NewMapDatastore()) + queue, err := NewQueue(ctx, "test", ds) + if err != nil { + t.Fatal(err) + } + + cids := makeCids(10) + for _, c := range cids { + queue.Enqueue(c) + } + + // put bad data in the queue + queueKey := datastore.NewKey("/test/0") + err = queue.ds.Put(ctx, queueKey, []byte("borked")) + if err != nil { + t.Fatal(err) + } + + // expect to only see the valid cids we entered + expected := cids + assertOrdered(expected, queue, t) +} + +func TestInitialization(t *testing.T) { + ctx := context.Background() + defer ctx.Done() + + ds := sync.MutexWrap(datastore.NewMapDatastore()) + queue, err := NewQueue(ctx, "test", ds) + if err != nil { + t.Fatal(err) + } + + cids := makeCids(10) + for _, c := range cids { + queue.Enqueue(c) + } + + assertOrdered(cids[:5], queue, t) + + // make a new queue, same data + queue, err = NewQueue(ctx, "test", ds) + if err != nil { + t.Fatal(err) + } + + assertOrdered(cids[5:], queue, t) +} + +func TestInitializationWithManyCids(t *testing.T) { + ctx := context.Background() + defer ctx.Done() + + ds := sync.MutexWrap(datastore.NewMapDatastore()) + queue, err := NewQueue(ctx, "test", ds) + if err != nil { + t.Fatal(err) + } + + cids := makeCids(25) + for _, c := range cids { + queue.Enqueue(c) + } + + // make a new queue, same data + queue, err = NewQueue(ctx, "test", ds) + if err != nil { + t.Fatal(err) + } + + assertOrdered(cids, queue, t) +} diff --git a/provider/simple/provider.go b/provider/simple/provider.go new file mode 100644 index 0000000000..d632b947ac --- /dev/null +++ b/provider/simple/provider.go @@ -0,0 +1,116 @@ +// Package simple implements structures and methods to provide blocks, +// keep track of which blocks are provided, and to allow those blocks to +// be reprovided. +package simple + +import ( + "context" + "time" + + "github.com/ipfs/go-cid" + q "github.com/ipfs/go-libipfs/provider/queue" + logging "github.com/ipfs/go-log" + "github.com/libp2p/go-libp2p/core/routing" +) + +var logP = logging.Logger("provider.simple") + +// Provider announces blocks to the network +type Provider struct { + ctx context.Context + // the CIDs for which provide announcements should be made + queue *q.Queue + // used to announce providing to the network + contentRouting routing.ContentRouting + // how long to wait for announce to complete before giving up + timeout time.Duration + // how many workers concurrently work through thhe queue + workerLimit int +} + +// Option defines the functional option type that can be used to configure +// provider instances +type Option func(*Provider) + +// WithTimeout is an option to set a timeout on a provider +func WithTimeout(timeout time.Duration) Option { + return func(p *Provider) { + p.timeout = timeout + } +} + +// MaxWorkers is an option to set the max workers on a provider +func MaxWorkers(count int) Option { + return func(p *Provider) { + p.workerLimit = count + } +} + +// NewProvider creates a provider that announces blocks to the network using a content router +func NewProvider(ctx context.Context, queue *q.Queue, contentRouting routing.ContentRouting, options ...Option) *Provider { + p := &Provider{ + ctx: ctx, + queue: queue, + contentRouting: contentRouting, + workerLimit: 8, + } + + for _, option := range options { + option(p) + } + + return p +} + +// Close stops the provider +func (p *Provider) Close() error { + return p.queue.Close() +} + +// Run workers to handle provide requests. +func (p *Provider) Run() { + p.handleAnnouncements() +} + +// Provide the given cid using specified strategy. +func (p *Provider) Provide(root cid.Cid) error { + return p.queue.Enqueue(root) +} + +// Handle all outgoing cids by providing (announcing) them +func (p *Provider) handleAnnouncements() { + for workers := 0; workers < p.workerLimit; workers++ { + go func() { + for p.ctx.Err() == nil { + select { + case <-p.ctx.Done(): + return + case c, ok := <-p.queue.Dequeue(): + if !ok { + // queue closed. + return + } + + p.doProvide(c) + } + } + }() + } +} + +func (p *Provider) doProvide(c cid.Cid) { + ctx := p.ctx + if p.timeout > 0 { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, p.timeout) + defer cancel() + } else { + ctx = p.ctx + } + + logP.Info("announce - start - ", c) + if err := p.contentRouting.Provide(ctx, c, true); err != nil { + logP.Warnf("Unable to provide entry: %s, %s", c, err) + } + logP.Info("announce - end - ", c) +} diff --git a/provider/simple/provider_test.go b/provider/simple/provider_test.go new file mode 100644 index 0000000000..46ef2d0723 --- /dev/null +++ b/provider/simple/provider_test.go @@ -0,0 +1,162 @@ +package simple_test + +import ( + "context" + "math/rand" + "testing" + "time" + + "github.com/ipfs/go-cid" + "github.com/ipfs/go-datastore" + "github.com/ipfs/go-datastore/sync" + blocksutil "github.com/ipfs/go-ipfs-blocksutil" + "github.com/libp2p/go-libp2p/core/peer" + + q "github.com/ipfs/go-libipfs/provider/queue" + + . "github.com/ipfs/go-libipfs/provider/simple" +) + +var blockGenerator = blocksutil.NewBlockGenerator() + +type mockRouting struct { + provided chan cid.Cid +} + +func (r *mockRouting) Provide(ctx context.Context, cid cid.Cid, recursive bool) error { + select { + case r.provided <- cid: + case <-ctx.Done(): + panic("context cancelled, but shouldn't have") + } + return nil +} + +func (r *mockRouting) FindProvidersAsync(ctx context.Context, cid cid.Cid, timeout int) <-chan peer.AddrInfo { + return nil +} + +func mockContentRouting() *mockRouting { + r := mockRouting{} + r.provided = make(chan cid.Cid) + return &r +} + +func TestAnnouncement(t *testing.T) { + ctx := context.Background() + defer ctx.Done() + + ds := sync.MutexWrap(datastore.NewMapDatastore()) + queue, err := q.NewQueue(ctx, "test", ds) + if err != nil { + t.Fatal(err) + } + + r := mockContentRouting() + + prov := NewProvider(ctx, queue, r) + prov.Run() + + cids := cid.NewSet() + + for i := 0; i < 100; i++ { + c := blockGenerator.Next().Cid() + cids.Add(c) + } + + go func() { + for _, c := range cids.Keys() { + err = prov.Provide(c) + // A little goroutine stirring to exercise some different states + r := rand.Intn(10) + time.Sleep(time.Microsecond * time.Duration(r)) + } + }() + + for cids.Len() > 0 { + select { + case cp := <-r.provided: + if !cids.Has(cp) { + t.Fatal("Wrong CID provided") + } + cids.Remove(cp) + case <-time.After(time.Second * 5): + t.Fatal("Timeout waiting for cids to be provided.") + } + } + prov.Close() + + select { + case cp := <-r.provided: + t.Fatal("did not expect to provide CID: ", cp) + case <-time.After(time.Second * 1): + } +} + +func TestClose(t *testing.T) { + ctx := context.Background() + defer ctx.Done() + + ds := sync.MutexWrap(datastore.NewMapDatastore()) + queue, err := q.NewQueue(ctx, "test", ds) + if err != nil { + t.Fatal(err) + } + + r := mockContentRouting() + + prov := NewProvider(ctx, queue, r) + prov.Run() + + prov.Close() + + select { + case cp := <-r.provided: + t.Fatal("did not expect to provide anything, provided: ", cp) + case <-time.After(time.Second * 1): + } +} + +func TestAnnouncementTimeout(t *testing.T) { + ctx := context.Background() + defer ctx.Done() + + ds := sync.MutexWrap(datastore.NewMapDatastore()) + queue, err := q.NewQueue(ctx, "test", ds) + if err != nil { + t.Fatal(err) + } + + r := mockContentRouting() + + prov := NewProvider(ctx, queue, r, WithTimeout(1*time.Second)) + prov.Run() + + cids := cid.NewSet() + + for i := 0; i < 100; i++ { + c := blockGenerator.Next().Cid() + cids.Add(c) + } + + go func() { + for _, c := range cids.Keys() { + err = prov.Provide(c) + // A little goroutine stirring to exercise some different states + r := rand.Intn(10) + time.Sleep(time.Microsecond * time.Duration(r)) + } + }() + + for cids.Len() > 0 { + select { + case cp := <-r.provided: + if !cids.Has(cp) { + t.Fatal("Wrong CID provided") + } + cids.Remove(cp) + case <-time.After(time.Second * 5): + t.Fatal("Timeout waiting for cids to be provided.") + } + } +} diff --git a/provider/simple/reprovide.go b/provider/simple/reprovide.go new file mode 100644 index 0000000000..faac68f020 --- /dev/null +++ b/provider/simple/reprovide.go @@ -0,0 +1,255 @@ +package simple + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/cenkalti/backoff" + "github.com/ipfs/go-cid" + "github.com/ipfs/go-cidutil" + blocks "github.com/ipfs/go-libipfs/blockstore" + "github.com/ipfs/go-libipfs/fetcher" + fetcherhelpers "github.com/ipfs/go-libipfs/fetcher/helpers" + "github.com/ipfs/go-libipfs/verifcid" + logging "github.com/ipfs/go-log" + cidlink "github.com/ipld/go-ipld-prime/linking/cid" + "github.com/libp2p/go-libp2p/core/routing" +) + +var logR = logging.Logger("reprovider.simple") + +// ErrClosed is returned by Trigger when operating on a closed reprovider. +var ErrClosed = errors.New("reprovider service stopped") + +// KeyChanFunc is function streaming CIDs to pass to content routing +type KeyChanFunc func(context.Context) (<-chan cid.Cid, error) + +// Reprovider reannounces blocks to the network +type Reprovider struct { + // Reprovider context. Cancel to stop, then wait on closedCh. + ctx context.Context + cancel context.CancelFunc + closedCh chan struct{} + + // Trigger triggers a reprovide. + trigger chan chan<- error + + // The routing system to provide values through + rsys routing.ContentRouting + + keyProvider KeyChanFunc + + tick time.Duration +} + +// NewReprovider creates new Reprovider instance. +func NewReprovider(ctx context.Context, reprovideInterval time.Duration, rsys routing.ContentRouting, keyProvider KeyChanFunc) *Reprovider { + ctx, cancel := context.WithCancel(ctx) + return &Reprovider{ + ctx: ctx, + cancel: cancel, + closedCh: make(chan struct{}), + trigger: make(chan chan<- error), + + rsys: rsys, + keyProvider: keyProvider, + tick: reprovideInterval, + } +} + +// Close the reprovider +func (rp *Reprovider) Close() error { + rp.cancel() + <-rp.closedCh + return nil +} + +// Run re-provides keys with 'tick' interval or when triggered +func (rp *Reprovider) Run() { + defer close(rp.closedCh) + + var initialReprovideCh, reprovideCh <-chan time.Time + + // If reproviding is enabled (non-zero) + if rp.tick > 0 { + reprovideTicker := time.NewTicker(rp.tick) + defer reprovideTicker.Stop() + reprovideCh = reprovideTicker.C + + // If the reprovide ticker is larger than a minute (likely), + // provide once after we've been up a minute. + // + // Don't provide _immediately_ as we might be just about to stop. + if rp.tick > time.Minute { + initialReprovideTimer := time.NewTimer(time.Minute) + defer initialReprovideTimer.Stop() + + initialReprovideCh = initialReprovideTimer.C + } + } + + var done chan<- error + for rp.ctx.Err() == nil { + select { + case <-initialReprovideCh: + case <-reprovideCh: + case done = <-rp.trigger: + case <-rp.ctx.Done(): + return + } + + err := rp.Reprovide() + + // only log if we've hit an actual error, otherwise just tell the client we're shutting down + if rp.ctx.Err() != nil { + err = ErrClosed + } else if err != nil { + logR.Errorf("failed to reprovide: %s", err) + } + + if done != nil { + if err != nil { + done <- err + } + close(done) + } + } +} + +// Reprovide registers all keys given by rp.keyProvider to libp2p content routing +func (rp *Reprovider) Reprovide() error { + keychan, err := rp.keyProvider(rp.ctx) + if err != nil { + return fmt.Errorf("failed to get key chan: %s", err) + } + for c := range keychan { + // hash security + if err := verifcid.ValidateCid(c); err != nil { + logR.Errorf("insecure hash in reprovider, %s (%s)", c, err) + continue + } + op := func() error { + err := rp.rsys.Provide(rp.ctx, c, true) + if err != nil { + logR.Debugf("Failed to provide key: %s", err) + } + return err + } + + err := backoff.Retry(op, backoff.WithContext(backoff.NewExponentialBackOff(), rp.ctx)) + if err != nil { + logR.Debugf("Providing failed after number of retries: %s", err) + return err + } + } + return nil +} + +// Trigger starts the reprovision process in rp.Run and waits for it to finish. +// +// Returns an error if a reprovide is already in progress. +func (rp *Reprovider) Trigger(ctx context.Context) error { + resultCh := make(chan error, 1) + select { + case rp.trigger <- resultCh: + default: + return fmt.Errorf("reprovider is already running") + } + + select { + case err := <-resultCh: + return err + case <-rp.ctx.Done(): + return ErrClosed + case <-ctx.Done(): + return ctx.Err() + } +} + +// Strategies + +// NewBlockstoreProvider returns key provider using bstore.AllKeysChan +func NewBlockstoreProvider(bstore blocks.Blockstore) KeyChanFunc { + return func(ctx context.Context) (<-chan cid.Cid, error) { + return bstore.AllKeysChan(ctx) + } +} + +// Pinner interface defines how the simple.Reprovider wants to interact +// with a Pinning service +type Pinner interface { + DirectKeys(ctx context.Context) ([]cid.Cid, error) + RecursiveKeys(ctx context.Context) ([]cid.Cid, error) +} + +// NewPinnedProvider returns provider supplying pinned keys +func NewPinnedProvider(onlyRoots bool, pinning Pinner, fetchConfig fetcher.Factory) KeyChanFunc { + return func(ctx context.Context) (<-chan cid.Cid, error) { + set, err := pinSet(ctx, pinning, fetchConfig, onlyRoots) + if err != nil { + return nil, err + } + + outCh := make(chan cid.Cid) + go func() { + defer close(outCh) + for c := range set.New { + select { + case <-ctx.Done(): + return + case outCh <- c: + } + } + + }() + + return outCh, nil + } +} + +func pinSet(ctx context.Context, pinning Pinner, fetchConfig fetcher.Factory, onlyRoots bool) (*cidutil.StreamingSet, error) { + set := cidutil.NewStreamingSet() + + go func() { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + defer close(set.New) + + dkeys, err := pinning.DirectKeys(ctx) + if err != nil { + logR.Errorf("reprovide direct pins: %s", err) + return + } + for _, key := range dkeys { + set.Visitor(ctx)(key) + } + + rkeys, err := pinning.RecursiveKeys(ctx) + if err != nil { + logR.Errorf("reprovide indirect pins: %s", err) + return + } + + session := fetchConfig.NewSession(ctx) + for _, key := range rkeys { + set.Visitor(ctx)(key) + if !onlyRoots { + err := fetcherhelpers.BlockAll(ctx, session, cidlink.Link{Cid: key}, func(res fetcher.FetchResult) error { + clink, ok := res.LastBlockLink.(cidlink.Link) + if ok { + set.Visitor(ctx)(clink.Cid) + } + return nil + }) + if err != nil { + logR.Errorf("reprovide indirect pins: %s", err) + return + } + } + } + }() + + return set, nil +} diff --git a/provider/simple/reprovide_test.go b/provider/simple/reprovide_test.go new file mode 100644 index 0000000000..f23db4e9fe --- /dev/null +++ b/provider/simple/reprovide_test.go @@ -0,0 +1,284 @@ +package simple_test + +import ( + "bytes" + "context" + "testing" + "time" + + "github.com/ipfs/go-cid" + ds "github.com/ipfs/go-datastore" + dssync "github.com/ipfs/go-datastore/sync" + blocks "github.com/ipfs/go-libipfs/blocks" + bsrv "github.com/ipfs/go-libipfs/blockservice" + blockstore "github.com/ipfs/go-libipfs/blockstore" + offline "github.com/ipfs/go-libipfs/exchange/offline" + bsfetcher "github.com/ipfs/go-libipfs/fetcher/impl/blockservice" + mock "github.com/ipfs/go-libipfs/routing/mock" + "github.com/ipld/go-ipld-prime" + "github.com/ipld/go-ipld-prime/codec/dagcbor" + "github.com/ipld/go-ipld-prime/fluent/qp" + cidlink "github.com/ipld/go-ipld-prime/linking/cid" + basicnode "github.com/ipld/go-ipld-prime/node/basic" + testutil "github.com/libp2p/go-libp2p-testing/net" + "github.com/libp2p/go-libp2p/core/peer" + mh "github.com/multiformats/go-multihash" + + . "github.com/ipfs/go-libipfs/provider/simple" +) + +func setupRouting(t *testing.T) (clA, clB mock.Client, idA, idB peer.ID) { + mrserv := mock.NewServer() + + iidA := testutil.RandIdentityOrFatal(t) + iidB := testutil.RandIdentityOrFatal(t) + + clA = mrserv.Client(iidA) + clB = mrserv.Client(iidB) + + return clA, clB, iidA.ID(), iidB.ID() +} + +func setupDag(t *testing.T) (nodes []cid.Cid, bstore blockstore.Blockstore) { + bstore = blockstore.NewBlockstore(dssync.MutexWrap(ds.NewMapDatastore())) + for _, data := range []string{"foo", "bar"} { + nb := basicnode.Prototype.Any.NewBuilder() + err := nb.AssignString(data) + if err != nil { + t.Fatal(err) + } + blk := toBlock(t, nb.Build()) + err = bstore.Put(context.Background(), blk) + if err != nil { + t.Fatal(err) + } + nodes = append(nodes, blk.Cid()) + nd, err := qp.BuildMap(basicnode.Prototype.Map, 1, func(ma ipld.MapAssembler) { + qp.MapEntry(ma, "child", qp.Link(cidlink.Link{Cid: blk.Cid()})) + }) + if err != nil { + t.Fatal(err) + } + blk = toBlock(t, nd) + err = bstore.Put(context.Background(), blk) + if err != nil { + t.Fatal(err) + } + nodes = append(nodes, blk.Cid()) + } + + return nodes, bstore +} + +func toBlock(t *testing.T, nd ipld.Node) blocks.Block { + buf := new(bytes.Buffer) + err := dagcbor.Encode(nd, buf) + if err != nil { + t.Fatal(err) + } + c, err := cid.Prefix{ + Version: 1, + Codec: cid.DagCBOR, + MhType: mh.SHA2_256, + MhLength: -1, + }.Sum(buf.Bytes()) + if err != nil { + t.Fatal(err) + } + blk, err := blocks.NewBlockWithCid(buf.Bytes(), c) + if err != nil { + t.Fatal(err) + } + return blk +} + +func TestReprovide(t *testing.T) { + testReprovide(t, func(r *Reprovider, ctx context.Context) error { + return r.Reprovide() + }) +} + +func TestTrigger(t *testing.T) { + testReprovide(t, func(r *Reprovider, ctx context.Context) error { + go r.Run() + time.Sleep(1 * time.Second) + defer r.Close() + err := r.Trigger(ctx) + return err + }) +} + +func testReprovide(t *testing.T, trigger func(r *Reprovider, ctx context.Context) error) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + clA, clB, idA, _ := setupRouting(t) + nodes, bstore := setupDag(t) + + keyProvider := NewBlockstoreProvider(bstore) + reprov := NewReprovider(ctx, time.Hour, clA, keyProvider) + reprov.Trigger(context.Background()) + err := trigger(reprov, ctx) + if err != nil { + t.Fatal(err) + } + + var providers []peer.AddrInfo + maxProvs := 100 + + for _, c := range nodes { + // We provide raw cids because of the multihash keying + // FIXME(@Jorropo): I think this change should be done in the DHT layer, probably an issue with our routing mock. + b := c.Bytes() + b[1] = 0x55 // rewrite the cid to raw + _, c, err := cid.CidFromBytes(b) + if err != nil { + t.Fatal(err) + } + provChan := clB.FindProvidersAsync(ctx, c, maxProvs) + for p := range provChan { + providers = append(providers, p) + } + + if len(providers) == 0 { + t.Fatal("Should have gotten a provider") + } + + if providers[0].ID != idA { + t.Fatal("Somehow got the wrong peer back as a provider.") + } + } +} + +func TestTriggerTwice(t *testing.T) { + // Ensure we can only trigger once at a time. + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + clA, _, _, _ := setupRouting(t) + + keyCh := make(chan cid.Cid) + startCh := make(chan struct{}) + keyFunc := func(ctx context.Context) (<-chan cid.Cid, error) { + <-startCh + return keyCh, nil + } + + reprov := NewReprovider(ctx, time.Hour, clA, keyFunc) + go reprov.Run() + defer reprov.Close() + + // Wait for the reprovider to start, otherwise, the reprovider will + // think a concurrent reprovide is running. + // + // We _could_ fix this race... but that would be complexity for nothing. + // 1. We start a reprovide 1 minute after startup anyways. + // 2. The window is really narrow. + time.Sleep(1 * time.Second) + + errCh := make(chan error, 2) + + // Trigger in the background + go func() { + errCh <- reprov.Trigger(ctx) + }() + + // Wait for the trigger to really start. + startCh <- struct{}{} + + start := time.Now() + // Try to trigger again, this should fail immediately. + if err := reprov.Trigger(ctx); err == nil { + t.Fatal("expected an error") + } + if time.Since(start) > 10*time.Millisecond { + t.Fatal("expected reprovide to fail instantly") + } + + // Let the trigger progress. + close(keyCh) + + // Check the result. + err := <-errCh + if err != nil { + t.Fatal(err) + } + + // Try to trigger again, this should work. + go func() { + errCh <- reprov.Trigger(ctx) + }() + startCh <- struct{}{} + err = <-errCh + if err != nil { + t.Fatal(err) + } +} + +type mockPinner struct { + recursive []cid.Cid + direct []cid.Cid +} + +func (mp *mockPinner) DirectKeys(ctx context.Context) ([]cid.Cid, error) { + return mp.direct, nil +} + +func (mp *mockPinner) RecursiveKeys(ctx context.Context) ([]cid.Cid, error) { + return mp.recursive, nil +} + +func TestReprovidePinned(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + nodes, bstore := setupDag(t) + + fetchConfig := bsfetcher.NewFetcherConfig(bsrv.New(bstore, offline.Exchange(bstore))) + + for i := 0; i < 2; i++ { + clA, clB, idA, _ := setupRouting(t) + + onlyRoots := i == 0 + t.Logf("only roots: %v", onlyRoots) + + var provide, dont []cid.Cid + if onlyRoots { + provide = []cid.Cid{nodes[1], nodes[3]} + dont = []cid.Cid{nodes[0], nodes[2]} + } else { + provide = []cid.Cid{nodes[0], nodes[1], nodes[3]} + dont = []cid.Cid{nodes[2]} + } + + keyProvider := NewPinnedProvider(onlyRoots, &mockPinner{ + recursive: []cid.Cid{nodes[1]}, + direct: []cid.Cid{nodes[3]}, + }, fetchConfig) + + reprov := NewReprovider(ctx, time.Hour, clA, keyProvider) + err := reprov.Reprovide() + if err != nil { + t.Fatal(err) + } + + for i, c := range provide { + prov, ok := <-clB.FindProvidersAsync(ctx, c, 1) + if !ok { + t.Errorf("Should have gotten a provider for %d", i) + continue + } + + if prov.ID != idA { + t.Errorf("Somehow got the wrong peer back as a provider.") + continue + } + } + for i, c := range dont { + prov, ok := <-clB.FindProvidersAsync(ctx, c, 1) + if ok { + t.Fatalf("found provider %s for %d, expected none", prov.ID, i) + } + } + } +} diff --git a/provider/system.go b/provider/system.go new file mode 100644 index 0000000000..9fc3e8879c --- /dev/null +++ b/provider/system.go @@ -0,0 +1,60 @@ +package provider + +import ( + "context" + + "github.com/ipfs/go-cid" +) + +// System defines the interface for interacting with the value +// provider system +type System interface { + Run() + Close() error + Provide(cid.Cid) error + Reprovide(context.Context) error +} + +type system struct { + provider Provider + reprovider Reprovider +} + +// NewSystem constructs a new provider system from a provider and reprovider +func NewSystem(provider Provider, reprovider Reprovider) System { + return &system{provider, reprovider} +} + +// Run the provider system by running the provider and reprovider +func (s *system) Run() { + go s.provider.Run() + go s.reprovider.Run() +} + +// Close the provider and reprovider +func (s *system) Close() error { + var errs []error + + if err := s.provider.Close(); err != nil { + errs = append(errs, err) + } + + if err := s.reprovider.Close(); err != nil { + errs = append(errs, err) + } + + if len(errs) > 0 { + return errs[0] + } + return nil +} + +// Provide a value +func (s *system) Provide(cid cid.Cid) error { + return s.provider.Provide(cid) +} + +// Reprovide all the previously provided values +func (s *system) Reprovide(ctx context.Context) error { + return s.reprovider.Trigger(ctx) +} diff --git a/routing/http/client/client.go b/routing/http/client/client.go index 34f90eb6a4..96263ab4fb 100644 --- a/routing/http/client/client.go +++ b/routing/http/client/client.go @@ -11,7 +11,7 @@ import ( "github.com/benbjohnson/clock" "github.com/ipfs/go-cid" - ipns "github.com/ipfs/go-ipns" + ipns "github.com/ipfs/go-libipfs/ipns" "github.com/ipfs/go-libipfs/routing/http/contentrouter" "github.com/ipfs/go-libipfs/routing/http/internal/drjson" "github.com/ipfs/go-libipfs/routing/http/server" diff --git a/routing/mock/centralized_client.go b/routing/mock/centralized_client.go new file mode 100644 index 0000000000..ac3f938ccd --- /dev/null +++ b/routing/mock/centralized_client.go @@ -0,0 +1,88 @@ +package mockrouting + +import ( + "context" + "time" + + "github.com/ipfs/go-cid" + logging "github.com/ipfs/go-log" + tnet "github.com/libp2p/go-libp2p-testing/net" + "github.com/libp2p/go-libp2p/core/peer" + "github.com/libp2p/go-libp2p/core/routing" + ma "github.com/multiformats/go-multiaddr" +) + +var log = logging.Logger("mockrouter") + +type client struct { + vs routing.ValueStore + server server + peer tnet.Identity +} + +// PutValue FIXME(brian): is this method meant to simulate putting a value into the network? +func (c *client) PutValue(ctx context.Context, key string, val []byte, opts ...routing.Option) error { + log.Debugf("PutValue: %s", key) + return c.vs.PutValue(ctx, key, val, opts...) +} + +// GetValue FIXME(brian): is this method meant to simulate getting a value from the network? +func (c *client) GetValue(ctx context.Context, key string, opts ...routing.Option) ([]byte, error) { + log.Debugf("GetValue: %s", key) + return c.vs.GetValue(ctx, key, opts...) +} + +func (c *client) SearchValue(ctx context.Context, key string, opts ...routing.Option) (<-chan []byte, error) { + log.Debugf("SearchValue: %s", key) + return c.vs.SearchValue(ctx, key, opts...) +} + +func (c *client) FindProviders(ctx context.Context, key cid.Cid) ([]peer.AddrInfo, error) { + return c.server.Providers(key), nil +} + +func (c *client) FindPeer(ctx context.Context, pid peer.ID) (peer.AddrInfo, error) { + log.Debugf("FindPeer: %s", pid) + return peer.AddrInfo{}, nil +} + +func (c *client) FindProvidersAsync(ctx context.Context, k cid.Cid, max int) <-chan peer.AddrInfo { + out := make(chan peer.AddrInfo) + go func() { + defer close(out) + for i, p := range c.server.Providers(k) { + if max <= i { + return + } + select { + case out <- p: + case <-ctx.Done(): + return + } + } + }() + return out +} + +// Provide returns once the message is on the network. Value is not necessarily +// visible yet. +func (c *client) Provide(_ context.Context, key cid.Cid, brd bool) error { + if !brd { + return nil + } + info := peer.AddrInfo{ + ID: c.peer.ID(), + Addrs: []ma.Multiaddr{c.peer.Address()}, + } + return c.server.Announce(info, key) +} + +func (c *client) Ping(ctx context.Context, p peer.ID) (time.Duration, error) { + return 0, nil +} + +func (c *client) Bootstrap(context.Context) error { + return nil +} + +var _ routing.Routing = &client{} diff --git a/routing/mock/centralized_server.go b/routing/mock/centralized_server.go new file mode 100644 index 0000000000..31ee7c0bab --- /dev/null +++ b/routing/mock/centralized_server.go @@ -0,0 +1,91 @@ +package mockrouting + +import ( + "context" + "math/rand" + "sync" + "time" + + "github.com/ipfs/go-cid" + ds "github.com/ipfs/go-datastore" + dssync "github.com/ipfs/go-datastore/sync" + "github.com/ipfs/go-libipfs/routing/offline" + tnet "github.com/libp2p/go-libp2p-testing/net" + "github.com/libp2p/go-libp2p/core/peer" +) + +// server is the mockrouting.Client's private interface to the routing server +type server interface { + Announce(peer.AddrInfo, cid.Cid) error + Providers(cid.Cid) []peer.AddrInfo + + Server +} + +// s is an implementation of the private server interface +type s struct { + delayConf DelayConfig + + lock sync.RWMutex + providers map[string]map[peer.ID]providerRecord +} + +type providerRecord struct { + Peer peer.AddrInfo + Created time.Time +} + +func (rs *s) Announce(p peer.AddrInfo, c cid.Cid) error { + rs.lock.Lock() + defer rs.lock.Unlock() + + k := c.KeyString() + + _, ok := rs.providers[k] + if !ok { + rs.providers[k] = make(map[peer.ID]providerRecord) + } + rs.providers[k][p.ID] = providerRecord{ + Created: time.Now(), + Peer: p, + } + return nil +} + +func (rs *s) Providers(c cid.Cid) []peer.AddrInfo { + rs.delayConf.Query.Wait() // before locking + + rs.lock.RLock() + defer rs.lock.RUnlock() + k := c.KeyString() + + var ret []peer.AddrInfo + records, ok := rs.providers[k] + if !ok { + return ret + } + for _, r := range records { + if time.Since(r.Created) > rs.delayConf.ValueVisibility.Get() { + ret = append(ret, r.Peer) + } + } + + for i := range ret { + j := rand.Intn(i + 1) + ret[i], ret[j] = ret[j], ret[i] + } + + return ret +} + +func (rs *s) Client(p tnet.Identity) Client { + return rs.ClientWithDatastore(context.Background(), p, dssync.MutexWrap(ds.NewMapDatastore())) +} + +func (rs *s) ClientWithDatastore(_ context.Context, p tnet.Identity, datastore ds.Datastore) Client { + return &client{ + peer: p, + vs: offline.NewOfflineRouter(datastore, MockValidator{}), + server: rs, + } +} diff --git a/routing/mock/centralized_test.go b/routing/mock/centralized_test.go new file mode 100644 index 0000000000..44fd9bd644 --- /dev/null +++ b/routing/mock/centralized_test.go @@ -0,0 +1,176 @@ +package mockrouting + +import ( + "context" + "testing" + "time" + + "github.com/ipfs/go-cid" + delay "github.com/ipfs/go-ipfs-delay" + u "github.com/ipfs/go-libipfs/util" + tnet "github.com/libp2p/go-libp2p-testing/net" + "github.com/libp2p/go-libp2p/core/peer" +) + +func TestKeyNotFound(t *testing.T) { + + var pi = tnet.RandIdentityOrFatal(t) + var key = cid.NewCidV0(u.Hash([]byte("mock key"))) + var ctx = context.Background() + + rs := NewServer() + providers := rs.Client(pi).FindProvidersAsync(ctx, key, 10) + _, ok := <-providers + if ok { + t.Fatal("should be closed") + } +} + +func TestClientFindProviders(t *testing.T) { + pi := tnet.RandIdentityOrFatal(t) + rs := NewServer() + client := rs.Client(pi) + + k := cid.NewCidV0(u.Hash([]byte("hello"))) + err := client.Provide(context.Background(), k, true) + if err != nil { + t.Fatal(err) + } + + // This is bad... but simulating networks is hard + time.Sleep(time.Millisecond * 300) + max := 100 + + providersFromClient := client.FindProvidersAsync(context.Background(), k, max) + isInClient := false + for p := range providersFromClient { + if p.ID == pi.ID() { + isInClient = true + } + } + if !isInClient { + t.Fatal("Despite client providing key, client didn't receive peer when finding providers") + } +} + +func TestClientOverMax(t *testing.T) { + rs := NewServer() + k := cid.NewCidV0(u.Hash([]byte("hello"))) + numProvidersForHelloKey := 100 + for i := 0; i < numProvidersForHelloKey; i++ { + pi := tnet.RandIdentityOrFatal(t) + err := rs.Client(pi).Provide(context.Background(), k, true) + if err != nil { + t.Fatal(err) + } + } + + max := 10 + pi := tnet.RandIdentityOrFatal(t) + client := rs.Client(pi) + + providersFromClient := client.FindProvidersAsync(context.Background(), k, max) + i := 0 + for range providersFromClient { + i++ + } + if i != max { + t.Fatal("Too many providers returned") + } +} + +// TODO does dht ensure won't receive self as a provider? probably not. +func TestCanceledContext(t *testing.T) { + rs := NewServer() + k := cid.NewCidV0(u.Hash([]byte("hello"))) + + // avoid leaking goroutine, without using the context to signal + // (we want the goroutine to keep trying to publish on a + // cancelled context until we've tested it doesnt do anything.) + done := make(chan struct{}) + defer func() { done <- struct{}{} }() + + t.Log("async'ly announce infinite stream of providers for key") + i := 0 + go func() { // infinite stream + for { + select { + case <-done: + t.Log("exiting async worker") + return + default: + } + + pi, err := tnet.RandIdentity() + if err != nil { + t.Error(err) + } + err = rs.Client(pi).Provide(context.Background(), k, true) + if err != nil { + t.Error(err) + } + i++ + } + }() + + local := tnet.RandIdentityOrFatal(t) + client := rs.Client(local) + + t.Log("warning: max is finite so this test is non-deterministic") + t.Log("context cancellation could simply take lower priority") + t.Log("and result in receiving the max number of results") + max := 1000 + + t.Log("cancel the context before consuming") + ctx, cancelFunc := context.WithCancel(context.Background()) + cancelFunc() + providers := client.FindProvidersAsync(ctx, k, max) + + numProvidersReturned := 0 + for range providers { + numProvidersReturned++ + } + t.Log(numProvidersReturned) + + if numProvidersReturned == max { + t.Fatal("Context cancel had no effect") + } +} + +func TestValidAfter(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + pi := tnet.RandIdentityOrFatal(t) + key := cid.NewCidV0(u.Hash([]byte("mock key"))) + conf := DelayConfig{ + ValueVisibility: delay.Fixed(1 * time.Hour), + Query: delay.Fixed(0), + } + + rs := NewServerWithDelay(conf) + + rs.Client(pi).Provide(ctx, key, true) + + var providers []peer.AddrInfo + max := 100 + providersChan := rs.Client(pi).FindProvidersAsync(ctx, key, max) + for p := range providersChan { + providers = append(providers, p) + } + if len(providers) > 0 { + t.Fail() + } + + conf.ValueVisibility.Set(0) + time.Sleep(100 * time.Millisecond) + + providersChan = rs.Client(pi).FindProvidersAsync(ctx, key, max) + t.Log("providers", providers) + for p := range providersChan { + providers = append(providers, p) + } + if len(providers) != 1 { + t.Fail() + } +} diff --git a/routing/mock/interface.go b/routing/mock/interface.go new file mode 100644 index 0000000000..35430a72c5 --- /dev/null +++ b/routing/mock/interface.go @@ -0,0 +1,59 @@ +// Package mockrouting provides a virtual routing server. To use it, +// create a virtual routing server and use the Client() method to get a +// routing client (Routing). The server quacks like a DHT but is +// really a local in-memory hash table. +package mockrouting + +import ( + "context" + + ds "github.com/ipfs/go-datastore" + delay "github.com/ipfs/go-ipfs-delay" + tnet "github.com/libp2p/go-libp2p-testing/net" + "github.com/libp2p/go-libp2p/core/peer" + "github.com/libp2p/go-libp2p/core/routing" +) + +// MockValidator is a record validator that always returns success. +type MockValidator struct{} + +func (MockValidator) Validate(_ string, _ []byte) error { return nil } +func (MockValidator) Select(_ string, _ [][]byte) (int, error) { return 0, nil } + +// Server provides mockrouting Clients +type Server interface { + Client(p tnet.Identity) Client + ClientWithDatastore(context.Context, tnet.Identity, ds.Datastore) Client +} + +// Client implements Routing +type Client interface { + routing.Routing +} + +// NewServer returns a mockrouting Server +func NewServer() Server { + return NewServerWithDelay(DelayConfig{ + ValueVisibility: delay.Fixed(0), + Query: delay.Fixed(0), + }) +} + +// NewServerWithDelay returns a mockrouting Server with a delay! +func NewServerWithDelay(conf DelayConfig) Server { + return &s{ + providers: make(map[string]map[peer.ID]providerRecord), + delayConf: conf, + } +} + +// DelayConfig can be used to configured the fake delays of a mock server. +// Use with NewServerWithDelay(). +type DelayConfig struct { + // ValueVisibility is the time it takes for a value to be visible in the network + // FIXME there _must_ be a better term for this + ValueVisibility delay.D + + // Query is the time it takes to receive a response from a routing query + Query delay.D +} diff --git a/routing/none/none_client.go b/routing/none/none_client.go new file mode 100644 index 0000000000..6f400b54a1 --- /dev/null +++ b/routing/none/none_client.go @@ -0,0 +1,55 @@ +// Package nilrouting implements a routing client that does nothing. +package nilrouting + +import ( + "context" + "errors" + + "github.com/ipfs/go-cid" + ds "github.com/ipfs/go-datastore" + record "github.com/libp2p/go-libp2p-record" + "github.com/libp2p/go-libp2p/core/host" + "github.com/libp2p/go-libp2p/core/peer" + "github.com/libp2p/go-libp2p/core/routing" +) + +type nilclient struct { +} + +func (c *nilclient) PutValue(_ context.Context, _ string, _ []byte, _ ...routing.Option) error { + return nil +} + +func (c *nilclient) GetValue(_ context.Context, _ string, _ ...routing.Option) ([]byte, error) { + return nil, errors.New("tried GetValue from nil routing") +} + +func (c *nilclient) SearchValue(_ context.Context, _ string, _ ...routing.Option) (<-chan []byte, error) { + return nil, errors.New("tried SearchValue from nil routing") +} + +func (c *nilclient) FindPeer(_ context.Context, _ peer.ID) (peer.AddrInfo, error) { + return peer.AddrInfo{}, nil +} + +func (c *nilclient) FindProvidersAsync(_ context.Context, _ cid.Cid, _ int) <-chan peer.AddrInfo { + out := make(chan peer.AddrInfo) + defer close(out) + return out +} + +func (c *nilclient) Provide(_ context.Context, _ cid.Cid, _ bool) error { + return nil +} + +func (c *nilclient) Bootstrap(_ context.Context) error { + return nil +} + +// ConstructNilRouting creates an Routing client which does nothing. +func ConstructNilRouting(_ context.Context, _ host.Host, _ ds.Batching, _ record.Validator) (routing.Routing, error) { + return &nilclient{}, nil +} + +// ensure nilclient satisfies interface +var _ routing.Routing = &nilclient{} diff --git a/routing/offline/offline.go b/routing/offline/offline.go new file mode 100644 index 0000000000..e6f4ee6ecd --- /dev/null +++ b/routing/offline/offline.go @@ -0,0 +1,126 @@ +// Package offline implements Routing with a client which +// is only able to perform offline operations. +package offline + +import ( + "bytes" + "context" + "errors" + "time" + + "github.com/gogo/protobuf/proto" + "github.com/ipfs/go-cid" + ds "github.com/ipfs/go-datastore" + dshelp "github.com/ipfs/go-libipfs/datastore/dshelp" + record "github.com/libp2p/go-libp2p-record" + pb "github.com/libp2p/go-libp2p-record/pb" + "github.com/libp2p/go-libp2p/core/peer" + "github.com/libp2p/go-libp2p/core/routing" +) + +// ErrOffline is returned when trying to perform operations that +// require connectivity. +var ErrOffline = errors.New("routing system in offline mode") + +// NewOfflineRouter returns an Routing implementation which only performs +// offline operations. It allows to Put and Get signed dht +// records to and from the local datastore. +func NewOfflineRouter(dstore ds.Datastore, validator record.Validator) routing.Routing { + return &offlineRouting{ + datastore: dstore, + validator: validator, + } +} + +// offlineRouting implements the Routing interface, +// but only provides the capability to Put and Get signed dht +// records to and from the local datastore. +type offlineRouting struct { + datastore ds.Datastore + validator record.Validator +} + +func (c *offlineRouting) PutValue(ctx context.Context, key string, val []byte, _ ...routing.Option) error { + if err := c.validator.Validate(key, val); err != nil { + return err + } + if old, err := c.GetValue(ctx, key); err == nil { + // be idempotent to be nice. + if bytes.Equal(old, val) { + return nil + } + // check to see if the older record is better + i, err := c.validator.Select(key, [][]byte{val, old}) + if err != nil { + // this shouldn't happen for validated records. + return err + } + if i != 0 { + return errors.New("can't replace a newer record with an older one") + } + } + rec := record.MakePutRecord(key, val) + data, err := proto.Marshal(rec) + if err != nil { + return err + } + + return c.datastore.Put(ctx, dshelp.NewKeyFromBinary([]byte(key)), data) +} + +func (c *offlineRouting) GetValue(ctx context.Context, key string, _ ...routing.Option) ([]byte, error) { + buf, err := c.datastore.Get(ctx, dshelp.NewKeyFromBinary([]byte(key))) + if err != nil { + return nil, err + } + + rec := new(pb.Record) + err = proto.Unmarshal(buf, rec) + if err != nil { + return nil, err + } + val := rec.GetValue() + + err = c.validator.Validate(key, val) + if err != nil { + return nil, err + } + return val, nil +} + +func (c *offlineRouting) SearchValue(ctx context.Context, key string, _ ...routing.Option) (<-chan []byte, error) { + out := make(chan []byte, 1) + go func() { + defer close(out) + v, err := c.GetValue(ctx, key) + if err == nil { + out <- v + } + }() + return out, nil +} + +func (c *offlineRouting) FindPeer(ctx context.Context, pid peer.ID) (peer.AddrInfo, error) { + return peer.AddrInfo{}, ErrOffline +} + +func (c *offlineRouting) FindProvidersAsync(ctx context.Context, k cid.Cid, max int) <-chan peer.AddrInfo { + out := make(chan peer.AddrInfo) + close(out) + return out +} + +func (c *offlineRouting) Provide(_ context.Context, k cid.Cid, _ bool) error { + return ErrOffline +} + +func (c *offlineRouting) Ping(ctx context.Context, p peer.ID) (time.Duration, error) { + return 0, ErrOffline +} + +func (c *offlineRouting) Bootstrap(context.Context) error { + return nil +} + +// ensure offlineRouting matches the Routing interface +var _ routing.Routing = &offlineRouting{} diff --git a/routing/offline/offline_test.go b/routing/offline/offline_test.go new file mode 100644 index 0000000000..9a17e86898 --- /dev/null +++ b/routing/offline/offline_test.go @@ -0,0 +1,91 @@ +package offline + +import ( + "bytes" + "context" + "testing" + + cid "github.com/ipfs/go-cid" + ds "github.com/ipfs/go-datastore" + + "github.com/libp2p/go-libp2p/core/routing" + "github.com/libp2p/go-libp2p/core/test" + + mh "github.com/multiformats/go-multihash" +) + +type blankValidator struct{} + +func (blankValidator) Validate(_ string, _ []byte) error { return nil } +func (blankValidator) Select(_ string, _ [][]byte) (int, error) { return 0, nil } + +func TestOfflineRouterStorage(t *testing.T) { + ctx := context.Background() + + nds := ds.NewMapDatastore() + offline := NewOfflineRouter(nds, blankValidator{}) + + if err := offline.PutValue(ctx, "key", []byte("testing 1 2 3")); err != nil { + t.Fatal(err) + } + + val, err := offline.GetValue(ctx, "key") + if err != nil { + t.Fatal(err) + } + if !bytes.Equal([]byte("testing 1 2 3"), val) { + t.Fatal("OfflineRouter does not properly store") + } + + _, err = offline.GetValue(ctx, "notHere") + if err == nil { + t.Fatal("Router should throw errors for unfound records") + } + + local, err := offline.GetValue(ctx, "key", routing.Offline) + if err != nil { + t.Fatal(err) + } + + _, err = offline.GetValue(ctx, "notHere", routing.Offline) + if err == nil { + t.Fatal("Router should throw errors for unfound records") + } + + if !bytes.Equal([]byte("testing 1 2 3"), local) { + t.Fatal("OfflineRouter does not properly store") + } +} + +func TestOfflineRouterLocal(t *testing.T) { + ctx := context.Background() + + nds := ds.NewMapDatastore() + offline := NewOfflineRouter(nds, blankValidator{}) + + id, _ := test.RandPeerID() + _, err := offline.FindPeer(ctx, id) + if err != ErrOffline { + t.Fatal("OfflineRouting should alert that its offline") + } + + h, _ := mh.Sum([]byte("test data1"), mh.SHA2_256, -1) + c1 := cid.NewCidV0(h) + pChan := offline.FindProvidersAsync(ctx, c1, 1) + p, ok := <-pChan + if ok { + t.Fatalf("FindProvidersAsync did not return a closed channel. Instead we got %+v !", p) + } + + h2, _ := mh.Sum([]byte("test data1"), mh.SHA2_256, -1) + c2 := cid.NewCidV0(h2) + err = offline.Provide(ctx, c2, true) + if err != ErrOffline { + t.Fatal("OfflineRouting should alert that its offline") + } + + err = offline.Bootstrap(ctx) + if err != nil { + t.Fatal("You shouldn't be able to bootstrap offline routing.") + } +} diff --git a/tar/extractor_test.go b/tar/extractor_test.go index 717c65d196..0ce1b99383 100644 --- a/tar/extractor_test.go +++ b/tar/extractor_test.go @@ -357,7 +357,7 @@ func testExtract(t *testing.T, tarFile string, extractDir string, expectedError } // Based on the `writeXXXHeader` family of functions in -// github.com/ipfs/go-ipfs-files@v0.0.8/tarwriter.go. +// github.com/ipfs/go-libipfs/files@v0.0.8/tarwriter.go. func writeTarFile(t *testing.T, path string, entries []tarEntry) { tarFile, err := os.Create(path) assert.NoError(t, err) diff --git a/unixfs/README.md b/unixfs/README.md new file mode 100644 index 0000000000..6c80d07c8c --- /dev/null +++ b/unixfs/README.md @@ -0,0 +1,28 @@ +## Package Directory +This package contains many subpackages, each of which can be very large on its own. + +### Top Level +The top level unixfs package defines the unixfs format datastructures, and some helper methods around it. + +### importers +The `importer` subpackage is what you'll use when you want to turn a normal file into a unixfs file. + +### io +The `io` subpackage provides helpers for reading files and manipulating directories. The `DagReader` takes a +reference to a unixfs file and returns a file handle that can be read from and seeked through. The `Directory` +interface allows you to easily read items in a directory, add items to a directory, and do lookups. + +### mod +The `mod` subpackage implements a `DagModifier` type that can be used to write to an existing unixfs file, or +create a new one. The logic for this is significantly more complicated than for the dagreader, so its a separate +type. (TODO: maybe it still belongs in the `io` subpackage though?) + +### hamt +The `hamt` subpackage implements a CHAMP hamt that is used in unixfs directory sharding. + +### archive +The `archive` subpackage implements a `tar` importer and exporter. The objects created here are not officially unixfs, +but in the future, this may be integrated more directly. + +### test +The `test` subpackage provides several utilities to make testing unixfs related things easier. diff --git a/unixfs/file/unixfile.go b/unixfs/file/unixfile.go new file mode 100644 index 0000000000..3d7e1a9b46 --- /dev/null +++ b/unixfs/file/unixfile.go @@ -0,0 +1,183 @@ +package unixfile + +import ( + "context" + "errors" + + ft "github.com/ipfs/go-libipfs/unixfs" + uio "github.com/ipfs/go-libipfs/unixfs/io" + + ipld "github.com/ipfs/go-ipld-format" + "github.com/ipfs/go-libipfs/files" + dag "github.com/ipfs/go-merkledag" +) + +// Number to file to prefetch in directories +// TODO: should we allow setting this via context hint? +const prefetchFiles = 4 + +type ufsDirectory struct { + ctx context.Context + dserv ipld.DAGService + dir uio.Directory + size int64 +} + +type ufsIterator struct { + ctx context.Context + files chan *ipld.Link + dserv ipld.DAGService + + curName string + curFile files.Node + + err error + errCh chan error +} + +func (it *ufsIterator) Name() string { + return it.curName +} + +func (it *ufsIterator) Node() files.Node { + return it.curFile +} + +func (it *ufsIterator) Next() bool { + if it.err != nil { + return false + } + + var l *ipld.Link + var ok bool + for !ok { + if it.files == nil && it.errCh == nil { + return false + } + select { + case l, ok = <-it.files: + if !ok { + it.files = nil + } + case err := <-it.errCh: + it.errCh = nil + it.err = err + + if err != nil { + return false + } + } + } + + it.curFile = nil + + nd, err := l.GetNode(it.ctx, it.dserv) + if err != nil { + it.err = err + return false + } + + it.curName = l.Name + it.curFile, it.err = NewUnixfsFile(it.ctx, it.dserv, nd) + return it.err == nil +} + +func (it *ufsIterator) Err() error { + return it.err +} + +func (d *ufsDirectory) Close() error { + return nil +} + +func (d *ufsDirectory) Entries() files.DirIterator { + fileCh := make(chan *ipld.Link, prefetchFiles) + errCh := make(chan error, 1) + go func() { + errCh <- d.dir.ForEachLink(d.ctx, func(link *ipld.Link) error { + if d.ctx.Err() != nil { + return d.ctx.Err() + } + select { + case fileCh <- link: + case <-d.ctx.Done(): + return d.ctx.Err() + } + return nil + }) + + close(errCh) + close(fileCh) + }() + + return &ufsIterator{ + ctx: d.ctx, + files: fileCh, + errCh: errCh, + dserv: d.dserv, + } +} + +func (d *ufsDirectory) Size() (int64, error) { + return d.size, nil +} + +type ufsFile struct { + uio.DagReader +} + +func (f *ufsFile) Size() (int64, error) { + return int64(f.DagReader.Size()), nil +} + +func newUnixfsDir(ctx context.Context, dserv ipld.DAGService, nd *dag.ProtoNode) (files.Directory, error) { + dir, err := uio.NewDirectoryFromNode(dserv, nd) + if err != nil { + return nil, err + } + + size, err := nd.Size() + if err != nil { + return nil, err + } + + return &ufsDirectory{ + ctx: ctx, + dserv: dserv, + + dir: dir, + size: int64(size), + }, nil +} + +func NewUnixfsFile(ctx context.Context, dserv ipld.DAGService, nd ipld.Node) (files.Node, error) { + switch dn := nd.(type) { + case *dag.ProtoNode: + fsn, err := ft.FSNodeFromBytes(dn.Data()) + if err != nil { + return nil, err + } + if fsn.IsDir() { + return newUnixfsDir(ctx, dserv, dn) + } + if fsn.Type() == ft.TSymlink { + return files.NewLinkFile(string(fsn.Data()), nil), nil + } + + case *dag.RawNode: + default: + return nil, errors.New("unknown node type") + } + + dr, err := uio.NewDagReader(ctx, nd, dserv) + if err != nil { + return nil, err + } + + return &ufsFile{ + DagReader: dr, + }, nil +} + +var _ files.Directory = &ufsDirectory{} +var _ files.File = &ufsFile{} diff --git a/unixfs/hamt/hamt.go b/unixfs/hamt/hamt.go new file mode 100644 index 0000000000..ab6cf6455f --- /dev/null +++ b/unixfs/hamt/hamt.go @@ -0,0 +1,946 @@ +// Package hamt implements a Hash Array Mapped Trie over ipfs merkledag nodes. +// It is implemented mostly as described in the wikipedia article on HAMTs, +// however the table size is variable (usually 256 in our usages) as opposed to +// 32 as suggested in the article. The hash function used is currently +// Murmur3, but this value is configurable (the datastructure reports which +// hash function its using). +// +// The one algorithmic change we implement that is not mentioned in the +// wikipedia article is the collapsing of empty shards. +// Given the following tree: ( '[' = shards, '{' = values ) +// [ 'A' ] -> [ 'B' ] -> { "ABC" } +// +// | L-> { "ABD" } +// L-> { "ASDF" } +// +// If we simply removed "ABC", we would end up with a tree where shard 'B' only +// has a single child. This causes two issues, the first, is that now we have +// an extra lookup required to get to "ABD". The second issue is that now we +// have a tree that contains only "ABD", but is not the same tree that we would +// get by simply inserting "ABD" into a new tree. To address this, we always +// check for empty shard nodes upon deletion and prune them to maintain a +// consistent tree, independent of insertion order. +package hamt + +import ( + "context" + "fmt" + "os" + "sync" + + "golang.org/x/sync/errgroup" + + format "github.com/ipfs/go-libipfs/unixfs" + "github.com/ipfs/go-libipfs/unixfs/internal" + + bitfield "github.com/ipfs/go-bitfield" + cid "github.com/ipfs/go-cid" + ipld "github.com/ipfs/go-ipld-format" + dag "github.com/ipfs/go-merkledag" +) + +const ( + // HashMurmur3 is the multiformats identifier for Murmur3 + HashMurmur3 uint64 = 0x22 +) + +func init() { + internal.HAMTHashFunction = murmur3Hash +} + +func (ds *Shard) isValueNode() bool { + return ds.key != "" && ds.val != nil +} + +// A Shard represents the HAMT. It should be initialized with NewShard(). +type Shard struct { + childer *childer + + // Entries per node (number of possible childs indexed by the partial key). + tableSize int + // Bits needed to encode child indexes (log2 of number of entries). This is + // the number of bits taken from the hash key on each level of the tree. + tableSizeLg2 int + + builder cid.Builder + hashFunc uint64 + + // String format with number of zeros that will be present in the hexadecimal + // encoding of the child index to always reach the fixed maxpadlen chars. + // Example: maxpadlen = 4 => prefixPadStr: "%04X" (print number in hexadecimal + // format padding with zeros to always reach 4 characters). + prefixPadStr string + // Length in chars of string that encodes child indexes. We encode indexes + // as hexadecimal strings to this is log4 of number of entries. + maxpadlen int + + dserv ipld.DAGService + + // FIXME: Remove. We don't actually store "value nodes". This confusing + // abstraction just removes the maxpadlen from the link names to extract + // the actual value link the trie is storing. + // leaf node + key string + val *ipld.Link +} + +// NewShard creates a new, empty HAMT shard with the given size. +func NewShard(dserv ipld.DAGService, size int) (*Shard, error) { + return NewShardValue(dserv, size, "", nil) +} + +// NewShardValue creates a new, empty HAMT shard with the given key, value and size. +func NewShardValue(dserv ipld.DAGService, size int, key string, value *ipld.Link) (*Shard, error) { + ds, err := makeShard(dserv, size, key, value) + if err != nil { + return nil, err + } + + // FIXME: Make this at least a static configuration for testing. + ds.hashFunc = HashMurmur3 + return ds, nil +} + +func makeShard(ds ipld.DAGService, size int, key string, val *ipld.Link) (*Shard, error) { + lg2s, err := Logtwo(size) + if err != nil { + return nil, err + } + childer, err := newChilder(ds, size) + if err != nil { + return nil, err + } + maxpadding := fmt.Sprintf("%X", size-1) + s := &Shard{ + tableSizeLg2: lg2s, + prefixPadStr: fmt.Sprintf("%%0%dX", len(maxpadding)), + maxpadlen: len(maxpadding), + childer: childer, + tableSize: size, + dserv: ds, + + key: key, + val: val, + } + + s.childer.sd = s + + return s, nil +} + +// NewHamtFromDag creates new a HAMT shard from the given DAG. +func NewHamtFromDag(dserv ipld.DAGService, nd ipld.Node) (*Shard, error) { + pbnd, ok := nd.(*dag.ProtoNode) + if !ok { + return nil, dag.ErrNotProtobuf + } + + fsn, err := format.FSNodeFromBytes(pbnd.Data()) + if err != nil { + return nil, err + } + + if fsn.Type() != format.THAMTShard { + return nil, fmt.Errorf("node was not a dir shard") + } + + if fsn.HashType() != HashMurmur3 { + return nil, fmt.Errorf("only murmur3 supported as hash function") + } + + size := int(fsn.Fanout()) + + ds, err := makeShard(dserv, size, "", nil) + if err != nil { + return nil, err + } + + ds.childer.makeChilder(fsn.Data(), pbnd.Links()) + + ds.hashFunc = fsn.HashType() + ds.builder = pbnd.CidBuilder() + + return ds, nil +} + +// SetCidBuilder sets the CID Builder +func (ds *Shard) SetCidBuilder(builder cid.Builder) { + ds.builder = builder +} + +// CidBuilder gets the CID Builder, may be nil if unset +func (ds *Shard) CidBuilder() cid.Builder { + return ds.builder +} + +// Node serializes the HAMT structure into a merkledag node with unixfs formatting +func (ds *Shard) Node() (ipld.Node, error) { + out := new(dag.ProtoNode) + out.SetCidBuilder(ds.builder) + + sliceIndex := 0 + // TODO: optimized 'for each set bit' + for childIndex := 0; childIndex < ds.tableSize; childIndex++ { + if !ds.childer.has(childIndex) { + continue + } + + ch := ds.childer.child(sliceIndex) + if ch != nil { + clnk, err := ch.Link() + if err != nil { + return nil, err + } + + err = out.AddRawLink(ds.linkNamePrefix(childIndex)+ch.key, clnk) + if err != nil { + return nil, err + } + } else { + // child unloaded, just copy in link with updated name + lnk := ds.childer.link(sliceIndex) + label := lnk.Name[ds.maxpadlen:] + + err := out.AddRawLink(ds.linkNamePrefix(childIndex)+label, lnk) + if err != nil { + return nil, err + } + } + sliceIndex++ + } + + data, err := format.HAMTShardData(ds.childer.bitfield.Bytes(), uint64(ds.tableSize), HashMurmur3) + if err != nil { + return nil, err + } + + out.SetData(data) + + err = ds.dserv.Add(context.TODO(), out) + if err != nil { + return nil, err + } + + return out, nil +} + +func (ds *Shard) makeShardValue(lnk *ipld.Link) (*Shard, error) { + lnk2 := *lnk + s, err := makeShard(ds.dserv, ds.tableSize, "", nil) + if err != nil { + return nil, err + } + + s.key = lnk.Name[ds.maxpadlen:] + s.val = &lnk2 + + return s, nil +} + +// Set sets 'name' = nd in the HAMT +func (ds *Shard) Set(ctx context.Context, name string, nd ipld.Node) error { + _, err := ds.Swap(ctx, name, nd) + return err +} + +// Set sets 'name' = nd in the HAMT, using directly the information in the +// given link. This avoids writing the given node, then reading it to making a +// link out of it. +func (ds *Shard) SetLink(ctx context.Context, name string, lnk *ipld.Link) error { + hv := newHashBits(name) + + newLink := ipld.Link{ + Name: lnk.Name, + Size: lnk.Size, + Cid: lnk.Cid, + } + + // FIXME: We don't need to set the name here, it will get overwritten. + // This is confusing, confirm and remove this line. + newLink.Name = ds.linkNamePrefix(0) + name + + _, err := ds.swapValue(ctx, hv, name, &newLink) + return err +} + +// Swap sets a link pointing to the passed node as the value under the +// name key in this Shard or its children. It also returns the previous link +// under that name key (if any). +func (ds *Shard) Swap(ctx context.Context, name string, node ipld.Node) (*ipld.Link, error) { + hv := newHashBits(name) + err := ds.dserv.Add(ctx, node) + if err != nil { + return nil, err + } + + lnk, err := ipld.MakeLink(node) + if err != nil { + return nil, err + } + + // FIXME: We don't need to set the name here, it will get overwritten. + // This is confusing, confirm and remove this line. + lnk.Name = ds.linkNamePrefix(0) + name + + return ds.swapValue(ctx, hv, name, lnk) +} + +// Remove deletes the named entry if it exists. Otherwise, it returns +// os.ErrNotExist. +func (ds *Shard) Remove(ctx context.Context, name string) error { + _, err := ds.Take(ctx, name) + return err +} + +// Take is similar to the public Remove but also returns the +// old removed link (if it exists). +func (ds *Shard) Take(ctx context.Context, name string) (*ipld.Link, error) { + hv := newHashBits(name) + return ds.swapValue(ctx, hv, name, nil) +} + +// Find searches for a child node by 'name' within this hamt +func (ds *Shard) Find(ctx context.Context, name string) (*ipld.Link, error) { + hv := newHashBits(name) + + var out *ipld.Link + err := ds.getValue(ctx, hv, name, func(sv *Shard) error { + out = sv.val + return nil + }) + if err != nil { + return nil, err + } + + return out, nil +} + +type linkType int + +const ( + invalidLink linkType = iota + shardLink + shardValueLink +) + +func (ds *Shard) childLinkType(lnk *ipld.Link) (linkType, error) { + if len(lnk.Name) < ds.maxpadlen { + return invalidLink, fmt.Errorf("invalid link name '%s'", lnk.Name) + } + if len(lnk.Name) == ds.maxpadlen { + return shardLink, nil + } + return shardValueLink, nil +} + +// Link returns a merklelink to this shard node +func (ds *Shard) Link() (*ipld.Link, error) { + if ds.isValueNode() { + return ds.val, nil + } + + nd, err := ds.Node() + if err != nil { + return nil, err + } + + err = ds.dserv.Add(context.TODO(), nd) + if err != nil { + return nil, err + } + + return ipld.MakeLink(nd) +} + +func (ds *Shard) getValue(ctx context.Context, hv *hashBits, key string, cb func(*Shard) error) error { + childIndex, err := hv.Next(ds.tableSizeLg2) + if err != nil { + return err + } + + if ds.childer.has(childIndex) { + child, err := ds.childer.get(ctx, ds.childer.sliceIndex(childIndex)) + if err != nil { + return err + } + + if child.isValueNode() { + if child.key == key { + return cb(child) + } + } else { + return child.getValue(ctx, hv, key, cb) + } + } + + return os.ErrNotExist +} + +// EnumLinks collects all links in the Shard. +func (ds *Shard) EnumLinks(ctx context.Context) ([]*ipld.Link, error) { + var links []*ipld.Link + + linkResults := ds.EnumLinksAsync(ctx) + + for linkResult := range linkResults { + if linkResult.Err != nil { + return links, linkResult.Err + } + links = append(links, linkResult.Link) + } + return links, nil +} + +// ForEachLink walks the Shard and calls the given function. +func (ds *Shard) ForEachLink(ctx context.Context, f func(*ipld.Link) error) error { + return ds.walkTrie(ctx, func(sv *Shard) error { + lnk := sv.val + lnk.Name = sv.key + + return f(lnk) + }) +} + +// EnumLinksAsync returns a channel which will receive Links in the directory +// as they are enumerated, where order is not guaranteed +func (ds *Shard) EnumLinksAsync(ctx context.Context) <-chan format.LinkResult { + linkResults := make(chan format.LinkResult) + ctx, cancel := context.WithCancel(ctx) + go func() { + defer close(linkResults) + defer cancel() + + err := parallelShardWalk(ctx, ds, ds.dserv, func(formattedLink *ipld.Link) error { + emitResult(ctx, linkResults, format.LinkResult{Link: formattedLink, Err: nil}) + return nil + }) + if err != nil { + emitResult(ctx, linkResults, format.LinkResult{Link: nil, Err: err}) + } + }() + return linkResults +} + +type listCidsAndShards struct { + cids []cid.Cid + shards []*Shard +} + +func (ds *Shard) walkChildren(processLinkValues func(formattedLink *ipld.Link) error) (*listCidsAndShards, error) { + res := &listCidsAndShards{} + + for idx, lnk := range ds.childer.links { + if nextShard := ds.childer.children[idx]; nextShard == nil { + lnkLinkType, err := ds.childLinkType(lnk) + if err != nil { + return nil, err + } + + switch lnkLinkType { + case shardValueLink: + sv, err := ds.makeShardValue(lnk) + if err != nil { + return nil, err + } + formattedLink := sv.val + formattedLink.Name = sv.key + + if err := processLinkValues(formattedLink); err != nil { + return nil, err + } + case shardLink: + res.cids = append(res.cids, lnk.Cid) + default: + return nil, fmt.Errorf("unsupported shard link type") + } + + } else { + if nextShard.val != nil { + formattedLink := &ipld.Link{ + Name: nextShard.key, + Size: nextShard.val.Size, + Cid: nextShard.val.Cid, + } + if err := processLinkValues(formattedLink); err != nil { + return nil, err + } + } else { + res.shards = append(res.shards, nextShard) + } + } + } + return res, nil +} + +// parallelShardWalk is quite similar to the DAG walking algorithm from https://github.com/ipfs/go-merkledag/blob/594e515f162e764183243b72c2ba84f743424c8c/merkledag.go#L464 +// However, there are a few notable differences: +// 1. Some children are actualized Shard structs and some are in the blockstore, this will leverage walking over the in memory Shards as well as the stored blocks +// 2. Instead of just passing each child into the worker pool by itself we group them so that we can leverage optimizations from GetMany. +// This optimization also makes the walk a little more biased towards depth (as opposed to BFS) in the earlier part of the DAG. +// This is particularly helpful for operations like estimating the directory size which should complete quickly when possible. +// 3. None of the extra options from that package are needed +func parallelShardWalk(ctx context.Context, root *Shard, dserv ipld.DAGService, processShardValues func(formattedLink *ipld.Link) error) error { + const concurrency = 32 + + var visitlk sync.Mutex + visitSet := cid.NewSet() + visit := visitSet.Visit + + // Setup synchronization + grp, errGrpCtx := errgroup.WithContext(ctx) + + // Input and output queues for workers. + feed := make(chan *listCidsAndShards) + out := make(chan *listCidsAndShards) + done := make(chan struct{}) + + for i := 0; i < concurrency; i++ { + grp.Go(func() error { + for feedChildren := range feed { + for _, nextShard := range feedChildren.shards { + nextChildren, err := nextShard.walkChildren(processShardValues) + if err != nil { + return err + } + + select { + case out <- nextChildren: + case <-errGrpCtx.Done(): + return nil + } + } + + var linksToVisit []cid.Cid + for _, nextCid := range feedChildren.cids { + var shouldVisit bool + + visitlk.Lock() + shouldVisit = visit(nextCid) + visitlk.Unlock() + + if shouldVisit { + linksToVisit = append(linksToVisit, nextCid) + } + } + + chNodes := dserv.GetMany(errGrpCtx, linksToVisit) + for optNode := range chNodes { + if optNode.Err != nil { + return optNode.Err + } + + nextShard, err := NewHamtFromDag(dserv, optNode.Node) + if err != nil { + return err + } + + nextChildren, err := nextShard.walkChildren(processShardValues) + if err != nil { + return err + } + + select { + case out <- nextChildren: + case <-errGrpCtx.Done(): + return nil + } + } + + select { + case done <- struct{}{}: + case <-errGrpCtx.Done(): + } + } + return nil + }) + } + + send := feed + var todoQueue []*listCidsAndShards + var inProgress int + + next := &listCidsAndShards{ + shards: []*Shard{root}, + } + +dispatcherLoop: + for { + select { + case send <- next: + inProgress++ + if len(todoQueue) > 0 { + next = todoQueue[0] + todoQueue = todoQueue[1:] + } else { + next = nil + send = nil + } + case <-done: + inProgress-- + if inProgress == 0 && next == nil { + break dispatcherLoop + } + case nextNodes := <-out: + if next == nil { + next = nextNodes + send = feed + } else { + todoQueue = append(todoQueue, nextNodes) + } + case <-errGrpCtx.Done(): + break dispatcherLoop + } + } + close(feed) + return grp.Wait() +} + +func emitResult(ctx context.Context, linkResults chan<- format.LinkResult, r format.LinkResult) { + // make sure that context cancel is processed first + // the reason is due to the concurrency of EnumerateChildrenAsync + // it's possible for EnumLinksAsync to complete and close the linkResults + // channel before this code runs + select { + case <-ctx.Done(): + return + default: + } + select { + case linkResults <- r: + case <-ctx.Done(): + } +} + +func (ds *Shard) walkTrie(ctx context.Context, cb func(*Shard) error) error { + return ds.childer.each(ctx, func(s *Shard) error { + if s.isValueNode() { + if err := cb(s); err != nil { + return err + } + } else { + if err := s.walkTrie(ctx, cb); err != nil { + return err + } + } + return nil + }) +} + +// swapValue sets the link `value` in the given key, either creating the entry +// if it didn't exist or overwriting the old one. It returns the old entry (if any). +func (ds *Shard) swapValue(ctx context.Context, hv *hashBits, key string, value *ipld.Link) (*ipld.Link, error) { + idx, err := hv.Next(ds.tableSizeLg2) + if err != nil { + return nil, err + } + + if !ds.childer.has(idx) { + // Entry does not exist, create a new one. + return nil, ds.childer.insert(key, value, idx) + } + + i := ds.childer.sliceIndex(idx) + child, err := ds.childer.get(ctx, i) + if err != nil { + return nil, err + } + + if child.isValueNode() { + // Leaf node. This is the base case of this recursive function. + if child.key == key { + // We are in the correct shard (tree level) so we modify this child + // and return. + oldValue := child.val + + if value == nil { // Remove old entry. + return oldValue, ds.childer.rm(idx) + } + + child.val = value // Overwrite entry. + return oldValue, nil + } + + if value == nil { + return nil, os.ErrNotExist + } + + // We are in the same slot with another entry with a different key + // so we need to fork this leaf node into a shard with two childs: + // the old entry and the new one being inserted here. + // We don't overwrite anything here so we keep: + // `oldValue = nil` + + // The child of this shard will now be a new shard. The old child value + // will be a child of this new shard (along with the new value being + // inserted). + grandChild := child + child, err = NewShard(ds.dserv, ds.tableSize) + if err != nil { + return nil, err + } + child.builder = ds.builder + chhv := newConsumedHashBits(grandChild.key, hv.consumed) + + // We explicitly ignore the oldValue returned by the next two insertions + // (which will be nil) to highlight there is no overwrite here: they are + // done with different keys to a new (empty) shard. (At best this shard + // will create new ones until we find different slots for both.) + _, err = child.swapValue(ctx, hv, key, value) + if err != nil { + return nil, err + } + _, err = child.swapValue(ctx, chhv, grandChild.key, grandChild.val) + if err != nil { + return nil, err + } + + // Replace this leaf node with the new Shard node. + ds.childer.set(child, i) + return nil, nil + } else { + // We are in a Shard (internal node). We will recursively call this + // function until finding the leaf (the logic of the `if` case above). + oldValue, err := child.swapValue(ctx, hv, key, value) + if err != nil { + return nil, err + } + + if value == nil { + // We have removed an entry, check if we should remove shards + // as well. + switch child.childer.length() { + case 0: + // empty sub-shard, prune it + // Note: this shouldnt normally ever happen + // in the event of another implementation creates flawed + // structures, this will help to normalize them. + return oldValue, ds.childer.rm(idx) + case 1: + // The single child _should_ be a value by + // induction. However, we allow for it to be a + // shard in case an implementation is broken. + + // Have we loaded the child? Prefer that. + schild := child.childer.child(0) + if schild != nil { + if schild.isValueNode() { + ds.childer.set(schild, i) + } + return oldValue, nil + } + + // Otherwise, work with the link. + slnk := child.childer.link(0) + var lnkType linkType + lnkType, err = child.childer.sd.childLinkType(slnk) + if err != nil { + return nil, err + } + if lnkType == shardValueLink { + // sub-shard with a single value element, collapse it + ds.childer.setLink(slnk, i) + } + return oldValue, nil + } + } + + return oldValue, nil + } +} + +// linkNamePrefix takes in the bitfield index of an entry and returns its hex prefix +func (ds *Shard) linkNamePrefix(idx int) string { + return fmt.Sprintf(ds.prefixPadStr, idx) +} + +// childer wraps the links, children and bitfield +// and provides basic operation (get, rm, insert and set) of manipulating children. +// The slices `links` and `children` are always coordinated to have the entries +// in the same index. A `childIndex` belonging to one of the original `Shard.size` +// entries corresponds to a `sliceIndex` in `links` and `children` (the conversion +// is done through `bitfield`). +type childer struct { + sd *Shard + dserv ipld.DAGService + bitfield bitfield.Bitfield + + // Only one of links/children will be non-nil for every child/link. + links []*ipld.Link + children []*Shard +} + +const maximumHamtWidth = 1 << 10 // FIXME: Spec this and decide of a correct value + +func newChilder(ds ipld.DAGService, size int) (*childer, error) { + if size > maximumHamtWidth { + return nil, fmt.Errorf("hamt witdh (%d) exceed maximum allowed (%d)", size, maximumHamtWidth) + } + bf, err := bitfield.NewBitfield(size) + if err != nil { + return nil, err + } + + return &childer{ + dserv: ds, + bitfield: bf, + }, nil +} + +func (s *childer) makeChilder(data []byte, links []*ipld.Link) *childer { + s.children = make([]*Shard, len(links)) + s.bitfield.SetBytes(data) + if len(links) > 0 { + s.links = make([]*ipld.Link, len(links)) + copy(s.links, links) + } + + return s +} + +// Return the `sliceIndex` associated with a child. +func (s *childer) sliceIndex(childIndex int) (sliceIndex int) { + return s.bitfield.OnesBefore(childIndex) +} + +func (s *childer) child(sliceIndex int) *Shard { + return s.children[sliceIndex] +} + +func (s *childer) link(sliceIndex int) *ipld.Link { + return s.links[sliceIndex] +} + +func (s *childer) insert(key string, lnk *ipld.Link, idx int) error { + if lnk == nil { + return os.ErrNotExist + } + + lnk.Name = s.sd.linkNamePrefix(idx) + key + i := s.sliceIndex(idx) + + sd, err := NewShardValue(s.dserv, 256, key, lnk) + if err != nil { + return err + } + + s.children = append(s.children[:i], append([]*Shard{sd}, s.children[i:]...)...) + s.links = append(s.links[:i], append([]*ipld.Link{nil}, s.links[i:]...)...) + // Add a `nil` placeholder in `links` so the rest of the entries keep the same + // index as `children`. + s.bitfield.SetBit(idx) + + return nil +} + +func (s *childer) set(sd *Shard, i int) { + s.children[i] = sd + s.links[i] = nil +} + +func (s *childer) setLink(lnk *ipld.Link, i int) { + s.children[i] = nil + s.links[i] = lnk +} + +func (s *childer) rm(childIndex int) error { + i := s.sliceIndex(childIndex) + + if err := s.check(i); err != nil { + return err + } + + copy(s.children[i:], s.children[i+1:]) + s.children = s.children[:len(s.children)-1] + + copy(s.links[i:], s.links[i+1:]) + s.links = s.links[:len(s.links)-1] + + s.bitfield.UnsetBit(childIndex) + + return nil +} + +// get returns the i'th child of this shard. If it is cached in the +// children array, it will return it from there. Otherwise, it loads the child +// node from disk. +func (s *childer) get(ctx context.Context, sliceIndex int) (*Shard, error) { + if err := s.check(sliceIndex); err != nil { + return nil, err + } + + c := s.child(sliceIndex) + if c != nil { + return c, nil + } + + return s.loadChild(ctx, sliceIndex) +} + +// loadChild reads the i'th child node of this shard from disk and returns it +// as a 'child' interface +func (s *childer) loadChild(ctx context.Context, sliceIndex int) (*Shard, error) { + lnk := s.link(sliceIndex) + lnkLinkType, err := s.sd.childLinkType(lnk) + if err != nil { + return nil, err + } + + var c *Shard + if lnkLinkType == shardLink { + nd, err := lnk.GetNode(ctx, s.dserv) + if err != nil { + return nil, err + } + cds, err := NewHamtFromDag(s.dserv, nd) + if err != nil { + return nil, err + } + + c = cds + } else { + s, err := s.sd.makeShardValue(lnk) + if err != nil { + return nil, err + } + c = s + } + + s.set(c, sliceIndex) + + return c, nil +} + +func (s *childer) has(childIndex int) bool { + return s.bitfield.Bit(childIndex) +} + +func (s *childer) length() int { + return len(s.children) +} + +func (s *childer) each(ctx context.Context, cb func(*Shard) error) error { + for i := range s.children { + c, err := s.get(ctx, i) + if err != nil { + return err + } + + if err := cb(c); err != nil { + return err + } + } + + return nil +} + +func (s *childer) check(sliceIndex int) error { + if sliceIndex >= len(s.children) || sliceIndex < 0 { + return fmt.Errorf("invalid index passed to operate children (likely corrupt bitfield)") + } + + if len(s.children) != len(s.links) { + return fmt.Errorf("inconsistent lengths between children array and Links array") + } + + return nil +} diff --git a/unixfs/hamt/hamt_stress_test.go b/unixfs/hamt/hamt_stress_test.go new file mode 100644 index 0000000000..74b34ae858 --- /dev/null +++ b/unixfs/hamt/hamt_stress_test.go @@ -0,0 +1,291 @@ +package hamt + +import ( + "context" + "fmt" + "math/rand" + "os" + "testing" + "time" + + ft "github.com/ipfs/go-libipfs/unixfs" + mdtest "github.com/ipfs/go-merkledag/test" + + ipld "github.com/ipfs/go-ipld-format" +) + +func getNames(prefix string, count int) []string { + out := make([]string, count) + for i := 0; i < count; i++ { + out[i] = fmt.Sprintf("%s%d", prefix, i) + } + return out +} + +const ( + opAdd = iota + opDel + opFind +) + +type testOp struct { + Op int + Val string +} + +func stringArrToSet(arr []string) map[string]bool { + out := make(map[string]bool) + for _, s := range arr { + out[s] = true + } + return out +} + +// generate two different random sets of operations to result in the same +// ending directory (same set of entries at the end) and execute each of them +// in turn, then compare to ensure the output is the same on each. +func TestOrderConsistency(t *testing.T) { + seed := time.Now().UnixNano() + t.Logf("using seed = %d", seed) + ds := mdtest.Mock() + + shardWidth := 1024 + + keep := getNames("good", 4000) + temp := getNames("tempo", 6000) + + ops := genOpSet(seed, keep, temp) + s, err := executeOpSet(t, ds, shardWidth, ops) + if err != nil { + t.Fatal(err) + } + + err = validateOpSetCompletion(t, s, keep, temp) + if err != nil { + t.Fatal(err) + } + + ops2 := genOpSet(seed+1000, keep, temp) + s2, err := executeOpSet(t, ds, shardWidth, ops2) + if err != nil { + t.Fatal(err) + } + + err = validateOpSetCompletion(t, s2, keep, temp) + if err != nil { + t.Fatal(err) + } + + nd, err := s.Node() + if err != nil { + t.Fatal(err) + } + + nd2, err := s2.Node() + if err != nil { + t.Fatal(err) + } + + k := nd.Cid() + k2 := nd2.Cid() + + if !k.Equals(k2) { + t.Fatal("got different results: ", k, k2) + } +} + +func validateOpSetCompletion(t *testing.T, s *Shard, keep, temp []string) error { + ctx := context.TODO() + for _, n := range keep { + _, err := s.Find(ctx, n) + if err != nil { + return fmt.Errorf("couldnt find %s: %s", n, err) + } + } + + for _, n := range temp { + _, err := s.Find(ctx, n) + if err != os.ErrNotExist { + return fmt.Errorf("expected not to find: %s", err) + } + } + + return nil +} + +func executeOpSet(t *testing.T, ds ipld.DAGService, width int, ops []testOp) (*Shard, error) { + ctx := context.TODO() + s, err := NewShard(ds, width) + if err != nil { + return nil, err + } + + e := ft.EmptyDirNode() + ds.Add(ctx, e) + + for _, o := range ops { + switch o.Op { + case opAdd: + err := s.Set(ctx, o.Val, e) + if err != nil { + return nil, fmt.Errorf("inserting %s: %s", o.Val, err) + } + case opDel: + err := s.Remove(ctx, o.Val) + if err != nil { + return nil, fmt.Errorf("deleting %s: %s", o.Val, err) + } + case opFind: + _, err := s.Find(ctx, o.Val) + if err != nil { + return nil, fmt.Errorf("finding %s: %s", o.Val, err) + } + } + } + + return s, nil +} + +func genOpSet(seed int64, keep, temp []string) []testOp { + tempset := stringArrToSet(temp) + + allnames := append(keep, temp...) + shuffle(seed, allnames) + + var todel []string + + var ops []testOp + + for { + n := len(allnames) + len(todel) + if n == 0 { + return ops + } + + rn := rand.Intn(n) + + if rn < len(allnames) { + next := allnames[0] + allnames = allnames[1:] + ops = append(ops, testOp{ + Op: opAdd, + Val: next, + }) + + if tempset[next] { + todel = append(todel, next) + } + } else { + shuffle(seed+100, todel) + next := todel[0] + todel = todel[1:] + + ops = append(ops, testOp{ + Op: opDel, + Val: next, + }) + } + } +} + +// executes the given op set with a repl to allow easier debugging +/*func debugExecuteOpSet(ds node.DAGService, width int, ops []testOp) (*Shard, error) { + + s, err := NewShard(ds, width) + if err != nil { + return nil, err + } + + e := ft.EmptyDirNode() + ds.Add(e) + ctx := context.TODO() + + run := 0 + + opnames := map[int]string{ + opAdd: "add", + opDel: "del", + } + +mainloop: + for i := 0; i < len(ops); i++ { + o := ops[i] + + fmt.Printf("Op %d: %s %s\n", i, opnames[o.Op], o.Val) + for run == 0 { + cmd := readCommand() + parts := strings.Split(cmd, " ") + switch parts[0] { + case "": + run = 1 + case "find": + _, err := s.Find(ctx, parts[1]) + if err == nil { + fmt.Println("success") + } else { + fmt.Println(err) + } + case "run": + if len(parts) > 1 { + n, err := strconv.Atoi(parts[1]) + if err != nil { + panic(err) + } + + run = n + } else { + run = -1 + } + case "lookop": + for k = 0; k < len(ops); k++ { + if ops[k].Val == parts[1] { + fmt.Printf(" Op %d: %s %s\n", k, opnames[ops[k].Op], parts[1]) + } + } + case "restart": + var err error + s, err = NewShard(ds, width) + if err != nil { + panic(err) + } + i = -1 + continue mainloop + case "print": + nd, err := s.Node() + if err != nil { + panic(err) + } + printDag(ds, nd.(*dag.ProtoNode), 0) + } + } + run-- + + switch o.Op { + case opAdd: + err := s.Set(ctx, o.Val, e) + if err != nil { + return nil, fmt.Errorf("inserting %s: %s", o.Val, err) + } + case opDel: + fmt.Println("deleting: ", o.Val) + err := s.Remove(ctx, o.Val) + if err != nil { + return nil, fmt.Errorf("deleting %s: %s", o.Val, err) + } + case opFind: + _, err := s.Find(ctx, o.Val) + if err != nil { + return nil, fmt.Errorf("finding %s: %s", o.Val, err) + } + } + } + + return s, nil +} + +func readCommand() string { + fmt.Print("> ") + scan := bufio.NewScanner(os.Stdin) + scan.Scan() + return scan.Text() +}*/ diff --git a/unixfs/hamt/hamt_test.go b/unixfs/hamt/hamt_test.go new file mode 100644 index 0000000000..6bec805d3b --- /dev/null +++ b/unixfs/hamt/hamt_test.go @@ -0,0 +1,752 @@ +package hamt + +import ( + "context" + "fmt" + "math/rand" + "os" + "sort" + "testing" + "time" + + ipld "github.com/ipfs/go-ipld-format" + ft "github.com/ipfs/go-libipfs/unixfs" + dag "github.com/ipfs/go-merkledag" + mdtest "github.com/ipfs/go-merkledag/test" +) + +func shuffle(seed int64, arr []string) { + r := rand.New(rand.NewSource(seed)) + for i := 0; i < len(arr); i++ { + a := r.Intn(len(arr)) + b := r.Intn(len(arr)) + arr[a], arr[b] = arr[b], arr[a] + } +} + +func makeDir(ds ipld.DAGService, size int) ([]string, *Shard, error) { + return makeDirWidth(ds, size, 256) +} + +func makeDirWidth(ds ipld.DAGService, size, width int) ([]string, *Shard, error) { + ctx := context.Background() + + s, err := NewShard(ds, width) + if err != nil { + return nil, nil, err + } + + var dirs []string + for i := 0; i < size; i++ { + dirs = append(dirs, fmt.Sprintf("DIRNAME%d", i)) + } + + shuffle(time.Now().UnixNano(), dirs) + + for i := 0; i < len(dirs); i++ { + nd := ft.EmptyDirNode() + err := ds.Add(ctx, nd) + if err != nil { + return nil, nil, err + } + err = s.Set(ctx, dirs[i], nd) + if err != nil { + return nil, nil, err + } + } + + return dirs, s, nil +} + +func assertLink(s *Shard, name string, found bool) error { + _, err := s.Find(context.Background(), name) + switch err { + case os.ErrNotExist: + if found { + return err + } + + return nil + case nil: + if found { + return nil + } + + return fmt.Errorf("expected not to find link named %s", name) + default: + return err + } +} + +func assertLinksEqual(linksA []*ipld.Link, linksB []*ipld.Link) error { + + if len(linksA) != len(linksB) { + return fmt.Errorf("links arrays are different sizes") + } + + sort.Stable(dag.LinkSlice(linksA)) + sort.Stable(dag.LinkSlice(linksB)) + for i, a := range linksA { + b := linksB[i] + if a.Name != b.Name { + return fmt.Errorf("links names mismatch") + } + + if a.Cid.String() != b.Cid.String() { + return fmt.Errorf("link hashes dont match") + } + + if a.Size != b.Size { + return fmt.Errorf("link sizes not the same") + } + } + + return nil +} + +func assertSerializationWorks(ds ipld.DAGService, s *Shard) error { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + nd, err := s.Node() + if err != nil { + return err + } + + nds, err := NewHamtFromDag(ds, nd) + if err != nil { + return err + } + + linksA, err := s.EnumLinks(ctx) + if err != nil { + return err + } + + linksB, err := nds.EnumLinks(ctx) + if err != nil { + return err + } + + return assertLinksEqual(linksA, linksB) +} + +func TestBasicSet(t *testing.T) { + ds := mdtest.Mock() + for _, w := range []int{128, 256, 512, 1024} { + t.Run(fmt.Sprintf("BasicSet%d", w), func(t *testing.T) { + names, s, err := makeDirWidth(ds, 1000, w) + if err != nil { + t.Fatal(err) + } + ctx := context.Background() + + for _, d := range names { + _, err := s.Find(ctx, d) + if err != nil { + t.Fatal(err) + } + } + }) + } +} + +func TestDirBuilding(t *testing.T) { + ds := mdtest.Mock() + _, _ = NewShard(ds, 256) + + _, s, err := makeDir(ds, 200) + if err != nil { + t.Fatal(err) + } + + nd, err := s.Node() + if err != nil { + t.Fatal(err) + } + + //printDag(ds, nd, 0) + + k := nd.Cid() + + if k.String() != "QmY89TkSEVHykWMHDmyejSWFj9CYNtvzw4UwnT9xbc4Zjc" { + t.Fatalf("output didnt match what we expected (got %s)", k.String()) + } +} + +func TestShardReload(t *testing.T) { + ds := mdtest.Mock() + _, _ = NewShard(ds, 256) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + _, s, err := makeDir(ds, 200) + if err != nil { + t.Fatal(err) + } + + nd, err := s.Node() + if err != nil { + t.Fatal(err) + } + + nds, err := NewHamtFromDag(ds, nd) + if err != nil { + t.Fatal(err) + } + + lnks, err := nds.EnumLinks(ctx) + if err != nil { + t.Fatal(err) + } + + if len(lnks) != 200 { + t.Fatal("not enough links back") + } + + _, err = nds.Find(ctx, "DIRNAME50") + if err != nil { + t.Fatal(err) + } + + // Now test roundtrip marshal with no operations + + nds, err = NewHamtFromDag(ds, nd) + if err != nil { + t.Fatal(err) + } + + ond, err := nds.Node() + if err != nil { + t.Fatal(err) + } + + outk := ond.Cid() + ndk := nd.Cid() + + if !outk.Equals(ndk) { + t.Fatal("roundtrip serialization failed") + } +} + +func TestRemoveElems(t *testing.T) { + ds := mdtest.Mock() + dirs, s, err := makeDir(ds, 500) + if err != nil { + t.Fatal(err) + } + ctx := context.Background() + + for i := 0; i < 100; i++ { + err := s.Remove(ctx, fmt.Sprintf("NOTEXIST%d", rand.Int())) + if err != os.ErrNotExist { + t.Fatal("shouldnt be able to remove things that don't exist") + } + } + + for _, d := range dirs { + _, err := s.Find(ctx, d) + if err != nil { + t.Fatal(err) + } + } + + shuffle(time.Now().UnixNano(), dirs) + + for _, d := range dirs { + err := s.Remove(ctx, d) + if err != nil { + t.Fatal(err) + } + } + + nd, err := s.Node() + if err != nil { + t.Fatal(err) + } + + if len(nd.Links()) > 0 { + t.Fatal("shouldnt have any links here") + } + + err = s.Remove(ctx, "doesnt exist") + if err != os.ErrNotExist { + t.Fatal("expected error does not exist") + } +} + +func TestRemoveAfterMarshal(t *testing.T) { + ds := mdtest.Mock() + dirs, s, err := makeDir(ds, 500) + if err != nil { + t.Fatal(err) + } + nd, err := s.Node() + if err != nil { + t.Fatal(err) + } + + s, err = NewHamtFromDag(ds, nd) + if err != nil { + t.Fatal(err) + } + + ctx := context.Background() + + shuffle(time.Now().UnixNano(), dirs) + + for i, d := range dirs { + err := s.Remove(ctx, d) + if err != nil { + t.Fatalf("%d/%d: %s", i, len(dirs), err) + } + } + + nd, err = s.Node() + if err != nil { + t.Fatal(err) + } + + if len(nd.Links()) > 0 { + t.Fatal("shouldnt have any links here") + } + + err = s.Remove(ctx, "doesnt exist") + if err != os.ErrNotExist { + t.Fatal("expected error does not exist") + } +} + +func TestSetAfterMarshal(t *testing.T) { + ds := mdtest.Mock() + _, s, err := makeDir(ds, 300) + if err != nil { + t.Fatal(err) + } + ctx := context.Background() + + nd, err := s.Node() + if err != nil { + t.Fatal(err) + } + + nds, err := NewHamtFromDag(ds, nd) + if err != nil { + t.Fatal(err) + } + + for i := 0; i < 100; i++ { + empty := ft.EmptyDirNode() + err := nds.Set(ctx, fmt.Sprintf("moredirs%d", i), empty) + if err != nil { + t.Fatal(err) + } + } + + nd, err = nds.Node() + if err != nil { + t.Fatal(err) + } + nds, err = NewHamtFromDag(ds, nd) + if err != nil { + t.Fatal(err) + } + + links, err := nds.EnumLinks(ctx) + if err != nil { + t.Fatal(err) + } + + if len(links) != 400 { + t.Fatal("expected 400 links") + } + + err = assertSerializationWorks(ds, nds) + if err != nil { + t.Fatal(err) + } +} + +func TestEnumLinksAsync(t *testing.T) { + ds := mdtest.Mock() + _, s, err := makeDir(ds, 300) + if err != nil { + t.Fatal(err) + } + ctx := context.Background() + + nd, err := s.Node() + if err != nil { + t.Fatal(err) + } + + nds, err := NewHamtFromDag(ds, nd) + if err != nil { + t.Fatal(err) + } + + linksA, err := nds.EnumLinks(ctx) + if err != nil { + t.Fatal(err) + } + + linkResults := nds.EnumLinksAsync(ctx) + + var linksB []*ipld.Link + + for linkResult := range linkResults { + if linkResult.Err != nil { + t.Fatal(linkResult.Err) + } + linksB = append(linksB, linkResult.Link) + } + + err = assertLinksEqual(linksA, linksB) + if err != nil { + t.Fatal(err) + } +} + +func TestDuplicateAddShard(t *testing.T) { + ds := mdtest.Mock() + dir, _ := NewShard(ds, 256) + nd := new(dag.ProtoNode) + ctx := context.Background() + + err := dir.Set(ctx, "test", nd) + if err != nil { + t.Fatal(err) + } + + err = dir.Set(ctx, "test", nd) + if err != nil { + t.Fatal(err) + } + + node, err := dir.Node() + if err != nil { + t.Fatal(err) + } + dir, err = NewHamtFromDag(ds, node) + if err != nil { + t.Fatal(err) + } + + lnks, err := dir.EnumLinks(ctx) + if err != nil { + t.Fatal(err) + } + + if len(lnks) != 1 { + t.Fatal("expected only one link") + } +} + +// fix https://github.com/ipfs/kubo/issues/9063 +func TestSetLink(t *testing.T) { + ds := mdtest.Mock() + dir, _ := NewShard(ds, 256) + _, s, err := makeDir(ds, 300) + if err != nil { + t.Fatal(err) + } + + lnk, err := s.Link() + if err != nil { + t.Fatal(err) + } + + ctx := context.Background() + + err = dir.SetLink(ctx, "test", lnk) + if err != nil { + t.Fatal(err) + } + + if len(dir.childer.children) != 1 { + t.Fatal("no child") + } + + for _, sh := range dir.childer.children { + if sh.childer == nil { + t.Fatal("no childer on shard") + } + } +} + +func TestLoadFailsFromNonShard(t *testing.T) { + ds := mdtest.Mock() + nd := ft.EmptyDirNode() + + _, err := NewHamtFromDag(ds, nd) + if err == nil { + t.Fatal("expected dir shard creation to fail when given normal directory") + } + + nd = new(dag.ProtoNode) + + _, err = NewHamtFromDag(ds, nd) + if err == nil { + t.Fatal("expected dir shard creation to fail when given normal directory") + } +} + +func TestFindNonExisting(t *testing.T) { + ds := mdtest.Mock() + _, s, err := makeDir(ds, 100) + if err != nil { + t.Fatal(err) + } + ctx := context.Background() + + for i := 0; i < 200; i++ { + _, err := s.Find(ctx, fmt.Sprintf("notfound%d", i)) + if err != os.ErrNotExist { + t.Fatal("expected ErrNotExist") + } + } +} + +func TestRemoveElemsAfterMarshal(t *testing.T) { + ds := mdtest.Mock() + dirs, s, err := makeDir(ds, 30) + if err != nil { + t.Fatal(err) + } + ctx := context.Background() + + sort.Strings(dirs) + + err = s.Remove(ctx, dirs[0]) + if err != nil { + t.Fatal(err) + } + + out, err := s.Find(ctx, dirs[0]) + if err == nil { + t.Fatal("expected error, got: ", out) + } + + nd, err := s.Node() + if err != nil { + t.Fatal(err) + } + + nds, err := NewHamtFromDag(ds, nd) + if err != nil { + t.Fatal(err) + } + + _, err = nds.Find(ctx, dirs[0]) + if err == nil { + t.Fatal("expected not to find ", dirs[0]) + } + + for _, d := range dirs[1:] { + _, err := nds.Find(ctx, d) + if err != nil { + t.Fatal("could not find expected link after unmarshaling") + } + } + + for _, d := range dirs[1:] { + err := nds.Remove(ctx, d) + if err != nil { + t.Fatal(err) + } + } + + nd, err = nds.Node() + if err != nil { + t.Fatal(err) + } + nds, err = NewHamtFromDag(ds, nd) + if err != nil { + t.Fatal(err) + } + + links, err := nds.EnumLinks(ctx) + if err != nil { + t.Fatal(err) + } + + if len(links) != 0 { + t.Fatal("expected all links to be removed") + } + + err = assertSerializationWorks(ds, nds) + if err != nil { + t.Fatal(err) + } +} + +func TestBitfieldIndexing(t *testing.T) { + ds := mdtest.Mock() + s, _ := NewShard(ds, 256) + + set := func(i int) { + s.childer.bitfield.SetBit(i) + } + + assert := func(i int, val int) { + if s.childer.sliceIndex(i) != val { + t.Fatalf("expected index %d to be %d", i, val) + } + } + + assert(50, 0) + set(4) + set(5) + set(60) + + assert(10, 2) + set(3) + assert(10, 3) + assert(1, 0) + + assert(100, 4) + set(50) + assert(45, 3) + set(100) + assert(100, 5) +} + +// test adding a sharded directory node as the child of another directory node. +// if improperly implemented, the parent hamt may assume the child is a part of +// itself. +func TestSetHamtChild(t *testing.T) { + ctx := context.Background() + + ds := mdtest.Mock() + s, _ := NewShard(ds, 256) + + e := ft.EmptyDirNode() + ds.Add(ctx, e) + + err := s.Set(ctx, "bar", e) + if err != nil { + t.Fatal(err) + } + + snd, err := s.Node() + if err != nil { + t.Fatal(err) + } + + _, ns, err := makeDir(ds, 50) + if err != nil { + t.Fatal(err) + } + + err = ns.Set(ctx, "foo", snd) + if err != nil { + t.Fatal(err) + } + + nsnd, err := ns.Node() + if err != nil { + t.Fatal(err) + } + + hs, err := NewHamtFromDag(ds, nsnd) + if err != nil { + t.Fatal(err) + } + + err = assertLink(hs, "bar", false) + if err != nil { + t.Fatal(err) + } + + err = assertLink(hs, "foo", true) + if err != nil { + t.Fatal(err) + } +} + +func BenchmarkHAMTWalk(b *testing.B) { + ctx := context.Background() + + ds := mdtest.Mock() + sh, _ := NewShard(ds, 256) + nd, err := sh.Node() + if err != nil { + b.Fatal(err) + } + + err = ds.Add(ctx, nd) + if err != nil { + b.Fatal(err) + } + ds.Add(ctx, ft.EmptyDirNode()) + + s, err := NewHamtFromDag(ds, nd) + if err != nil { + b.Fatal(err) + } + + for j := 0; j < 1000; j++ { + err = s.Set(ctx, fmt.Sprintf("%d", j), ft.EmptyDirNode()) + if err != nil { + b.Fatal(err) + } + } + + for i := 0; i < b.N; i++ { + cnt := 0 + err = s.ForEachLink(ctx, func(l *ipld.Link) error { + cnt++ + return nil + }) + if err != nil { + b.Fatal(err) + } + if cnt < 1000 { + b.Fatal("expected 100 children") + } + } +} + +func BenchmarkHAMTSet(b *testing.B) { + ctx := context.Background() + + ds := mdtest.Mock() + sh, _ := NewShard(ds, 256) + nd, err := sh.Node() + if err != nil { + b.Fatal(err) + } + + err = ds.Add(ctx, nd) + if err != nil { + b.Fatal(err) + } + ds.Add(ctx, ft.EmptyDirNode()) + + for i := 0; i < b.N; i++ { + s, err := NewHamtFromDag(ds, nd) + if err != nil { + b.Fatal(err) + } + + err = s.Set(context.TODO(), fmt.Sprint(i), ft.EmptyDirNode()) + if err != nil { + b.Fatal(err) + } + + out, err := s.Node() + if err != nil { + b.Fatal(err) + } + + nd = out + } +} + +func TestHamtBadSize(t *testing.T) { + for _, size := range [...]int{-8, 7, 2, 1337, 1024 + 8, -3} { + _, err := NewShard(nil, size) + if err == nil { + t.Errorf("should have failed to construct hamt with bad size: %d", size) + } + } +} diff --git a/unixfs/hamt/util.go b/unixfs/hamt/util.go new file mode 100644 index 0000000000..b7678c120f --- /dev/null +++ b/unixfs/hamt/util.go @@ -0,0 +1,80 @@ +package hamt + +import ( + "fmt" + "math/bits" + + "github.com/ipfs/go-libipfs/unixfs/internal" + + "github.com/spaolacci/murmur3" +) + +// hashBits is a helper that allows the reading of the 'next n bits' as an integer. +type hashBits struct { + b []byte + consumed int +} + +func newHashBits(val string) *hashBits { + return &hashBits{b: internal.HAMTHashFunction([]byte(val))} +} + +func newConsumedHashBits(val string, consumed int) *hashBits { + hv := &hashBits{b: internal.HAMTHashFunction([]byte(val))} + hv.consumed = consumed + return hv +} + +func mkmask(n int) byte { + return (1 << uint(n)) - 1 +} + +// Next returns the next 'i' bits of the hashBits value as an integer, or an +// error if there aren't enough bits. +func (hb *hashBits) Next(i int) (int, error) { + if hb.consumed+i > len(hb.b)*8 { + return 0, fmt.Errorf("sharded directory too deep") + } + return hb.next(i), nil +} + +func (hb *hashBits) next(i int) int { + curbi := hb.consumed / 8 + leftb := 8 - (hb.consumed % 8) + + curb := hb.b[curbi] + if i == leftb { + out := int(mkmask(i) & curb) + hb.consumed += i + return out + } else if i < leftb { + a := curb & mkmask(leftb) // mask out the high bits we don't want + b := a & ^mkmask(leftb-i) // mask out the low bits we don't want + c := b >> uint(leftb-i) // shift whats left down + hb.consumed += i + return int(c) + } else { + out := int(mkmask(leftb) & curb) + out <<= uint(i - leftb) + hb.consumed += leftb + out += hb.next(i - leftb) + return out + } +} + +func Logtwo(v int) (int, error) { + if v <= 0 { + return 0, fmt.Errorf("hamt size should be a power of two") + } + lg2 := bits.TrailingZeros(uint(v)) + if 1<= 0; coff -= 4096 { + t.Log(coff) + n, err := rs.Seek(coff, io.SeekStart) + if err != nil { + t.Fatal(err) + } + if n != coff { + t.Fatal("wasnt able to seek to the right position") + } + nread, err := rs.Read(out[coff : coff+4096]) + if err != nil { + t.Fatal(err) + } + if nread != 4096 { + t.Fatal("didnt read the correct number of bytes") + } + } + + err = arrComp(out, should) + if err != nil { + t.Fatal(err) + } +} diff --git a/unixfs/importer/balanced/builder.go b/unixfs/importer/balanced/builder.go new file mode 100644 index 0000000000..be3ab8149d --- /dev/null +++ b/unixfs/importer/balanced/builder.go @@ -0,0 +1,267 @@ +// Package balanced provides methods to build balanced DAGs, which are generalistic +// DAGs in which all leaves (nodes representing chunks of data) are at the same +// distance from the root. Nodes can have only a maximum number of children; to be +// able to store more leaf data nodes balanced DAGs are extended by increasing its +// depth (and having more intermediary nodes). +// +// Internal nodes are always represented by UnixFS nodes (of type `File`) encoded +// inside DAG nodes (see the `go-unixfs` package for details of UnixFS). In +// contrast, leaf nodes with data have multiple possible representations: UnixFS +// nodes as above, raw nodes with just the file data (no format) and Filestore +// nodes (that directly link to the file on disk using a format stored on a raw +// node, see the `go-ipfs/filestore` package for details of Filestore.) +// +// In the case the entire file fits into just one node it will be formatted as a +// (single) leaf node (without parent) with the possible representations already +// mentioned. This is the only scenario where the root can be of a type different +// that the UnixFS node. +// +// Notes: +// +// 1. In the implementation. `FSNodeOverDag` structure is used for representing +// the UnixFS node encoded inside the DAG node. +// (see https://github.com/ipfs/go-ipfs/pull/5118.) +// +// 2. `TFile` is used for backwards-compatibility. It was a bug causing the leaf +// nodes to be generated with this type instead of `TRaw`. The former one +// should be used (like the trickle builder does). +// (See https://github.com/ipfs/go-ipfs/pull/5120.) +// +// +-------------+ +// | Root 4 | +// +-------------+ +// | +// +--------------------------+----------------------------+ +// | | +// +-------------+ +-------------+ +// | Node 2 | | Node 5 | +// +-------------+ +-------------+ +// | | +// +-------------+-------------+ +-------------+ +// | | | +// +-------------+ +-------------+ +-------------+ +// | Node 1 | | Node 3 | | Node 6 | +// +-------------+ +-------------+ +-------------+ +// | | | +// +------+------+ +------+------+ +------+ +// | | | | | +// +=========+ +=========+ +=========+ +=========+ +=========+ +// | Chunk 1 | | Chunk 2 | | Chunk 3 | | Chunk 4 | | Chunk 5 | +// +=========+ +=========+ +=========+ +=========+ +=========+ +package balanced + +import ( + "errors" + + ft "github.com/ipfs/go-libipfs/unixfs" + h "github.com/ipfs/go-libipfs/unixfs/importer/helpers" + + ipld "github.com/ipfs/go-ipld-format" +) + +// Layout builds a balanced DAG layout. In a balanced DAG of depth 1, leaf nodes +// with data are added to a single `root` until the maximum number of links is +// reached. Then, to continue adding more data leaf nodes, a `newRoot` is created +// pointing to the old `root` (which will now become and intermediary node), +// increasing the depth of the DAG to 2. This will increase the maximum number of +// data leaf nodes the DAG can have (`Maxlinks() ^ depth`). The `fillNodeRec` +// function will add more intermediary child nodes to `newRoot` (which already has +// `root` as child) that in turn will have leaf nodes with data added to them. +// After that process is completed (the maximum number of links is reached), +// `fillNodeRec` will return and the loop will be repeated: the `newRoot` created +// will become the old `root` and a new root will be created again to increase the +// depth of the DAG. The process is repeated until there is no more data to add +// (i.e. the DagBuilderHelper’s Done() function returns true). +// +// The nodes are filled recursively, so the DAG is built from the bottom up. Leaf +// nodes are created first using the chunked file data and its size. The size is +// then bubbled up to the parent (internal) node, which aggregates all the sizes of +// its children and bubbles that combined size up to its parent, and so on up to +// the root. This way, a balanced DAG acts like a B-tree when seeking to a byte +// offset in the file the graph represents: each internal node uses the file size +// of its children as an index when seeking. +// +// `Layout` creates a root and hands it off to be filled: +// +// +-------------+ +// | Root 1 | +// +-------------+ +// | +// ( fillNodeRec fills in the ) +// ( chunks on the root. ) +// | +// +------+------+ +// | | +// + - - - - + + - - - - + +// | Chunk 1 | | Chunk 2 | +// + - - - - + + - - - - + +// +// ↓ +// When the root is full but there's more data... +// ↓ +// +// +-------------+ +// | Root 1 | +// +-------------+ +// | +// +------+------+ +// | | +// +=========+ +=========+ + - - - - + +// | Chunk 1 | | Chunk 2 | | Chunk 3 | +// +=========+ +=========+ + - - - - + +// +// ↓ +// ...Layout's job is to create a new root. +// ↓ +// +// +-------------+ +// | Root 2 | +// +-------------+ +// | +// +-------------+ - - - - - - - - + +// | | +// +-------------+ ( fillNodeRec creates the ) +// | Node 1 | ( branch that connects ) +// +-------------+ ( "Root 2" to "Chunk 3." ) +// | | +// +------+------+ + - - - - -+ +// | | | +// +=========+ +=========+ + - - - - + +// | Chunk 1 | | Chunk 2 | | Chunk 3 | +// +=========+ +=========+ + - - - - + +func Layout(db *h.DagBuilderHelper) (ipld.Node, error) { + if db.Done() { + // No data, return just an empty node. + root, err := db.NewLeafNode(nil, ft.TFile) + if err != nil { + return nil, err + } + // This works without Filestore support (`ProcessFileStore`). + // TODO: Why? Is there a test case missing? + + return root, db.Add(root) + } + + // The first `root` will be a single leaf node with data + // (corner case), after that subsequent `root` nodes will + // always be internal nodes (with a depth > 0) that can + // be handled by the loop. + root, fileSize, err := db.NewLeafDataNode(ft.TFile) + if err != nil { + return nil, err + } + + // Each time a DAG of a certain `depth` is filled (because it + // has reached its maximum capacity of `db.Maxlinks()` per node) + // extend it by making it a sub-DAG of a bigger DAG with `depth+1`. + for depth := 1; !db.Done(); depth++ { + + // Add the old `root` as a child of the `newRoot`. + newRoot := db.NewFSNodeOverDag(ft.TFile) + err = newRoot.AddChild(root, fileSize, db) + if err != nil { + return nil, err + } + + // Fill the `newRoot` (that has the old `root` already as child) + // and make it the current `root` for the next iteration (when + // it will become "old"). + root, fileSize, err = fillNodeRec(db, newRoot, depth) + if err != nil { + return nil, err + } + } + + return root, db.Add(root) +} + +// fillNodeRec will "fill" the given internal (non-leaf) `node` with data by +// adding child nodes to it, either leaf data nodes (if `depth` is 1) or more +// internal nodes with higher depth (and calling itself recursively on them +// until *they* are filled with data). The data to fill the node with is +// provided by DagBuilderHelper. +// +// `node` represents a (sub-)DAG root that is being filled. If called recursively, +// it is `nil`, a new node is created. If it has been called from `Layout` (see +// diagram below) it points to the new root (that increases the depth of the DAG), +// it already has a child (the old root). New children will be added to this new +// root, and those children will in turn be filled (calling `fillNodeRec` +// recursively). +// +// +-------------+ +// | `node` | +// | (new root) | +// +-------------+ +// | +// +-------------+ - - - - - - + - - - - - - - - - - - + +// | | | +// +--------------+ + - - - - - + + - - - - - + +// | (old root) | | new child | | | +// +--------------+ + - - - - - + + - - - - - + +// | | | +// +------+------+ + - - + - - - + +// | | | | +// +=========+ +=========+ + - - - - + + - - - - + +// | Chunk 1 | | Chunk 2 | | Chunk 3 | | Chunk 4 | +// +=========+ +=========+ + - - - - + + - - - - + +// +// The `node` to be filled uses the `FSNodeOverDag` abstraction that allows adding +// child nodes without packing/unpacking the UnixFS layer node (having an internal +// `ft.FSNode` cache). +// +// It returns the `ipld.Node` representation of the passed `node` filled with +// children and the `nodeFileSize` with the total size of the file chunk (leaf) +// nodes stored under this node (parent nodes store this to enable efficient +// seeking through the DAG when reading data later). +// +// warning: **children** pinned indirectly, but input node IS NOT pinned. +func fillNodeRec(db *h.DagBuilderHelper, node *h.FSNodeOverDag, depth int) (filledNode ipld.Node, nodeFileSize uint64, err error) { + if depth < 1 { + return nil, 0, errors.New("attempt to fillNode at depth < 1") + } + + if node == nil { + node = db.NewFSNodeOverDag(ft.TFile) + } + + // Child node created on every iteration to add to parent `node`. + // It can be a leaf node or another internal node. + var childNode ipld.Node + // File size from the child node needed to update the `FSNode` + // in `node` when adding the child. + var childFileSize uint64 + + // While we have room and there is data available to be added. + for node.NumChildren() < db.Maxlinks() && !db.Done() { + + if depth == 1 { + // Base case: add leaf node with data. + childNode, childFileSize, err = db.NewLeafDataNode(ft.TFile) + if err != nil { + return nil, 0, err + } + } else { + // Recursion case: create an internal node to in turn keep + // descending in the DAG and adding child nodes to it. + childNode, childFileSize, err = fillNodeRec(db, nil, depth-1) + if err != nil { + return nil, 0, err + } + } + + err = node.AddChild(childNode, childFileSize, db) + if err != nil { + return nil, 0, err + } + } + + nodeFileSize = node.FileSize() + + // Get the final `dag.ProtoNode` with the `FSNode` data encoded inside. + filledNode, err = node.Commit() + if err != nil { + return nil, 0, err + } + + return filledNode, nodeFileSize, nil +} diff --git a/unixfs/importer/helpers/dagbuilder.go b/unixfs/importer/helpers/dagbuilder.go new file mode 100644 index 0000000000..d976dd759f --- /dev/null +++ b/unixfs/importer/helpers/dagbuilder.go @@ -0,0 +1,399 @@ +package helpers + +import ( + "context" + "errors" + "io" + "os" + + dag "github.com/ipfs/go-merkledag" + + ft "github.com/ipfs/go-libipfs/unixfs" + pb "github.com/ipfs/go-libipfs/unixfs/pb" + + cid "github.com/ipfs/go-cid" + ipld "github.com/ipfs/go-ipld-format" + chunker "github.com/ipfs/go-libipfs/chunker" + "github.com/ipfs/go-libipfs/files" + pi "github.com/ipfs/go-libipfs/filestore/posinfo" +) + +var ErrMissingFsRef = errors.New("missing file path or URL, can't create filestore reference") + +// DagBuilderHelper wraps together a bunch of objects needed to +// efficiently create unixfs dag trees +type DagBuilderHelper struct { + dserv ipld.DAGService + spl chunker.Splitter + recvdErr error + rawLeaves bool + nextData []byte // the next item to return. + maxlinks int + cidBuilder cid.Builder + + // Filestore support variables. + // ---------------------------- + // TODO: Encapsulate in `FilestoreNode` (which is basically what they are). + // + // Besides having the path this variable (if set) is used as a flag + // to indicate that Filestore should be used. + fullPath string + stat os.FileInfo + // Keeps track of the current file size added to the DAG (used in + // the balanced builder). It is assumed that the `DagBuilderHelper` + // is not reused to construct another DAG, but a new one (with a + // zero `offset`) is created. + offset uint64 +} + +// DagBuilderParams wraps configuration options to create a DagBuilderHelper +// from a chunker.Splitter. +type DagBuilderParams struct { + // Maximum number of links per intermediate node + Maxlinks int + + // RawLeaves signifies that the importer should use raw ipld nodes as leaves + // instead of using the unixfs TRaw type + RawLeaves bool + + // CID Builder to use if set + CidBuilder cid.Builder + + // DAGService to write blocks to (required) + Dagserv ipld.DAGService + + // NoCopy signals to the chunker that it should track fileinfo for + // filestore adds + NoCopy bool +} + +// New generates a new DagBuilderHelper from the given params and a given +// chunker.Splitter as data source. +func (dbp *DagBuilderParams) New(spl chunker.Splitter) (*DagBuilderHelper, error) { + db := &DagBuilderHelper{ + dserv: dbp.Dagserv, + spl: spl, + rawLeaves: dbp.RawLeaves, + cidBuilder: dbp.CidBuilder, + maxlinks: dbp.Maxlinks, + } + if fi, ok := spl.Reader().(files.FileInfo); dbp.NoCopy && ok { + db.fullPath = fi.AbsPath() + db.stat = fi.Stat() + } + + if dbp.NoCopy && db.fullPath == "" { // Enforce NoCopy + return nil, ErrMissingFsRef + } + + return db, nil +} + +// prepareNext consumes the next item from the splitter and puts it +// in the nextData field. it is idempotent-- if nextData is full +// it will do nothing. +func (db *DagBuilderHelper) prepareNext() { + // if we already have data waiting to be consumed, we're ready + if db.nextData != nil || db.recvdErr != nil { + return + } + + db.nextData, db.recvdErr = db.spl.NextBytes() + if db.recvdErr == io.EOF { + db.recvdErr = nil + } +} + +// Done returns whether or not we're done consuming the incoming data. +func (db *DagBuilderHelper) Done() bool { + // ensure we have an accurate perspective on data + // as `done` this may be called before `next`. + db.prepareNext() // idempotent + if db.recvdErr != nil { + return false + } + return db.nextData == nil +} + +// Next returns the next chunk of data to be inserted into the dag +// if it returns nil, that signifies that the stream is at an end, and +// that the current building operation should finish. +func (db *DagBuilderHelper) Next() ([]byte, error) { + db.prepareNext() // idempotent + d := db.nextData + db.nextData = nil // signal we've consumed it + if db.recvdErr != nil { + return nil, db.recvdErr + } + return d, nil +} + +// GetDagServ returns the dagservice object this Helper is using +func (db *DagBuilderHelper) GetDagServ() ipld.DAGService { + return db.dserv +} + +// GetCidBuilder returns the internal `cid.CidBuilder` set in the builder. +func (db *DagBuilderHelper) GetCidBuilder() cid.Builder { + return db.cidBuilder +} + +// NewLeafNode creates a leaf node filled with data. If rawLeaves is +// defined then a raw leaf will be returned. Otherwise, it will create +// and return `FSNodeOverDag` with `fsNodeType`. +func (db *DagBuilderHelper) NewLeafNode(data []byte, fsNodeType pb.Data_DataType) (ipld.Node, error) { + if len(data) > BlockSizeLimit { + return nil, ErrSizeLimitExceeded + } + + if db.rawLeaves { + // Encapsulate the data in a raw node. + if db.cidBuilder == nil { + return dag.NewRawNode(data), nil + } + rawnode, err := dag.NewRawNodeWPrefix(data, db.cidBuilder) + if err != nil { + return nil, err + } + return rawnode, nil + } + + // Encapsulate the data in UnixFS node (instead of a raw node). + fsNodeOverDag := db.NewFSNodeOverDag(fsNodeType) + fsNodeOverDag.SetFileData(data) + node, err := fsNodeOverDag.Commit() + if err != nil { + return nil, err + } + // TODO: Encapsulate this sequence of calls into a function that + // just returns the final `ipld.Node` avoiding going through + // `FSNodeOverDag`. + + return node, nil +} + +// FillNodeLayer will add datanodes as children to the give node until +// it is full in this layer or no more data. +// NOTE: This function creates raw data nodes so it only works +// for the `trickle.Layout`. +func (db *DagBuilderHelper) FillNodeLayer(node *FSNodeOverDag) error { + + // while we have room AND we're not done + for node.NumChildren() < db.maxlinks && !db.Done() { + child, childFileSize, err := db.NewLeafDataNode(ft.TRaw) + if err != nil { + return err + } + + if err := node.AddChild(child, childFileSize, db); err != nil { + return err + } + } + // TODO: Do we need to commit here? The caller who created the + // `FSNodeOverDag` should be in charge of that. + _, err := node.Commit() + return err +} + +// NewLeafDataNode builds the `node` with the data obtained from the +// Splitter with the given constraints (BlockSizeLimit, RawLeaves) +// specified when creating the DagBuilderHelper. It returns +// `ipld.Node` with the `dataSize` (that will be used to keep track of +// the DAG file size). The size of the data is computed here because +// after that it will be hidden by `NewLeafNode` inside a generic +// `ipld.Node` representation. +func (db *DagBuilderHelper) NewLeafDataNode(fsNodeType pb.Data_DataType) (node ipld.Node, dataSize uint64, err error) { + fileData, err := db.Next() + if err != nil { + return nil, 0, err + } + dataSize = uint64(len(fileData)) + + // Create a new leaf node containing the file chunk data. + node, err = db.NewLeafNode(fileData, fsNodeType) + if err != nil { + return nil, 0, err + } + + // Convert this leaf to a `FilestoreNode` if needed. + node = db.ProcessFileStore(node, dataSize) + + return node, dataSize, nil +} + +// ProcessFileStore generates, if Filestore is being used, the +// `FilestoreNode` representation of the `ipld.Node` that +// contains the file data. If Filestore is not being used just +// return the same node to continue with its addition to the DAG. +// +// The `db.offset` is updated at this point (instead of when +// `NewLeafDataNode` is called, both work in tandem but the +// offset is more related to this function). +func (db *DagBuilderHelper) ProcessFileStore(node ipld.Node, dataSize uint64) ipld.Node { + // Check if Filestore is being used. + if db.fullPath != "" { + // Check if the node is actually a raw node (needed for + // Filestore support). + if _, ok := node.(*dag.RawNode); ok { + fn := &pi.FilestoreNode{ + Node: node, + PosInfo: &pi.PosInfo{ + Offset: db.offset, + FullPath: db.fullPath, + Stat: db.stat, + }, + } + + // Update `offset` with the size of the data generated by `db.Next`. + db.offset += dataSize + + return fn + } + } + + // Filestore is not used, return the same `node` argument. + return node +} + +// Add inserts the given node in the DAGService. +func (db *DagBuilderHelper) Add(node ipld.Node) error { + return db.dserv.Add(context.TODO(), node) +} + +// Maxlinks returns the configured maximum number for links +// for nodes built with this helper. +func (db *DagBuilderHelper) Maxlinks() int { + return db.maxlinks +} + +// FSNodeOverDag encapsulates an `unixfs.FSNode` that will be stored in a +// `dag.ProtoNode`. Instead of just having a single `ipld.Node` that +// would need to be constantly (un)packed to access and modify its +// internal `FSNode` in the process of creating a UnixFS DAG, this +// structure stores an `FSNode` cache to manipulate it (add child nodes) +// directly , and only when the node has reached its final (immutable) state +// (signaled by calling `Commit()`) is it committed to a single (indivisible) +// `ipld.Node`. +// +// It is used mainly for internal (non-leaf) nodes, and for some +// representations of data leaf nodes (that don't use raw nodes or +// Filestore). +// +// It aims to replace the `UnixfsNode` structure which encapsulated too +// many possible node state combinations. +// +// TODO: Revisit the name. +type FSNodeOverDag struct { + dag *dag.ProtoNode + file *ft.FSNode +} + +// NewFSNodeOverDag creates a new `dag.ProtoNode` and `ft.FSNode` +// decoupled from one onther (and will continue in that way until +// `Commit` is called), with `fsNodeType` specifying the type of +// the UnixFS layer node (either `File` or `Raw`). +func (db *DagBuilderHelper) NewFSNodeOverDag(fsNodeType pb.Data_DataType) *FSNodeOverDag { + node := new(FSNodeOverDag) + node.dag = new(dag.ProtoNode) + node.dag.SetCidBuilder(db.GetCidBuilder()) + + node.file = ft.NewFSNode(fsNodeType) + + return node +} + +// NewFSNFromDag reconstructs a FSNodeOverDag node from a given dag node +func (db *DagBuilderHelper) NewFSNFromDag(nd *dag.ProtoNode) (*FSNodeOverDag, error) { + return NewFSNFromDag(nd) +} + +// NewFSNFromDag reconstructs a FSNodeOverDag node from a given dag node +func NewFSNFromDag(nd *dag.ProtoNode) (*FSNodeOverDag, error) { + mb, err := ft.FSNodeFromBytes(nd.Data()) + if err != nil { + return nil, err + } + + return &FSNodeOverDag{ + dag: nd, + file: mb, + }, nil +} + +// AddChild adds a `child` `ipld.Node` to both node layers. The +// `dag.ProtoNode` creates a link to the child node while the +// `ft.FSNode` stores its file size (that is, not the size of the +// node but the size of the file data that it is storing at the +// UnixFS layer). The child is also stored in the `DAGService`. +func (n *FSNodeOverDag) AddChild(child ipld.Node, fileSize uint64, db *DagBuilderHelper) error { + err := n.dag.AddNodeLink("", child) + if err != nil { + return err + } + + n.file.AddBlockSize(fileSize) + + return db.Add(child) +} + +// RemoveChild deletes the child node at the given index. +func (n *FSNodeOverDag) RemoveChild(index int, dbh *DagBuilderHelper) { + n.file.RemoveBlockSize(index) + n.dag.SetLinks(append(n.dag.Links()[:index], n.dag.Links()[index+1:]...)) +} + +// Commit unifies (resolves) the cache nodes into a single `ipld.Node` +// that represents them: the `ft.FSNode` is encoded inside the +// `dag.ProtoNode`. +// +// TODO: Make it read-only after committing, allow to commit only once. +func (n *FSNodeOverDag) Commit() (ipld.Node, error) { + fileData, err := n.file.GetBytes() + if err != nil { + return nil, err + } + n.dag.SetData(fileData) + + return n.dag, nil +} + +// NumChildren returns the number of children of the `ft.FSNode`. +func (n *FSNodeOverDag) NumChildren() int { + return n.file.NumChildren() +} + +// FileSize returns the `Filesize` attribute from the underlying +// representation of the `ft.FSNode`. +func (n *FSNodeOverDag) FileSize() uint64 { + return n.file.FileSize() +} + +// SetFileData stores the `fileData` in the `ft.FSNode`. It +// should be used only when `FSNodeOverDag` represents a leaf +// node (internal nodes don't carry data, just file sizes). +func (n *FSNodeOverDag) SetFileData(fileData []byte) { + n.file.SetData(fileData) +} + +// GetDagNode fills out the proper formatting for the FSNodeOverDag node +// inside of a DAG node and returns the dag node. +// TODO: Check if we have committed (passed the UnixFS information +// to the DAG layer) before returning this. +func (n *FSNodeOverDag) GetDagNode() (ipld.Node, error) { + return n.dag, nil +} + +// GetChild gets the ith child of this node from the given DAGService. +func (n *FSNodeOverDag) GetChild(ctx context.Context, i int, ds ipld.DAGService) (*FSNodeOverDag, error) { + nd, err := n.dag.Links()[i].GetNode(ctx, ds) + if err != nil { + return nil, err + } + + pbn, ok := nd.(*dag.ProtoNode) + if !ok { + return nil, dag.ErrNotProtobuf + } + + return NewFSNFromDag(pbn) +} diff --git a/unixfs/importer/helpers/helpers.go b/unixfs/importer/helpers/helpers.go new file mode 100644 index 0000000000..20cb598e62 --- /dev/null +++ b/unixfs/importer/helpers/helpers.go @@ -0,0 +1,31 @@ +package helpers + +import ( + "fmt" +) + +// BlockSizeLimit specifies the maximum size an imported block can have. +var BlockSizeLimit = 1048576 // 1 MB + +// rough estimates on expected sizes +var roughLinkBlockSize = 1 << 13 // 8KB +var roughLinkSize = 34 + 8 + 5 // sha256 multihash + size + no name + protobuf framing + +// DefaultLinksPerBlock governs how the importer decides how many links there +// will be per block. This calculation is based on expected distributions of: +// - the expected distribution of block sizes +// - the expected distribution of link sizes +// - desired access speed +// +// For now, we use: +// +// var roughLinkBlockSize = 1 << 13 // 8KB +// var roughLinkSize = 34 + 8 + 5 // sha256 multihash + size + no name +// // + protobuf framing +// var DefaultLinksPerBlock = (roughLinkBlockSize / roughLinkSize) +// = ( 8192 / 47 ) +// = (approximately) 174 +var DefaultLinksPerBlock = roughLinkBlockSize / roughLinkSize + +// ErrSizeLimitExceeded signals that a block is larger than BlockSizeLimit. +var ErrSizeLimitExceeded = fmt.Errorf("object size limit exceeded") diff --git a/unixfs/importer/importer.go b/unixfs/importer/importer.go new file mode 100644 index 0000000000..04afa186fb --- /dev/null +++ b/unixfs/importer/importer.go @@ -0,0 +1,41 @@ +// Package importer implements utilities used to create IPFS DAGs from files +// and readers. +package importer + +import ( + bal "github.com/ipfs/go-libipfs/unixfs/importer/balanced" + h "github.com/ipfs/go-libipfs/unixfs/importer/helpers" + trickle "github.com/ipfs/go-libipfs/unixfs/importer/trickle" + + ipld "github.com/ipfs/go-ipld-format" + chunker "github.com/ipfs/go-libipfs/chunker" +) + +// BuildDagFromReader creates a DAG given a DAGService and a Splitter +// implementation (Splitters are io.Readers), using a Balanced layout. +func BuildDagFromReader(ds ipld.DAGService, spl chunker.Splitter) (ipld.Node, error) { + dbp := h.DagBuilderParams{ + Dagserv: ds, + Maxlinks: h.DefaultLinksPerBlock, + } + db, err := dbp.New(spl) + if err != nil { + return nil, err + } + return bal.Layout(db) +} + +// BuildTrickleDagFromReader creates a DAG given a DAGService and a Splitter +// implementation (Splitters are io.Readers), using a Trickle Layout. +func BuildTrickleDagFromReader(ds ipld.DAGService, spl chunker.Splitter) (ipld.Node, error) { + dbp := h.DagBuilderParams{ + Dagserv: ds, + Maxlinks: h.DefaultLinksPerBlock, + } + + db, err := dbp.New(spl) + if err != nil { + return nil, err + } + return trickle.Layout(db) +} diff --git a/unixfs/importer/importer_test.go b/unixfs/importer/importer_test.go new file mode 100644 index 0000000000..b90e6c2079 --- /dev/null +++ b/unixfs/importer/importer_test.go @@ -0,0 +1,152 @@ +package importer + +import ( + "bytes" + "context" + "io" + "testing" + + uio "github.com/ipfs/go-libipfs/unixfs/io" + + cid "github.com/ipfs/go-cid" + ipld "github.com/ipfs/go-ipld-format" + chunker "github.com/ipfs/go-libipfs/chunker" + u "github.com/ipfs/go-libipfs/util" + mdtest "github.com/ipfs/go-merkledag/test" +) + +func getBalancedDag(t testing.TB, size int64, blksize int64) (ipld.Node, ipld.DAGService) { + ds := mdtest.Mock() + r := io.LimitReader(u.NewTimeSeededRand(), size) + nd, err := BuildDagFromReader(ds, chunker.NewSizeSplitter(r, blksize)) + if err != nil { + t.Fatal(err) + } + return nd, ds +} + +func getTrickleDag(t testing.TB, size int64, blksize int64) (ipld.Node, ipld.DAGService) { + ds := mdtest.Mock() + r := io.LimitReader(u.NewTimeSeededRand(), size) + nd, err := BuildTrickleDagFromReader(ds, chunker.NewSizeSplitter(r, blksize)) + if err != nil { + t.Fatal(err) + } + return nd, ds +} + +func TestStableCid(t *testing.T) { + ds := mdtest.Mock() + buf := make([]byte, 10*1024*1024) + u.NewSeededRand(0xdeadbeef).Read(buf) + r := bytes.NewReader(buf) + + nd, err := BuildDagFromReader(ds, chunker.DefaultSplitter(r)) + if err != nil { + t.Fatal(err) + } + + expected, err := cid.Decode("QmZN1qquw84zhV4j6vT56tCcmFxaDaySL1ezTXFvMdNmrK") + if err != nil { + t.Fatal(err) + } + if !expected.Equals(nd.Cid()) { + t.Fatalf("expected CID %s, got CID %s", expected, nd) + } + + dr, err := uio.NewDagReader(context.Background(), nd, ds) + if err != nil { + t.Fatal(err) + } + + out, err := io.ReadAll(dr) + if err != nil { + t.Fatal(err) + } + + if !bytes.Equal(out, buf) { + t.Fatal("bad read") + } +} + +func TestBalancedDag(t *testing.T) { + ds := mdtest.Mock() + buf := make([]byte, 10000) + u.NewTimeSeededRand().Read(buf) + r := bytes.NewReader(buf) + + nd, err := BuildDagFromReader(ds, chunker.DefaultSplitter(r)) + if err != nil { + t.Fatal(err) + } + + dr, err := uio.NewDagReader(context.Background(), nd, ds) + if err != nil { + t.Fatal(err) + } + + out, err := io.ReadAll(dr) + if err != nil { + t.Fatal(err) + } + + if !bytes.Equal(out, buf) { + t.Fatal("bad read") + } +} + +func BenchmarkBalancedReadSmallBlock(b *testing.B) { + b.StopTimer() + nbytes := int64(10000000) + nd, ds := getBalancedDag(b, nbytes, 4096) + + b.SetBytes(nbytes) + b.StartTimer() + runReadBench(b, nd, ds) +} + +func BenchmarkTrickleReadSmallBlock(b *testing.B) { + b.StopTimer() + nbytes := int64(10000000) + nd, ds := getTrickleDag(b, nbytes, 4096) + + b.SetBytes(nbytes) + b.StartTimer() + runReadBench(b, nd, ds) +} + +func BenchmarkBalancedReadFull(b *testing.B) { + b.StopTimer() + nbytes := int64(10000000) + nd, ds := getBalancedDag(b, nbytes, chunker.DefaultBlockSize) + + b.SetBytes(nbytes) + b.StartTimer() + runReadBench(b, nd, ds) +} + +func BenchmarkTrickleReadFull(b *testing.B) { + b.StopTimer() + nbytes := int64(10000000) + nd, ds := getTrickleDag(b, nbytes, chunker.DefaultBlockSize) + + b.SetBytes(nbytes) + b.StartTimer() + runReadBench(b, nd, ds) +} + +func runReadBench(b *testing.B, nd ipld.Node, ds ipld.DAGService) { + for i := 0; i < b.N; i++ { + ctx, cancel := context.WithCancel(context.Background()) + read, err := uio.NewDagReader(ctx, nd, ds) + if err != nil { + b.Fatal(err) + } + + _, err = read.WriteTo(io.Discard) + if err != nil && err != io.EOF { + b.Fatal(err) + } + cancel() + } +} diff --git a/unixfs/importer/trickle/trickle_test.go b/unixfs/importer/trickle/trickle_test.go new file mode 100644 index 0000000000..0a81310469 --- /dev/null +++ b/unixfs/importer/trickle/trickle_test.go @@ -0,0 +1,666 @@ +package trickle + +import ( + "bytes" + "context" + "fmt" + "io" + mrand "math/rand" + "testing" + + ft "github.com/ipfs/go-libipfs/unixfs" + h "github.com/ipfs/go-libipfs/unixfs/importer/helpers" + uio "github.com/ipfs/go-libipfs/unixfs/io" + + ipld "github.com/ipfs/go-ipld-format" + chunker "github.com/ipfs/go-libipfs/chunker" + u "github.com/ipfs/go-libipfs/util" + merkledag "github.com/ipfs/go-merkledag" + mdtest "github.com/ipfs/go-merkledag/test" +) + +type UseRawLeaves bool + +const ( + ProtoBufLeaves UseRawLeaves = false + RawLeaves UseRawLeaves = true +) + +func runBothSubtests(t *testing.T, tfunc func(*testing.T, UseRawLeaves)) { + t.Run("leaves=ProtoBuf", func(t *testing.T) { tfunc(t, ProtoBufLeaves) }) + t.Run("leaves=Raw", func(t *testing.T) { tfunc(t, RawLeaves) }) +} + +func buildTestDag(ds ipld.DAGService, spl chunker.Splitter, rawLeaves UseRawLeaves) (*merkledag.ProtoNode, error) { + dbp := h.DagBuilderParams{ + Dagserv: ds, + Maxlinks: h.DefaultLinksPerBlock, + RawLeaves: bool(rawLeaves), + } + + db, err := dbp.New(spl) + if err != nil { + return nil, err + } + + nd, err := Layout(db) + if err != nil { + return nil, err + } + + pbnd, ok := nd.(*merkledag.ProtoNode) + if !ok { + return nil, merkledag.ErrNotProtobuf + } + + return pbnd, VerifyTrickleDagStructure(pbnd, VerifyParams{ + Getter: ds, + Direct: dbp.Maxlinks, + LayerRepeat: depthRepeat, + RawLeaves: bool(rawLeaves), + }) +} + +// Test where calls to read are smaller than the chunk size +func TestSizeBasedSplit(t *testing.T) { + runBothSubtests(t, testSizeBasedSplit) +} + +func testSizeBasedSplit(t *testing.T, rawLeaves UseRawLeaves) { + if testing.Short() { + t.SkipNow() + } + bs := chunker.SizeSplitterGen(512) + testFileConsistency(t, bs, 32*512, rawLeaves) + + bs = chunker.SizeSplitterGen(4096) + testFileConsistency(t, bs, 32*4096, rawLeaves) + + // Uneven offset + testFileConsistency(t, bs, 31*4095, rawLeaves) +} + +func dup(b []byte) []byte { + o := make([]byte, len(b)) + copy(o, b) + return o +} + +func testFileConsistency(t *testing.T, bs chunker.SplitterGen, nbytes int, rawLeaves UseRawLeaves) { + should := make([]byte, nbytes) + u.NewTimeSeededRand().Read(should) + + read := bytes.NewReader(should) + ds := mdtest.Mock() + nd, err := buildTestDag(ds, bs(read), rawLeaves) + if err != nil { + t.Fatal(err) + } + + r, err := uio.NewDagReader(context.Background(), nd, ds) + if err != nil { + t.Fatal(err) + } + + out, err := io.ReadAll(r) + if err != nil { + t.Fatal(err) + } + + err = arrComp(out, should) + if err != nil { + t.Fatal(err) + } +} + +func TestBuilderConsistency(t *testing.T) { + runBothSubtests(t, testBuilderConsistency) +} + +func testBuilderConsistency(t *testing.T, rawLeaves UseRawLeaves) { + nbytes := 100000 + buf := new(bytes.Buffer) + io.CopyN(buf, u.NewTimeSeededRand(), int64(nbytes)) + should := dup(buf.Bytes()) + dagserv := mdtest.Mock() + nd, err := buildTestDag(dagserv, chunker.DefaultSplitter(buf), rawLeaves) + if err != nil { + t.Fatal(err) + } + r, err := uio.NewDagReader(context.Background(), nd, dagserv) + if err != nil { + t.Fatal(err) + } + + out, err := io.ReadAll(r) + if err != nil { + t.Fatal(err) + } + + err = arrComp(out, should) + if err != nil { + t.Fatal(err) + } +} + +func arrComp(a, b []byte) error { + if len(a) != len(b) { + return fmt.Errorf("arrays differ in length. %d != %d", len(a), len(b)) + } + for i, v := range a { + if v != b[i] { + return fmt.Errorf("arrays differ at index: %d", i) + } + } + return nil +} + +func TestIndirectBlocks(t *testing.T) { + runBothSubtests(t, testIndirectBlocks) +} + +func testIndirectBlocks(t *testing.T, rawLeaves UseRawLeaves) { + splitter := chunker.SizeSplitterGen(512) + nbytes := 1024 * 1024 + buf := make([]byte, nbytes) + u.NewTimeSeededRand().Read(buf) + + read := bytes.NewReader(buf) + + ds := mdtest.Mock() + dag, err := buildTestDag(ds, splitter(read), rawLeaves) + if err != nil { + t.Fatal(err) + } + + reader, err := uio.NewDagReader(context.Background(), dag, ds) + if err != nil { + t.Fatal(err) + } + + out, err := io.ReadAll(reader) + if err != nil { + t.Fatal(err) + } + + if !bytes.Equal(out, buf) { + t.Fatal("Not equal!") + } +} + +func TestSeekingBasic(t *testing.T) { + runBothSubtests(t, testSeekingBasic) +} + +func testSeekingBasic(t *testing.T, rawLeaves UseRawLeaves) { + nbytes := int64(10 * 1024) + should := make([]byte, nbytes) + u.NewTimeSeededRand().Read(should) + + read := bytes.NewReader(should) + ds := mdtest.Mock() + nd, err := buildTestDag(ds, chunker.NewSizeSplitter(read, 512), rawLeaves) + if err != nil { + t.Fatal(err) + } + + rs, err := uio.NewDagReader(context.Background(), nd, ds) + if err != nil { + t.Fatal(err) + } + + start := int64(4000) + n, err := rs.Seek(start, io.SeekStart) + if err != nil { + t.Fatal(err) + } + if n != start { + t.Fatal("Failed to seek to correct offset") + } + + out, err := io.ReadAll(rs) + if err != nil { + t.Fatal(err) + } + + err = arrComp(out, should[start:]) + if err != nil { + t.Fatal(err) + } +} + +func TestSeekToBegin(t *testing.T) { + runBothSubtests(t, testSeekToBegin) +} + +func testSeekToBegin(t *testing.T, rawLeaves UseRawLeaves) { + nbytes := int64(10 * 1024) + should := make([]byte, nbytes) + u.NewTimeSeededRand().Read(should) + + read := bytes.NewReader(should) + ds := mdtest.Mock() + nd, err := buildTestDag(ds, chunker.NewSizeSplitter(read, 500), rawLeaves) + if err != nil { + t.Fatal(err) + } + + rs, err := uio.NewDagReader(context.Background(), nd, ds) + if err != nil { + t.Fatal(err) + } + + n, err := io.CopyN(io.Discard, rs, 1024*4) + if err != nil { + t.Fatal(err) + } + if n != 4096 { + t.Fatal("Copy didnt copy enough bytes") + } + + seeked, err := rs.Seek(0, io.SeekStart) + if err != nil { + t.Fatal(err) + } + if seeked != 0 { + t.Fatal("Failed to seek to beginning") + } + + out, err := io.ReadAll(rs) + if err != nil { + t.Fatal(err) + } + + err = arrComp(out, should) + if err != nil { + t.Fatal(err) + } +} + +func TestSeekToAlmostBegin(t *testing.T) { + runBothSubtests(t, testSeekToAlmostBegin) +} + +func testSeekToAlmostBegin(t *testing.T, rawLeaves UseRawLeaves) { + nbytes := int64(10 * 1024) + should := make([]byte, nbytes) + u.NewTimeSeededRand().Read(should) + + read := bytes.NewReader(should) + ds := mdtest.Mock() + nd, err := buildTestDag(ds, chunker.NewSizeSplitter(read, 500), rawLeaves) + if err != nil { + t.Fatal(err) + } + + rs, err := uio.NewDagReader(context.Background(), nd, ds) + if err != nil { + t.Fatal(err) + } + + n, err := io.CopyN(io.Discard, rs, 1024*4) + if err != nil { + t.Fatal(err) + } + if n != 4096 { + t.Fatal("Copy didnt copy enough bytes") + } + + seeked, err := rs.Seek(1, io.SeekStart) + if err != nil { + t.Fatal(err) + } + if seeked != 1 { + t.Fatal("Failed to seek to almost beginning") + } + + out, err := io.ReadAll(rs) + if err != nil { + t.Fatal(err) + } + + err = arrComp(out, should[1:]) + if err != nil { + t.Fatal(err) + } +} + +func TestSeekEnd(t *testing.T) { + runBothSubtests(t, testSeekEnd) +} + +func testSeekEnd(t *testing.T, rawLeaves UseRawLeaves) { + nbytes := int64(50 * 1024) + should := make([]byte, nbytes) + u.NewTimeSeededRand().Read(should) + + read := bytes.NewReader(should) + ds := mdtest.Mock() + nd, err := buildTestDag(ds, chunker.NewSizeSplitter(read, 500), rawLeaves) + if err != nil { + t.Fatal(err) + } + + rs, err := uio.NewDagReader(context.Background(), nd, ds) + if err != nil { + t.Fatal(err) + } + + seeked, err := rs.Seek(0, io.SeekEnd) + if err != nil { + t.Fatal(err) + } + if seeked != nbytes { + t.Fatal("Failed to seek to end") + } +} + +func TestSeekEndSingleBlockFile(t *testing.T) { + runBothSubtests(t, testSeekEndSingleBlockFile) +} + +func testSeekEndSingleBlockFile(t *testing.T, rawLeaves UseRawLeaves) { + nbytes := int64(100) + should := make([]byte, nbytes) + u.NewTimeSeededRand().Read(should) + + read := bytes.NewReader(should) + ds := mdtest.Mock() + nd, err := buildTestDag(ds, chunker.NewSizeSplitter(read, 5000), rawLeaves) + if err != nil { + t.Fatal(err) + } + + rs, err := uio.NewDagReader(context.Background(), nd, ds) + if err != nil { + t.Fatal(err) + } + + seeked, err := rs.Seek(0, io.SeekEnd) + if err != nil { + t.Fatal(err) + } + if seeked != nbytes { + t.Fatal("Failed to seek to end") + } +} + +func TestSeekingStress(t *testing.T) { + runBothSubtests(t, testSeekingStress) +} + +func testSeekingStress(t *testing.T, rawLeaves UseRawLeaves) { + nbytes := int64(1024 * 1024) + should := make([]byte, nbytes) + u.NewTimeSeededRand().Read(should) + + read := bytes.NewReader(should) + ds := mdtest.Mock() + nd, err := buildTestDag(ds, chunker.NewSizeSplitter(read, 1000), rawLeaves) + if err != nil { + t.Fatal(err) + } + + rs, err := uio.NewDagReader(context.Background(), nd, ds) + if err != nil { + t.Fatal(err) + } + + testbuf := make([]byte, nbytes) + for i := 0; i < 50; i++ { + offset := mrand.Intn(int(nbytes)) + l := int(nbytes) - offset + n, err := rs.Seek(int64(offset), io.SeekStart) + if err != nil { + t.Fatal(err) + } + if n != int64(offset) { + t.Fatal("Seek failed to move to correct position") + } + + nread, err := rs.Read(testbuf[:l]) + if err != nil { + t.Fatal(err) + } + if nread != l { + t.Fatal("Failed to read enough bytes") + } + + err = arrComp(testbuf[:l], should[offset:offset+l]) + if err != nil { + t.Fatal(err) + } + } + +} + +func TestSeekingConsistency(t *testing.T) { + runBothSubtests(t, testSeekingConsistency) +} + +func testSeekingConsistency(t *testing.T, rawLeaves UseRawLeaves) { + nbytes := int64(128 * 1024) + should := make([]byte, nbytes) + u.NewTimeSeededRand().Read(should) + + read := bytes.NewReader(should) + ds := mdtest.Mock() + nd, err := buildTestDag(ds, chunker.NewSizeSplitter(read, 500), rawLeaves) + if err != nil { + t.Fatal(err) + } + + rs, err := uio.NewDagReader(context.Background(), nd, ds) + if err != nil { + t.Fatal(err) + } + + out := make([]byte, nbytes) + + for coff := nbytes - 4096; coff >= 0; coff -= 4096 { + t.Log(coff) + n, err := rs.Seek(coff, io.SeekStart) + if err != nil { + t.Fatal(err) + } + if n != coff { + t.Fatal("wasnt able to seek to the right position") + } + nread, err := rs.Read(out[coff : coff+4096]) + if err != nil { + t.Fatal(err) + } + if nread != 4096 { + t.Fatal("didnt read the correct number of bytes") + } + } + + err = arrComp(out, should) + if err != nil { + t.Fatal(err) + } +} + +func TestAppend(t *testing.T) { + runBothSubtests(t, testAppend) +} + +func testAppend(t *testing.T, rawLeaves UseRawLeaves) { + nbytes := int64(128 * 1024) + should := make([]byte, nbytes) + u.NewTimeSeededRand().Read(should) + + // Reader for half the bytes + read := bytes.NewReader(should[:nbytes/2]) + ds := mdtest.Mock() + nd, err := buildTestDag(ds, chunker.NewSizeSplitter(read, 500), rawLeaves) + if err != nil { + t.Fatal(err) + } + + dbp := &h.DagBuilderParams{ + Dagserv: ds, + Maxlinks: h.DefaultLinksPerBlock, + RawLeaves: bool(rawLeaves), + } + + r := bytes.NewReader(should[nbytes/2:]) + + ctx := context.Background() + + db, err := dbp.New(chunker.NewSizeSplitter(r, 500)) + if err != nil { + t.Fatal(err) + } + + nnode, err := Append(ctx, nd, db) + if err != nil { + t.Fatal(err) + } + + err = VerifyTrickleDagStructure(nnode, VerifyParams{ + Getter: ds, + Direct: dbp.Maxlinks, + LayerRepeat: depthRepeat, + RawLeaves: bool(rawLeaves), + }) + if err != nil { + t.Fatal(err) + } + + fread, err := uio.NewDagReader(ctx, nnode, ds) + if err != nil { + t.Fatal(err) + } + + out, err := io.ReadAll(fread) + if err != nil { + t.Fatal(err) + } + + err = arrComp(out, should) + if err != nil { + t.Fatal(err) + } +} + +// This test appends one byte at a time to an empty file +func TestMultipleAppends(t *testing.T) { + runBothSubtests(t, testMultipleAppends) +} + +func testMultipleAppends(t *testing.T, rawLeaves UseRawLeaves) { + ds := mdtest.Mock() + + // TODO: fix small size appends and make this number bigger + nbytes := int64(1000) + should := make([]byte, nbytes) + u.NewTimeSeededRand().Read(should) + + read := bytes.NewReader(nil) + nd, err := buildTestDag(ds, chunker.NewSizeSplitter(read, 500), rawLeaves) + if err != nil { + t.Fatal(err) + } + + dbp := &h.DagBuilderParams{ + Dagserv: ds, + Maxlinks: 4, + RawLeaves: bool(rawLeaves), + } + + spl := chunker.SizeSplitterGen(500) + + ctx := context.Background() + for i := 0; i < len(should); i++ { + + db, err := dbp.New(spl(bytes.NewReader(should[i : i+1]))) + if err != nil { + t.Fatal(err) + } + + nnode, err := Append(ctx, nd, db) + if err != nil { + t.Fatal(err) + } + + err = VerifyTrickleDagStructure(nnode, VerifyParams{ + Getter: ds, + Direct: dbp.Maxlinks, + LayerRepeat: depthRepeat, + RawLeaves: bool(rawLeaves), + }) + if err != nil { + t.Fatal(err) + } + + fread, err := uio.NewDagReader(ctx, nnode, ds) + if err != nil { + t.Fatal(err) + } + + out, err := io.ReadAll(fread) + if err != nil { + t.Fatal(err) + } + + err = arrComp(out, should[:i+1]) + if err != nil { + t.Fatal(err) + } + } +} + +func TestAppendSingleBytesToEmpty(t *testing.T) { + ds := mdtest.Mock() + + data := []byte("AB") + + nd := new(merkledag.ProtoNode) + nd.SetData(ft.FilePBData(nil, 0)) + + dbp := &h.DagBuilderParams{ + Dagserv: ds, + Maxlinks: 4, + } + + spl := chunker.SizeSplitterGen(500) + + ctx := context.Background() + + db, err := dbp.New(spl(bytes.NewReader(data[:1]))) + if err != nil { + t.Fatal(err) + } + + nnode, err := Append(ctx, nd, db) + if err != nil { + t.Fatal(err) + } + + db, err = dbp.New(spl(bytes.NewReader(data[1:]))) + if err != nil { + t.Fatal(err) + } + + nnode, err = Append(ctx, nnode, db) + if err != nil { + t.Fatal(err) + } + + fread, err := uio.NewDagReader(ctx, nnode, ds) + if err != nil { + t.Fatal(err) + } + + out, err := io.ReadAll(fread) + if err != nil { + t.Fatal(err) + } + + fmt.Println(out, data) + err = arrComp(out, data) + if err != nil { + t.Fatal(err) + } +} diff --git a/unixfs/importer/trickle/trickledag.go b/unixfs/importer/trickle/trickledag.go new file mode 100644 index 0000000000..0fa55abf7a --- /dev/null +++ b/unixfs/importer/trickle/trickledag.go @@ -0,0 +1,389 @@ +// Package trickle allows to build trickle DAGs. +// In this type of DAG, non-leave nodes are first filled +// with data leaves, and then incorporate "layers" of subtrees +// as additional links. +// +// Each layer is a trickle sub-tree and is limited by an increasing +// maximum depth. Thus, the nodes first layer +// can only hold leaves (depth 1) but subsequent layers can grow deeper. +// By default, this module places 4 nodes per layer (that is, 4 subtrees +// of the same maximum depth before increasing it). +// +// Trickle DAGs are very good for sequentially reading data, as the +// first data leaves are directly reachable from the root and those +// coming next are always nearby. They are +// suited for things like streaming applications. +package trickle + +import ( + "context" + "errors" + "fmt" + + ft "github.com/ipfs/go-libipfs/unixfs" + h "github.com/ipfs/go-libipfs/unixfs/importer/helpers" + + cid "github.com/ipfs/go-cid" + ipld "github.com/ipfs/go-ipld-format" + dag "github.com/ipfs/go-merkledag" +) + +// depthRepeat specifies how many times to append a child tree of a +// given depth. Higher values increase the width of a given node, which +// improves seek speeds. +const depthRepeat = 4 + +// Layout builds a new DAG with the trickle format using the provided +// DagBuilderHelper. See the module's description for a more detailed +// explanation. +func Layout(db *h.DagBuilderHelper) (ipld.Node, error) { + newRoot := db.NewFSNodeOverDag(ft.TFile) + root, _, err := fillTrickleRec(db, newRoot, -1) + if err != nil { + return nil, err + } + + return root, db.Add(root) +} + +// fillTrickleRec creates a trickle (sub-)tree with an optional maximum specified depth +// in the case maxDepth is greater than zero, or with unlimited depth otherwise +// (where the DAG builder will signal the end of data to end the function). +func fillTrickleRec(db *h.DagBuilderHelper, node *h.FSNodeOverDag, maxDepth int) (filledNode ipld.Node, nodeFileSize uint64, err error) { + // Always do this, even in the base case + if err := db.FillNodeLayer(node); err != nil { + return nil, 0, err + } + + // For each depth in [1, `maxDepth`) (or without limit if `maxDepth` is -1, + // initial call from `Layout`) add `depthRepeat` sub-graphs of that depth. + for depth := 1; maxDepth == -1 || depth < maxDepth; depth++ { + if db.Done() { + break + // No more data, stop here, posterior append calls will figure out + // where we left off. + } + + for repeatIndex := 0; repeatIndex < depthRepeat && !db.Done(); repeatIndex++ { + + childNode, childFileSize, err := fillTrickleRec(db, db.NewFSNodeOverDag(ft.TFile), depth) + if err != nil { + return nil, 0, err + } + + if err := node.AddChild(childNode, childFileSize, db); err != nil { + return nil, 0, err + } + } + } + + // Get the final `dag.ProtoNode` with the `FSNode` data encoded inside. + filledNode, err = node.Commit() + if err != nil { + return nil, 0, err + } + + return filledNode, node.FileSize(), nil +} + +// Append appends the data in `db` to the dag, using the Trickledag format +func Append(ctx context.Context, basen ipld.Node, db *h.DagBuilderHelper) (out ipld.Node, errOut error) { + base, ok := basen.(*dag.ProtoNode) + if !ok { + return nil, dag.ErrNotProtobuf + } + + // Convert to unixfs node for working with easily + + fsn, err := h.NewFSNFromDag(base) + if err != nil { + return nil, err + } + + // Get depth of this 'tree' + depth, repeatNumber := trickleDepthInfo(fsn, db.Maxlinks()) + if depth == 0 { + // If direct blocks not filled... + if err := db.FillNodeLayer(fsn); err != nil { + return nil, err + } + + if db.Done() { + // TODO: If `FillNodeLayer` stop `Commit`ing this should be + // the place (besides the function end) to call it. + return fsn.GetDagNode() + } + + // If continuing, our depth has increased by one + depth++ + } + + // Last child in this node may not be a full tree, lets fill it up. + if err := appendFillLastChild(ctx, fsn, depth-1, repeatNumber, db); err != nil { + return nil, err + } + + // after appendFillLastChild, our depth is now increased by one + if !db.Done() { + depth++ + } + + // Now, continue filling out tree like normal + for i := depth; !db.Done(); i++ { + for j := 0; j < depthRepeat && !db.Done(); j++ { + nextChild := db.NewFSNodeOverDag(ft.TFile) + childNode, childFileSize, err := fillTrickleRec(db, nextChild, i) + if err != nil { + return nil, err + } + err = fsn.AddChild(childNode, childFileSize, db) + if err != nil { + return nil, err + } + } + } + _, err = fsn.Commit() + if err != nil { + return nil, err + } + return fsn.GetDagNode() +} + +func appendFillLastChild(ctx context.Context, fsn *h.FSNodeOverDag, depth int, repeatNumber int, db *h.DagBuilderHelper) error { + if fsn.NumChildren() <= db.Maxlinks() { + return nil + } + // TODO: Why do we need this check, didn't the caller already take + // care of this? + + // Recursive step, grab last child + last := fsn.NumChildren() - 1 + lastChild, err := fsn.GetChild(ctx, last, db.GetDagServ()) + if err != nil { + return err + } + + // Fill out last child (may not be full tree) + newChild, nchildSize, err := appendRec(ctx, lastChild, db, depth-1) + if err != nil { + return err + } + + // Update changed child in parent node + fsn.RemoveChild(last, db) + filledNode, err := newChild.Commit() + if err != nil { + return err + } + err = fsn.AddChild(filledNode, nchildSize, db) + if err != nil { + return err + } + + // Partially filled depth layer + if repeatNumber != 0 { + for ; repeatNumber < depthRepeat && !db.Done(); repeatNumber++ { + nextChild := db.NewFSNodeOverDag(ft.TFile) + childNode, childFileSize, err := fillTrickleRec(db, nextChild, depth) + if err != nil { + return err + } + + if err := fsn.AddChild(childNode, childFileSize, db); err != nil { + return err + } + } + } + + return nil +} + +// recursive call for Append +func appendRec(ctx context.Context, fsn *h.FSNodeOverDag, db *h.DagBuilderHelper, maxDepth int) (*h.FSNodeOverDag, uint64, error) { + if maxDepth == 0 || db.Done() { + return fsn, fsn.FileSize(), nil + } + + // Get depth of this 'tree' + depth, repeatNumber := trickleDepthInfo(fsn, db.Maxlinks()) + if depth == 0 { + // If direct blocks not filled... + if err := db.FillNodeLayer(fsn); err != nil { + return nil, 0, err + } + depth++ + } + // TODO: Same as `appendFillLastChild`, when is this case possible? + + // If at correct depth, no need to continue + if depth == maxDepth { + return fsn, fsn.FileSize(), nil + } + + if err := appendFillLastChild(ctx, fsn, depth, repeatNumber, db); err != nil { + return nil, 0, err + } + + // after appendFillLastChild, our depth is now increased by one + if !db.Done() { + depth++ + } + + // Now, continue filling out tree like normal + for i := depth; i < maxDepth && !db.Done(); i++ { + for j := 0; j < depthRepeat && !db.Done(); j++ { + nextChild := db.NewFSNodeOverDag(ft.TFile) + childNode, childFileSize, err := fillTrickleRec(db, nextChild, i) + if err != nil { + return nil, 0, err + } + + if err := fsn.AddChild(childNode, childFileSize, db); err != nil { + return nil, 0, err + } + } + } + + return fsn, fsn.FileSize(), nil +} + +// Deduce where we left off in `fillTrickleRec`, returns the `depth` +// with which new sub-graphs were being added and, within that depth, +// in which `repeatNumber` of the total `depthRepeat` we should add. +func trickleDepthInfo(node *h.FSNodeOverDag, maxlinks int) (depth int, repeatNumber int) { + n := node.NumChildren() + + if n < maxlinks { + // We didn't even added the initial `maxlinks` leaf nodes (`FillNodeLayer`). + return 0, 0 + } + + nonLeafChildren := n - maxlinks + // The number of non-leaf child nodes added in `fillTrickleRec` (after + // the `FillNodeLayer` call). + + depth = nonLeafChildren/depthRepeat + 1 + // "Deduplicate" the added `depthRepeat` sub-graphs at each depth + // (rounding it up since we may be on an unfinished depth with less + // than `depthRepeat` sub-graphs). + + repeatNumber = nonLeafChildren % depthRepeat + // What's left after taking full depths of `depthRepeat` sub-graphs + // is the current `repeatNumber` we're at (this fractional part is + // what we rounded up before). + + return +} + +// VerifyParams is used by VerifyTrickleDagStructure +type VerifyParams struct { + Getter ipld.NodeGetter + Direct int + LayerRepeat int + Prefix *cid.Prefix + RawLeaves bool +} + +// VerifyTrickleDagStructure checks that the given dag matches exactly the trickle dag datastructure +// layout +func VerifyTrickleDagStructure(nd ipld.Node, p VerifyParams) error { + return verifyTDagRec(nd, -1, p) +} + +// Recursive call for verifying the structure of a trickledag +func verifyTDagRec(n ipld.Node, depth int, p VerifyParams) error { + codec := cid.DagProtobuf + if depth == 0 { + if len(n.Links()) > 0 { + return errors.New("expected direct block") + } + // zero depth dag is raw data block + switch nd := n.(type) { + case *dag.ProtoNode: + fsn, err := ft.FSNodeFromBytes(nd.Data()) + if err != nil { + return err + } + + if fsn.Type() != ft.TRaw { + return errors.New("expected raw block") + } + + if p.RawLeaves { + return errors.New("expected raw leaf, got a protobuf node") + } + case *dag.RawNode: + if !p.RawLeaves { + return errors.New("expected protobuf node as leaf") + } + codec = cid.Raw + default: + return errors.New("expected ProtoNode or RawNode") + } + } + + // verify prefix + if p.Prefix != nil { + prefix := n.Cid().Prefix() + expect := *p.Prefix // make a copy + expect.Codec = uint64(codec) + if codec == cid.Raw && expect.Version == 0 { + expect.Version = 1 + } + if expect.MhLength == -1 { + expect.MhLength = prefix.MhLength + } + if prefix != expect { + return fmt.Errorf("unexpected cid prefix: expected: %v; got %v", expect, prefix) + } + } + + if depth == 0 { + return nil + } + + nd, ok := n.(*dag.ProtoNode) + if !ok { + return errors.New("expected ProtoNode") + } + + // Verify this is a branch node + fsn, err := ft.FSNodeFromBytes(nd.Data()) + if err != nil { + return err + } + + if fsn.Type() != ft.TFile { + return fmt.Errorf("expected file as branch node, got: %s", fsn.Type()) + } + + if len(fsn.Data()) > 0 { + return errors.New("branch node should not have data") + } + + for i := 0; i < len(nd.Links()); i++ { + child, err := nd.Links()[i].GetNode(context.TODO(), p.Getter) + if err != nil { + return err + } + + if i < p.Direct { + // Direct blocks + err := verifyTDagRec(child, 0, p) + if err != nil { + return err + } + } else { + // Recursive trickle dags + rdepth := ((i - p.Direct) / p.LayerRepeat) + 1 + if rdepth >= depth && depth > 0 { + return errors.New("child dag was too deep") + } + err := verifyTDagRec(child, rdepth, p) + if err != nil { + return err + } + } + } + return nil +} diff --git a/unixfs/internal/config.go b/unixfs/internal/config.go new file mode 100644 index 0000000000..9250ae2ae3 --- /dev/null +++ b/unixfs/internal/config.go @@ -0,0 +1,3 @@ +package internal + +var HAMTHashFunction func(val []byte) []byte diff --git a/unixfs/io/completehamt_test.go b/unixfs/io/completehamt_test.go new file mode 100644 index 0000000000..d055892599 --- /dev/null +++ b/unixfs/io/completehamt_test.go @@ -0,0 +1,101 @@ +package io + +import ( + "context" + "encoding/binary" + "fmt" + "math" + "testing" + + "github.com/ipfs/go-libipfs/unixfs/internal" + + mdtest "github.com/ipfs/go-merkledag/test" + "github.com/stretchr/testify/assert" + + "github.com/ipfs/go-libipfs/unixfs" + "github.com/ipfs/go-libipfs/unixfs/hamt" + + ipld "github.com/ipfs/go-ipld-format" +) + +// CreateCompleteHAMT creates a HAMT the following properties: +// - its height (distance/edges from root to deepest node) is specified by treeHeight. +// - all leaf Shard nodes have the same depth (and have only 'value' links). +// - all internal Shard nodes point only to other Shards (and hence have zero 'value' links). +// - the total number of 'value' links (directory entries) is: +// childsPerNode ^ (treeHeight). +// treeHeight: The number of layers of non-value HAMT nodes (e.g. height = 1 is a single shard pointing to some values) +// +// FIXME: HAMTHashFunction needs to be set to idHash by the caller. We depend on +// +// this simplification for the current logic to work. +func CreateCompleteHAMT(ds ipld.DAGService, treeHeight int, childsPerNode int) (ipld.Node, error) { + if treeHeight < 1 { + panic("treeHeight < 1") + } + if treeHeight > 8 { + panic("treeHeight > 8: we don't allow a key larger than what can be encoded in a 64-bit word") + } + + rootShard, err := hamt.NewShard(ds, childsPerNode) + if err != nil { + return nil, err + } + + // Assuming we are using the ID hash function we can just insert all + // the combinations of a byte slice that will reach the desired height. + totalChildren := int(math.Pow(float64(childsPerNode), float64(treeHeight))) + log2ofChilds, err := hamt.Logtwo(childsPerNode) + if err != nil { + return nil, err + } + if log2ofChilds*treeHeight%8 != 0 { + return nil, fmt.Errorf("childsPerNode * treeHeight should be multiple of 8") + } + bytesInKey := log2ofChilds * treeHeight / 8 + for i := 0; i < totalChildren; i++ { + var hashbuf [8]byte + binary.LittleEndian.PutUint64(hashbuf[:], uint64(i)) + var oldLink *ipld.Link + oldLink, err = rootShard.Swap(context.Background(), string(hashbuf[:bytesInKey]), unixfs.EmptyFileNode()) + if err != nil { + return nil, err + } + if oldLink != nil { + // We shouldn't be overwriting any value, otherwise the tree + // won't be complete. + return nil, fmt.Errorf("we have overwritten entry %s", + oldLink.Cid) + } + } + + return rootShard.Node() +} + +// Return the same value as the hash. +func idHash(val []byte) []byte { + return val +} + +// FIXME: This is not checking the exact height of the tree but just making +// +// sure there are as many children as we would have with a complete HAMT. +func TestCreateCompleteShard(t *testing.T) { + oldHashFunc := internal.HAMTHashFunction + defer func() { internal.HAMTHashFunction = oldHashFunc }() + internal.HAMTHashFunction = idHash + + ds := mdtest.Mock() + childsPerNode := 16 + treeHeight := 2 + node, err := CreateCompleteHAMT(ds, treeHeight, childsPerNode) + assert.NoError(t, err) + + shard, err := hamt.NewHamtFromDag(ds, node) + assert.NoError(t, err) + links, err := shard.EnumLinks(context.Background()) + assert.NoError(t, err) + + childNodes := int(math.Pow(float64(childsPerNode), float64(treeHeight))) + assert.Equal(t, childNodes, len(links)) +} diff --git a/unixfs/io/dagreader.go b/unixfs/io/dagreader.go new file mode 100644 index 0000000000..587a1b439e --- /dev/null +++ b/unixfs/io/dagreader.go @@ -0,0 +1,488 @@ +package io + +import ( + "bytes" + "context" + "errors" + "io" + + ipld "github.com/ipfs/go-ipld-format" + unixfs "github.com/ipfs/go-libipfs/unixfs" + mdag "github.com/ipfs/go-merkledag" +) + +// Common errors +var ( + ErrIsDir = errors.New("this dag node is a directory") + ErrCantReadSymlinks = errors.New("cannot currently read symlinks") + ErrUnkownNodeType = errors.New("unknown node type") + ErrSeekNotSupported = errors.New("file does not support seeking") +) + +// TODO: Rename the `DagReader` interface, this doesn't read *any* DAG, just +// DAGs with UnixFS node (and it *belongs* to the `unixfs` package). Some +// alternatives: `FileReader`, `UnixFSFileReader`, `UnixFSReader`. + +// A DagReader provides read-only read and seek acess to a unixfs file. +// Different implementations of readers are used for the different +// types of unixfs/protobuf-encoded nodes. +type DagReader interface { + ReadSeekCloser + Size() uint64 + CtxReadFull(context.Context, []byte) (int, error) +} + +// A ReadSeekCloser implements interfaces to read, copy, seek and close. +type ReadSeekCloser interface { + io.Reader + io.Seeker + io.Closer + io.WriterTo +} + +// NewDagReader creates a new reader object that reads the data represented by +// the given node, using the passed in DAGService for data retrieval. +func NewDagReader(ctx context.Context, n ipld.Node, serv ipld.NodeGetter) (DagReader, error) { + var size uint64 + + switch n := n.(type) { + case *mdag.RawNode: + size = uint64(len(n.RawData())) + + case *mdag.ProtoNode: + fsNode, err := unixfs.FSNodeFromBytes(n.Data()) + if err != nil { + return nil, err + } + + switch fsNode.Type() { + case unixfs.TFile, unixfs.TRaw: + size = fsNode.FileSize() + + case unixfs.TDirectory, unixfs.THAMTShard: + // Dont allow reading directories + return nil, ErrIsDir + + case unixfs.TMetadata: + if len(n.Links()) == 0 { + return nil, errors.New("incorrectly formatted metadata object") + } + child, err := n.Links()[0].GetNode(ctx, serv) + if err != nil { + return nil, err + } + + childpb, ok := child.(*mdag.ProtoNode) + if !ok { + return nil, mdag.ErrNotProtobuf + } + return NewDagReader(ctx, childpb, serv) + case unixfs.TSymlink: + return nil, ErrCantReadSymlinks + default: + return nil, unixfs.ErrUnrecognizedType + } + default: + return nil, ErrUnkownNodeType + } + + ctxWithCancel, cancel := context.WithCancel(ctx) + + return &dagReader{ + ctx: ctxWithCancel, + cancel: cancel, + serv: serv, + size: size, + rootNode: n, + dagWalker: ipld.NewWalker(ctxWithCancel, ipld.NewNavigableIPLDNode(n, serv)), + }, nil +} + +// dagReader provides a way to easily read the data contained in a dag. +type dagReader struct { + + // Structure to perform the DAG iteration and search, the reader + // just needs to add logic to the `Visitor` callback passed to + // `Iterate` and `Seek`. + dagWalker *ipld.Walker + + // Buffer with the data extracted from the current node being visited. + // To avoid revisiting a node to complete a (potential) partial read + // (or read after seek) the node's data is fully extracted in a single + // `readNodeDataBuffer` operation. + currentNodeData *bytes.Reader + + // Implements the `Size()` API. + size uint64 + + // Current offset for the read head within the DAG file. + offset int64 + + // Root node of the DAG, stored to re-create the `dagWalker` (effectively + // re-setting the position of the reader, used during `Seek`). + rootNode ipld.Node + + // Context passed to the `dagWalker`, the `cancel` function is used to + // cancel read operations (cancelling requested child node promises, + // see `ipld.NavigableIPLDNode.FetchChild` for details). + ctx context.Context + cancel func() + + // Passed to the `dagWalker` that will use it to request nodes. + // TODO: Revisit name. + serv ipld.NodeGetter +} + +// Size returns the total size of the data from the DAG structured file. +func (dr *dagReader) Size() uint64 { + return dr.size +} + +// Read implements the `io.Reader` interface through the `CtxReadFull` +// method using the DAG reader's internal context. +func (dr *dagReader) Read(b []byte) (int, error) { + return dr.CtxReadFull(dr.ctx, b) +} + +// CtxReadFull reads data from the DAG structured file. It always +// attempts a full read of the DAG until the `out` buffer is full. +// It uses the `Walker` structure to iterate the file DAG and read +// every node's data into the `out` buffer. +func (dr *dagReader) CtxReadFull(ctx context.Context, out []byte) (n int, err error) { + // Set the `dagWalker`'s context to the `ctx` argument, it will be used + // to fetch the child node promises (see + // `ipld.NavigableIPLDNode.FetchChild` for details). + dr.dagWalker.SetContext(ctx) + + // If there was a partially read buffer from the last visited + // node read it before visiting a new one. + if dr.currentNodeData != nil { + // TODO: Move this check inside `readNodeDataBuffer`? + n = dr.readNodeDataBuffer(out) + + if n == len(out) { + return n, nil + // Output buffer full, no need to traverse the DAG. + } + } + + // Iterate the DAG calling the passed `Visitor` function on every node + // to read its data into the `out` buffer, stop if there is an error or + // if the entire DAG is traversed (`EndOfDag`). + err = dr.dagWalker.Iterate(func(visitedNode ipld.NavigableNode) error { + node := ipld.ExtractIPLDNode(visitedNode) + + // Skip internal nodes, they shouldn't have any file data + // (see the `balanced` package for more details). + if len(node.Links()) > 0 { + return nil + } + + err = dr.saveNodeData(node) + if err != nil { + return err + } + // Save the leaf node file data in a buffer in case it is only + // partially read now and future `CtxReadFull` calls reclaim the + // rest (as each node is visited only once during `Iterate`). + // + // TODO: We could check if the entire node's data can fit in the + // remaining `out` buffer free space to skip this intermediary step. + + n += dr.readNodeDataBuffer(out[n:]) + + if n == len(out) { + // Output buffer full, no need to keep traversing the DAG, + // signal the `Walker` to pause the iteration. + dr.dagWalker.Pause() + } + + return nil + }) + + if err == ipld.EndOfDag { + return n, io.EOF + // Reached the end of the (DAG) file, no more data to read. + } else if err != nil { + return n, err + // Pass along any other errors from the `Visitor`. + } + + return n, nil +} + +// Save the UnixFS `node`'s data into the internal `currentNodeData` buffer to +// later move it to the output buffer (`Read`) or seek into it (`Seek`). +func (dr *dagReader) saveNodeData(node ipld.Node) error { + extractedNodeData, err := unixfs.ReadUnixFSNodeData(node) + if err != nil { + return err + } + + dr.currentNodeData = bytes.NewReader(extractedNodeData) + return nil +} + +// Read the `currentNodeData` buffer into `out`. This function can't have +// any errors as it's always reading from a `bytes.Reader` and asking only +// the available data in it. +func (dr *dagReader) readNodeDataBuffer(out []byte) int { + + n, _ := dr.currentNodeData.Read(out) + // Ignore the error as the EOF may not be returned in the first + // `Read` call, explicitly ask for an empty buffer below to check + // if we've reached the end. + + if dr.currentNodeData.Len() == 0 { + dr.currentNodeData = nil + // Signal that the buffer was consumed (for later `Read` calls). + // This shouldn't return an EOF error as it's just the end of a + // single node's data, not the entire DAG. + } + + dr.offset += int64(n) + // TODO: Should `offset` be incremented here or in the calling function? + // (Doing it here saves LoC but may be confusing as it's more hidden). + + return n +} + +// Similar to `readNodeDataBuffer` but it writes the contents to +// an `io.Writer` argument. +// +// TODO: Check what part of the logic between the two functions +// can be extracted away. +func (dr *dagReader) writeNodeDataBuffer(w io.Writer) (int64, error) { + + n, err := dr.currentNodeData.WriteTo(w) + if err != nil { + return n, err + } + + if dr.currentNodeData.Len() == 0 { + dr.currentNodeData = nil + // Signal that the buffer was consumed (for later `Read` calls). + // This shouldn't return an EOF error as it's just the end of a + // single node's data, not the entire DAG. + } + + dr.offset += int64(n) + return n, nil +} + +// WriteTo writes to the given writer. +// This follows the `bytes.Reader.WriteTo` implementation +// where it starts from the internal index that may have +// been modified by other `Read` calls. +// +// TODO: This implementation is very similar to `CtxReadFull`, +// the common parts should be abstracted away. +func (dr *dagReader) WriteTo(w io.Writer) (n int64, err error) { + // Use the internal reader's context to fetch the child node promises + // (see `ipld.NavigableIPLDNode.FetchChild` for details). + dr.dagWalker.SetContext(dr.ctx) + + // If there was a partially read buffer from the last visited + // node read it before visiting a new one. + if dr.currentNodeData != nil { + n, err = dr.writeNodeDataBuffer(w) + if err != nil { + return n, err + } + } + + // Iterate the DAG calling the passed `Visitor` function on every node + // to read its data into the `out` buffer, stop if there is an error or + // if the entire DAG is traversed (`EndOfDag`). + err = dr.dagWalker.Iterate(func(visitedNode ipld.NavigableNode) error { + node := ipld.ExtractIPLDNode(visitedNode) + + // Skip internal nodes, they shouldn't have any file data + // (see the `balanced` package for more details). + if len(node.Links()) > 0 { + return nil + } + + err = dr.saveNodeData(node) + if err != nil { + return err + } + // Save the leaf node file data in a buffer in case it is only + // partially read now and future `CtxReadFull` calls reclaim the + // rest (as each node is visited only once during `Iterate`). + + written, err := dr.writeNodeDataBuffer(w) + n += written + if err != nil { + return err + } + + return nil + }) + + if err == ipld.EndOfDag { + return n, nil + } + + return n, err +} + +// Close the reader (cancelling fetch node operations requested with +// the internal context, that is, `Read` calls but not `CtxReadFull` +// with user-supplied contexts). +func (dr *dagReader) Close() error { + dr.cancel() + return nil +} + +// Seek implements `io.Seeker` seeking to a given offset in the DAG file, +// it matches the standard unix `seek`. It moves the position of the internal +// `dagWalker` and may also leave a `currentNodeData` buffer loaded in case +// the seek is performed to the middle of the data in a node. +// +// TODO: Support seeking from the current position (relative seek) +// through the `dagWalker` in `io.SeekCurrent`. +func (dr *dagReader) Seek(offset int64, whence int) (int64, error) { + switch whence { + case io.SeekStart: + if offset < 0 { + return dr.offset, errors.New("invalid offset") + } + + if offset == dr.offset { + return offset, nil + // Already at the requested `offset`, nothing to do. + } + + left := offset + // Amount left to seek. + + // Seek from the beginning of the DAG. + dr.resetPosition() + + // Shortcut seeking to the beginning, we're already there. + if offset == 0 { + return 0, nil + } + + // Use the internal reader's context to fetch the child node promises + // (see `ipld.NavigableIPLDNode.FetchChild` for details). + dr.dagWalker.SetContext(dr.ctx) + // TODO: Performance: we could adjust here `preloadSize` of + // `ipld.NavigableIPLDNode` also, when seeking we only want + // to fetch one child at a time. + + // Seek the DAG by calling the provided `Visitor` function on every + // node the `dagWalker` descends to while searching which can be + // either an internal or leaf node. In the internal node case, check + // the child node sizes and set the corresponding child index to go + // down to next. In the leaf case (last visit of the search), if there + // is still an amount `left` to seek do it inside the node's data + // saved in the `currentNodeData` buffer, leaving it ready for a `Read` + // call. + err := dr.dagWalker.Seek(func(visitedNode ipld.NavigableNode) error { + node := ipld.ExtractIPLDNode(visitedNode) + + if len(node.Links()) > 0 { + // Internal node, should be a `mdag.ProtoNode` containing a + // `unixfs.FSNode` (see the `balanced` package for more details). + fsNode, err := unixfs.ExtractFSNode(node) + if err != nil { + return err + } + + // If there aren't enough size hints don't seek + // (see the `io.EOF` handling error comment below). + if fsNode.NumChildren() != len(node.Links()) { + return ErrSeekNotSupported + } + + // Internal nodes have no data, so just iterate through the + // sizes of its children (advancing the child index of the + // `dagWalker`) to find where we need to go down to next in + // the search. + for { + childSize := fsNode.BlockSize(int(dr.dagWalker.ActiveChildIndex())) + + if childSize > uint64(left) { + // This child's data contains the position requested + // in `offset`, go down this child. + return nil + } + + // Else, skip this child. + left -= int64(childSize) + err := dr.dagWalker.NextChild() + if err == ipld.ErrNextNoChild { + // No more child nodes available, nothing to do, + // the `Seek` will stop on its own. + return nil + } else if err != nil { + return err + // Pass along any other errors (that may in future + // implementations be returned by `Next`) to stop + // the search. + } + } + + } else { + // Leaf node, seek inside its data. + err := dr.saveNodeData(node) + if err != nil { + return err + } + + _, err = dr.currentNodeData.Seek(left, io.SeekStart) + if err != nil { + return err + } + // The corner case of a DAG consisting only of a single (leaf) + // node should make no difference here. In that case, where the + // node doesn't have a parent UnixFS node with size hints, this + // implementation would allow this `Seek` to be called with an + // argument larger than the buffer size which normally wouldn't + // happen (because we would skip the node based on the size + // hint) but that would just mean that a future `CtxReadFull` + // call would read no data from the `currentNodeData` buffer. + // TODO: Re-check this reasoning. + + return nil + // In the leaf node case the search will stop here. + } + }) + + if err != nil { + return 0, err + } + + dr.offset = offset + return dr.offset, nil + + case io.SeekCurrent: + if offset == 0 { + return dr.offset, nil + } + + return dr.Seek(dr.offset+offset, io.SeekStart) + // TODO: Performance. This can be improved supporting relative + // searches in the `Walker` (see `Walker.Seek`). + + case io.SeekEnd: + return dr.Seek(int64(dr.Size())+offset, io.SeekStart) + + default: + return 0, errors.New("invalid whence") + } +} + +// Reset the reader position by resetting the `dagWalker` and discarding +// any partially used node's data in the `currentNodeData` buffer, used +// in the `SeekStart` case. +func (dr *dagReader) resetPosition() { + dr.currentNodeData = nil + dr.offset = 0 + + dr.dagWalker = ipld.NewWalker(dr.ctx, ipld.NewNavigableIPLDNode(dr.rootNode, dr.serv)) + // TODO: This could be avoided (along with storing the `dr.rootNode` and + // `dr.serv` just for this call) if `Reset` is supported in the `Walker`. +} diff --git a/unixfs/io/dagreader_test.go b/unixfs/io/dagreader_test.go new file mode 100644 index 0000000000..d2c586314d --- /dev/null +++ b/unixfs/io/dagreader_test.go @@ -0,0 +1,326 @@ +package io + +import ( + "bytes" + "io" + "strings" + "testing" + + "github.com/ipfs/go-libipfs/unixfs" + mdag "github.com/ipfs/go-merkledag" + + context "context" + + testu "github.com/ipfs/go-libipfs/unixfs/test" +) + +func TestBasicRead(t *testing.T) { + dserv := testu.GetDAGServ() + inbuf, node := testu.GetRandomNode(t, dserv, 1024, testu.UseProtoBufLeaves) + ctx, closer := context.WithCancel(context.Background()) + defer closer() + + reader, err := NewDagReader(ctx, node, dserv) + if err != nil { + t.Fatal(err) + } + + outbuf, err := io.ReadAll(reader) + if err != nil { + t.Fatal(err) + } + + err = testu.ArrComp(inbuf, outbuf) + if err != nil { + t.Fatal(err) + } +} + +func TestSeekAndRead(t *testing.T) { + dserv := testu.GetDAGServ() + inbuf := make([]byte, 256) + for i := 0; i <= 255; i++ { + inbuf[i] = byte(i) + } + + node := testu.GetNode(t, dserv, inbuf, testu.UseProtoBufLeaves) + ctx, closer := context.WithCancel(context.Background()) + defer closer() + + reader, err := NewDagReader(ctx, node, dserv) + if err != nil { + t.Fatal(err) + } + + for i := 255; i >= 0; i-- { + reader.Seek(int64(i), io.SeekStart) + + if getOffset(reader) != int64(i) { + t.Fatal("expected offset to be increased by one after read") + } + + out := readByte(t, reader) + + if int(out) != i { + t.Fatalf("read %d at index %d, expected %d", out, i, i) + } + + if getOffset(reader) != int64(i+1) { + t.Fatal("expected offset to be increased by one after read") + } + } +} + +func TestSeekWithoutBlocksizes(t *testing.T) { + dserv := testu.GetDAGServ() + ctx, closer := context.WithCancel(context.Background()) + defer closer() + + inbuf := make([]byte, 1024) + + for i := 0; i < 256; i++ { + inbuf[i*4] = byte(i) + } + + inbuf[1023] = 1 // force the reader to be 1024 bytes + node := testu.GetNode(t, dserv, inbuf, testu.UseProtoBufLeaves) + + // remove the blocksizes + pbnode := node.Copy().(*mdag.ProtoNode) + fsnode, err := unixfs.FSNodeFromBytes(pbnode.Data()) + if err != nil { + t.Fatal(err) + } + fsnode.RemoveAllBlockSizes() + newData, err := fsnode.GetBytes() + if err != nil { + t.Fatal(err) + } + pbnode.SetData(newData) + err = dserv.Add(ctx, pbnode) + if err != nil { + t.Fatal(err) + } + node = pbnode + + reader, err := NewDagReader(ctx, node, dserv) + if err != nil { + t.Fatal(err) + } + + _, err = reader.Seek(-4, io.SeekEnd) + if err == nil { + t.Fatal("seeking shouldn't work without blocksizes") + } + + _, err = reader.Seek(4, io.SeekStart) + if err == nil { + t.Fatal("seeking shouldn't work without blocksizes") + } + + _, err = reader.Seek(4, io.SeekCurrent) + if err == nil { + t.Fatal("seeking shouldn't work without blocksizes") + } + + // Seeking to the current position or the end should still work. + + _, err = reader.Seek(0, io.SeekCurrent) + if err != nil { + t.Fatal(err) + } + + _, err = reader.Seek(0, io.SeekStart) + if err != nil { + t.Fatal(err) + } +} + +func TestRelativeSeek(t *testing.T) { + dserv := testu.GetDAGServ() + ctx, closer := context.WithCancel(context.Background()) + defer closer() + + inbuf := make([]byte, 1024) + + for i := 0; i < 256; i++ { + inbuf[i*4] = byte(i) + } + + inbuf[1023] = 1 // force the reader to be 1024 bytes + node := testu.GetNode(t, dserv, inbuf, testu.UseProtoBufLeaves) + + reader, err := NewDagReader(ctx, node, dserv) + if err != nil { + t.Fatal(err) + } + + for i := 0; i < 256; i++ { + if getOffset(reader) != int64(i*4) { + t.Fatalf("offset should be %d, was %d", i*4, getOffset(reader)) + } + out := readByte(t, reader) + if int(out) != i { + t.Fatalf("expected to read: %d at %d, read %d", i, getOffset(reader)-1, out) + } + if i != 255 { + _, err := reader.Seek(3, io.SeekCurrent) + if err != nil { + t.Fatal(err) + } + } + } + + _, err = reader.Seek(-4, io.SeekEnd) + if err != nil { + t.Fatal(err) + } + + for i := 0; i < 256; i++ { + if getOffset(reader) != int64(1020-i*4) { + t.Fatalf("offset should be %d, was %d", 1020-i*4, getOffset(reader)) + } + out := readByte(t, reader) + if int(out) != 255-i { + t.Fatalf("expected to read: %d at %d, read %d", 255-i, getOffset(reader)-1, out) + } + reader.Seek(-5, io.SeekCurrent) // seek 4 bytes but we read one byte every time so 5 bytes + } + +} + +func TestTypeFailures(t *testing.T) { + dserv := testu.GetDAGServ() + ctx, closer := context.WithCancel(context.Background()) + defer closer() + + node := unixfs.EmptyDirNode() + if _, err := NewDagReader(ctx, node, dserv); err != ErrIsDir { + t.Fatalf("excepted to get %v, got %v", ErrIsDir, err) + } + + data, err := unixfs.SymlinkData("/somelink") + if err != nil { + t.Fatal(err) + } + node = mdag.NodeWithData(data) + + if _, err := NewDagReader(ctx, node, dserv); err != ErrCantReadSymlinks { + t.Fatalf("excepted to get %v, got %v", ErrCantReadSymlinks, err) + } +} + +func TestBadPBData(t *testing.T) { + dserv := testu.GetDAGServ() + ctx, closer := context.WithCancel(context.Background()) + defer closer() + + node := mdag.NodeWithData([]byte{42}) + _, err := NewDagReader(ctx, node, dserv) + if err == nil { + t.Fatal("excepted error, got nil") + } +} + +func TestMetadataNode(t *testing.T) { + ctx, closer := context.WithCancel(context.Background()) + defer closer() + + dserv := testu.GetDAGServ() + rdata, rnode := testu.GetRandomNode(t, dserv, 512, testu.UseProtoBufLeaves) + err := dserv.Add(ctx, rnode) + if err != nil { + t.Fatal(err) + } + + data, err := unixfs.BytesForMetadata(&unixfs.Metadata{ + MimeType: "text", + Size: 125, + }) + if err != nil { + t.Fatal(err) + } + node := mdag.NodeWithData(data) + + _, err = NewDagReader(ctx, node, dserv) + if err == nil { + t.Fatal("expected an error") + } + if !strings.Contains(err.Error(), "incorrectly formatted") { + t.Fatal("expected different error") + } + + node.AddNodeLink("", rnode) + + reader, err := NewDagReader(ctx, node, dserv) + if err != nil { + t.Fatal(err) + } + readdata, err := io.ReadAll(reader) + if err != nil { + t.Fatal(err) + } + if err := testu.ArrComp(rdata, readdata); err != nil { + t.Fatal(err) + } +} + +func TestWriteTo(t *testing.T) { + dserv := testu.GetDAGServ() + inbuf, node := testu.GetRandomNode(t, dserv, 1024, testu.UseProtoBufLeaves) + ctx, closer := context.WithCancel(context.Background()) + defer closer() + + reader, err := NewDagReader(ctx, node, dserv) + if err != nil { + t.Fatal(err) + } + + outbuf := new(bytes.Buffer) + reader.WriteTo(outbuf) + + err = testu.ArrComp(inbuf, outbuf.Bytes()) + if err != nil { + t.Fatal(err) + } + +} + +func TestReaderSzie(t *testing.T) { + dserv := testu.GetDAGServ() + size := int64(1024) + _, node := testu.GetRandomNode(t, dserv, size, testu.UseProtoBufLeaves) + ctx, closer := context.WithCancel(context.Background()) + defer closer() + + reader, err := NewDagReader(ctx, node, dserv) + if err != nil { + t.Fatal(err) + } + + if reader.Size() != uint64(size) { + t.Fatal("wrong reader size") + } +} + +func readByte(t testing.TB, reader DagReader) byte { + out := make([]byte, 1) + c, err := reader.Read(out) + + if c != 1 { + t.Fatal("reader should have read just one byte") + } + if err != nil { + t.Fatal(err) + } + + return out[0] +} + +func getOffset(reader DagReader) int64 { + offset, err := reader.Seek(0, io.SeekCurrent) + if err != nil { + panic("failed to retrieve offset: " + err.Error()) + } + return offset +} diff --git a/unixfs/io/directory.go b/unixfs/io/directory.go new file mode 100644 index 0000000000..0710f7226e --- /dev/null +++ b/unixfs/io/directory.go @@ -0,0 +1,610 @@ +package io + +import ( + "context" + "fmt" + "os" + + "github.com/ipfs/go-libipfs/unixfs/hamt" + "github.com/ipfs/go-libipfs/unixfs/private/linksize" + + "github.com/alecthomas/units" + "github.com/ipfs/go-cid" + ipld "github.com/ipfs/go-ipld-format" + format "github.com/ipfs/go-libipfs/unixfs" + logging "github.com/ipfs/go-log" + mdag "github.com/ipfs/go-merkledag" +) + +var log = logging.Logger("unixfs") + +// HAMTShardingSize is a global option that allows switching to a HAMTDirectory +// when the BasicDirectory grows above the size (in bytes) signalled by this +// flag. The default size of 0 disables the option. +// The size is not the *exact* block size of the encoded BasicDirectory but just +// the estimated size based byte length of links name and CID (BasicDirectory's +// ProtoNode doesn't use the Data field so this estimate is pretty accurate). +var HAMTShardingSize = int(256 * units.KiB) + +// DefaultShardWidth is the default value used for hamt sharding width. +// Needs to be a power of two (shard entry size) and multiple of 8 (bitfield size). +var DefaultShardWidth = 256 + +// Directory defines a UnixFS directory. It is used for creating, reading and +// editing directories. It allows to work with different directory schemes, +// like the basic or the HAMT implementation. +// +// It just allows to perform explicit edits on a single directory, working with +// directory trees is out of its scope, they are managed by the MFS layer +// (which is the main consumer of this interface). +type Directory interface { + + // SetCidBuilder sets the CID Builder of the root node. + SetCidBuilder(cid.Builder) + + // AddChild adds a (name, key) pair to the root node. + AddChild(context.Context, string, ipld.Node) error + + // ForEachLink applies the given function to Links in the directory. + ForEachLink(context.Context, func(*ipld.Link) error) error + + // EnumLinksAsync returns a channel which will receive Links in the directory + // as they are enumerated, where order is not gauranteed + EnumLinksAsync(context.Context) <-chan format.LinkResult + + // Links returns the all the links in the directory node. + Links(context.Context) ([]*ipld.Link, error) + + // Find returns the root node of the file named 'name' within this directory. + // In the case of HAMT-directories, it will traverse the tree. + // + // Returns os.ErrNotExist if the child does not exist. + Find(context.Context, string) (ipld.Node, error) + + // RemoveChild removes the child with the given name. + // + // Returns os.ErrNotExist if the child doesn't exist. + RemoveChild(context.Context, string) error + + // GetNode returns the root of this directory. + GetNode() (ipld.Node, error) + + // GetCidBuilder returns the CID Builder used. + GetCidBuilder() cid.Builder +} + +// TODO: Evaluate removing `dserv` from this layer and providing it in MFS. +// (The functions should in that case add a `DAGService` argument.) + +// Link size estimation function. For production it's usually the one here +// but during test we may mock it to get fixed sizes. +func productionLinkSize(linkName string, linkCid cid.Cid) int { + return len(linkName) + linkCid.ByteLen() +} + +func init() { + linksize.LinkSizeFunction = productionLinkSize +} + +// BasicDirectory is the basic implementation of `Directory`. All the entries +// are stored in a single node. +type BasicDirectory struct { + node *mdag.ProtoNode + dserv ipld.DAGService + + // Internal variable used to cache the estimated size of the basic directory: + // for each link, aggregate link name + link CID. DO NOT CHANGE THIS + // as it will affect the HAMT transition behavior in HAMTShardingSize. + // (We maintain this value up to date even if the HAMTShardingSize is off + // since potentially the option could be activated on the fly.) + estimatedSize int +} + +// HAMTDirectory is the HAMT implementation of `Directory`. +// (See package `hamt` for more information.) +type HAMTDirectory struct { + shard *hamt.Shard + dserv ipld.DAGService + + // Track the changes in size by the AddChild and RemoveChild calls + // for the HAMTShardingSize option. + sizeChange int +} + +func newEmptyBasicDirectory(dserv ipld.DAGService) *BasicDirectory { + return newBasicDirectoryFromNode(dserv, format.EmptyDirNode()) +} + +func newBasicDirectoryFromNode(dserv ipld.DAGService, node *mdag.ProtoNode) *BasicDirectory { + basicDir := new(BasicDirectory) + basicDir.node = node + basicDir.dserv = dserv + + // Scan node links (if any) to restore estimated size. + basicDir.computeEstimatedSize() + + return basicDir +} + +// NewDirectory returns a Directory implemented by DynamicDirectory +// containing a BasicDirectory that can be converted to a HAMTDirectory. +func NewDirectory(dserv ipld.DAGService) Directory { + return &DynamicDirectory{newEmptyBasicDirectory(dserv)} +} + +// ErrNotADir implies that the given node was not a unixfs directory +var ErrNotADir = fmt.Errorf("merkledag node was not a directory or shard") + +// NewDirectoryFromNode loads a unixfs directory from the given IPLD node and +// DAGService. +func NewDirectoryFromNode(dserv ipld.DAGService, node ipld.Node) (Directory, error) { + protoBufNode, ok := node.(*mdag.ProtoNode) + if !ok { + return nil, ErrNotADir + } + + fsNode, err := format.FSNodeFromBytes(protoBufNode.Data()) + if err != nil { + return nil, err + } + + switch fsNode.Type() { + case format.TDirectory: + return &DynamicDirectory{newBasicDirectoryFromNode(dserv, protoBufNode.Copy().(*mdag.ProtoNode))}, nil + case format.THAMTShard: + shard, err := hamt.NewHamtFromDag(dserv, node) + if err != nil { + return nil, err + } + return &DynamicDirectory{&HAMTDirectory{shard, dserv, 0}}, nil + } + + return nil, ErrNotADir +} + +func (d *BasicDirectory) computeEstimatedSize() { + d.estimatedSize = 0 + // err is just breaking the iteration and we always return nil + _ = d.ForEachLink(context.TODO(), func(l *ipld.Link) error { + d.addToEstimatedSize(l.Name, l.Cid) + return nil + }) + // ForEachLink will never fail traversing the BasicDirectory + // and neither the inner callback `addToEstimatedSize`. +} + +func (d *BasicDirectory) addToEstimatedSize(name string, linkCid cid.Cid) { + d.estimatedSize += linksize.LinkSizeFunction(name, linkCid) +} + +func (d *BasicDirectory) removeFromEstimatedSize(name string, linkCid cid.Cid) { + d.estimatedSize -= linksize.LinkSizeFunction(name, linkCid) + if d.estimatedSize < 0 { + // Something has gone very wrong. Log an error and recompute the + // size from scratch. + log.Error("BasicDirectory's estimatedSize went below 0") + d.computeEstimatedSize() + } +} + +// SetCidBuilder implements the `Directory` interface. +func (d *BasicDirectory) SetCidBuilder(builder cid.Builder) { + d.node.SetCidBuilder(builder) +} + +// AddChild implements the `Directory` interface. It adds (or replaces) +// a link to the given `node` under `name`. +func (d *BasicDirectory) AddChild(ctx context.Context, name string, node ipld.Node) error { + link, err := ipld.MakeLink(node) + if err != nil { + return err + } + + return d.addLinkChild(ctx, name, link) +} + +func (d *BasicDirectory) needsToSwitchToHAMTDir(name string, nodeToAdd ipld.Node) (bool, error) { + if HAMTShardingSize == 0 { // Option disabled. + return false, nil + } + + operationSizeChange := 0 + // Find if there is an old entry under that name that will be overwritten. + entryToRemove, err := d.node.GetNodeLink(name) + if err != mdag.ErrLinkNotFound { + if err != nil { + return false, err + } + operationSizeChange -= linksize.LinkSizeFunction(name, entryToRemove.Cid) + } + if nodeToAdd != nil { + operationSizeChange += linksize.LinkSizeFunction(name, nodeToAdd.Cid()) + } + + return d.estimatedSize+operationSizeChange >= HAMTShardingSize, nil +} + +// addLinkChild adds the link as an entry to this directory under the given +// name. Plumbing function for the AddChild API. +func (d *BasicDirectory) addLinkChild(ctx context.Context, name string, link *ipld.Link) error { + // Remove old link and account for size change (if it existed; ignore + // `ErrNotExist` otherwise). + err := d.RemoveChild(ctx, name) + if err != nil && err != os.ErrNotExist { + return err + } + + err = d.node.AddRawLink(name, link) + if err != nil { + return err + } + d.addToEstimatedSize(name, link.Cid) + return nil +} + +// EnumLinksAsync returns a channel which will receive Links in the directory +// as they are enumerated, where order is not gauranteed +func (d *BasicDirectory) EnumLinksAsync(ctx context.Context) <-chan format.LinkResult { + linkResults := make(chan format.LinkResult) + go func() { + defer close(linkResults) + for _, l := range d.node.Links() { + select { + case linkResults <- format.LinkResult{ + Link: l, + Err: nil, + }: + case <-ctx.Done(): + return + } + } + }() + return linkResults +} + +// ForEachLink implements the `Directory` interface. +func (d *BasicDirectory) ForEachLink(_ context.Context, f func(*ipld.Link) error) error { + for _, l := range d.node.Links() { + if err := f(l); err != nil { + return err + } + } + return nil +} + +// Links implements the `Directory` interface. +func (d *BasicDirectory) Links(ctx context.Context) ([]*ipld.Link, error) { + return d.node.Links(), nil +} + +// Find implements the `Directory` interface. +func (d *BasicDirectory) Find(ctx context.Context, name string) (ipld.Node, error) { + lnk, err := d.node.GetNodeLink(name) + if err == mdag.ErrLinkNotFound { + err = os.ErrNotExist + } + if err != nil { + return nil, err + } + + return d.dserv.Get(ctx, lnk.Cid) +} + +// RemoveChild implements the `Directory` interface. +func (d *BasicDirectory) RemoveChild(ctx context.Context, name string) error { + // We need to *retrieve* the link before removing it to update the estimated + // size. This means we may iterate the links slice twice: if traversing this + // becomes a problem, a factor of 2 isn't going to make much of a difference. + // We'd likely need to cache a link resolution map in that case. + link, err := d.node.GetNodeLink(name) + if err == mdag.ErrLinkNotFound { + return os.ErrNotExist + } + if err != nil { + return err // at the moment there is no other error besides ErrLinkNotFound + } + + // The name actually existed so we should update the estimated size. + d.removeFromEstimatedSize(link.Name, link.Cid) + + return d.node.RemoveNodeLink(name) + // GetNodeLink didn't return ErrLinkNotFound so this won't fail with that + // and we don't need to convert the error again. +} + +// GetNode implements the `Directory` interface. +func (d *BasicDirectory) GetNode() (ipld.Node, error) { + return d.node, nil +} + +// GetCidBuilder implements the `Directory` interface. +func (d *BasicDirectory) GetCidBuilder() cid.Builder { + return d.node.CidBuilder() +} + +// switchToSharding returns a HAMT implementation of this directory. +func (d *BasicDirectory) switchToSharding(ctx context.Context) (*HAMTDirectory, error) { + hamtDir := new(HAMTDirectory) + hamtDir.dserv = d.dserv + + shard, err := hamt.NewShard(d.dserv, DefaultShardWidth) + if err != nil { + return nil, err + } + shard.SetCidBuilder(d.node.CidBuilder()) + hamtDir.shard = shard + + for _, lnk := range d.node.Links() { + err = hamtDir.shard.SetLink(ctx, lnk.Name, lnk) + if err != nil { + return nil, err + } + } + + return hamtDir, nil +} + +// SetCidBuilder implements the `Directory` interface. +func (d *HAMTDirectory) SetCidBuilder(builder cid.Builder) { + d.shard.SetCidBuilder(builder) +} + +// AddChild implements the `Directory` interface. +func (d *HAMTDirectory) AddChild(ctx context.Context, name string, nd ipld.Node) error { + oldChild, err := d.shard.Swap(ctx, name, nd) + if err != nil { + return err + } + + if oldChild != nil { + d.removeFromSizeChange(oldChild.Name, oldChild.Cid) + } + d.addToSizeChange(name, nd.Cid()) + return nil +} + +// ForEachLink implements the `Directory` interface. +func (d *HAMTDirectory) ForEachLink(ctx context.Context, f func(*ipld.Link) error) error { + return d.shard.ForEachLink(ctx, f) +} + +// EnumLinksAsync returns a channel which will receive Links in the directory +// as they are enumerated, where order is not gauranteed +func (d *HAMTDirectory) EnumLinksAsync(ctx context.Context) <-chan format.LinkResult { + return d.shard.EnumLinksAsync(ctx) +} + +// Links implements the `Directory` interface. +func (d *HAMTDirectory) Links(ctx context.Context) ([]*ipld.Link, error) { + return d.shard.EnumLinks(ctx) +} + +// Find implements the `Directory` interface. It will traverse the tree. +func (d *HAMTDirectory) Find(ctx context.Context, name string) (ipld.Node, error) { + lnk, err := d.shard.Find(ctx, name) + if err != nil { + return nil, err + } + + return lnk.GetNode(ctx, d.dserv) +} + +// RemoveChild implements the `Directory` interface. +func (d *HAMTDirectory) RemoveChild(ctx context.Context, name string) error { + oldChild, err := d.shard.Take(ctx, name) + if err != nil { + return err + } + + if oldChild != nil { + d.removeFromSizeChange(oldChild.Name, oldChild.Cid) + } + + return nil +} + +// GetNode implements the `Directory` interface. +func (d *HAMTDirectory) GetNode() (ipld.Node, error) { + return d.shard.Node() +} + +// GetCidBuilder implements the `Directory` interface. +func (d *HAMTDirectory) GetCidBuilder() cid.Builder { + return d.shard.CidBuilder() +} + +// switchToBasic returns a BasicDirectory implementation of this directory. +func (d *HAMTDirectory) switchToBasic(ctx context.Context) (*BasicDirectory, error) { + basicDir := newEmptyBasicDirectory(d.dserv) + basicDir.SetCidBuilder(d.GetCidBuilder()) + + err := d.ForEachLink(ctx, func(lnk *ipld.Link) error { + err := basicDir.addLinkChild(ctx, lnk.Name, lnk) + if err != nil { + return err + } + + return nil + // This function enumerates all the links in the Directory requiring all + // shards to be accessible but it is only called *after* sizeBelowThreshold + // returns true, which means we have already enumerated and fetched *all* + // shards in the first place (that's the only way we can be really sure + // we are actually below the threshold). + }) + if err != nil { + return nil, err + } + + return basicDir, nil +} + +func (d *HAMTDirectory) addToSizeChange(name string, linkCid cid.Cid) { + d.sizeChange += linksize.LinkSizeFunction(name, linkCid) +} + +func (d *HAMTDirectory) removeFromSizeChange(name string, linkCid cid.Cid) { + d.sizeChange -= linksize.LinkSizeFunction(name, linkCid) +} + +// Evaluate a switch from HAMTDirectory to BasicDirectory in case the size will +// go above the threshold when we are adding or removing an entry. +// In both the add/remove operations any old name will be removed, and for the +// add operation in particular a new entry will be added under that name (otherwise +// nodeToAdd is nil). We compute both (potential) future subtraction and +// addition to the size change. +func (d *HAMTDirectory) needsToSwitchToBasicDir(ctx context.Context, name string, nodeToAdd ipld.Node) (switchToBasic bool, err error) { + if HAMTShardingSize == 0 { // Option disabled. + return false, nil + } + + operationSizeChange := 0 + + // Find if there is an old entry under that name that will be overwritten + // (AddEntry) or flat out removed (RemoveEntry). + entryToRemove, err := d.shard.Find(ctx, name) + if err != os.ErrNotExist { + if err != nil { + return false, err + } + operationSizeChange -= linksize.LinkSizeFunction(name, entryToRemove.Cid) + } + + // For the AddEntry case compute the size addition of the new entry. + if nodeToAdd != nil { + operationSizeChange += linksize.LinkSizeFunction(name, nodeToAdd.Cid()) + } + + if d.sizeChange+operationSizeChange >= 0 { + // We won't have reduced the HAMT net size. + return false, nil + } + + // We have reduced the directory size, check if went below the + // HAMTShardingSize threshold to trigger a switch. + return d.sizeBelowThreshold(ctx, operationSizeChange) +} + +// Evaluate directory size and a future sizeChange and check if it will be below +// HAMTShardingSize threshold (to trigger a transition to a BasicDirectory). +// Instead of enumerating the entire tree we eagerly call EnumLinksAsync +// until we either reach a value above the threshold (in that case no need +// to keep counting) or an error occurs (like the context being canceled +// if we take too much time fetching the necessary shards). +func (d *HAMTDirectory) sizeBelowThreshold(ctx context.Context, sizeChange int) (below bool, err error) { + if HAMTShardingSize == 0 { + panic("asked to compute HAMT size with HAMTShardingSize option off (0)") + } + + // We don't necessarily compute the full size of *all* shards as we might + // end early if we already know we're above the threshold or run out of time. + partialSize := 0 + + // We stop the enumeration once we have enough information and exit this function. + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + for linkResult := range d.EnumLinksAsync(ctx) { + if linkResult.Err != nil { + return false, linkResult.Err + } + + partialSize += linksize.LinkSizeFunction(linkResult.Link.Name, linkResult.Link.Cid) + if partialSize+sizeChange >= HAMTShardingSize { + // We have already fetched enough shards to assert we are + // above the threshold, so no need to keep fetching. + return false, nil + } + } + + // We enumerated *all* links in all shards and didn't reach the threshold. + return true, nil +} + +// DynamicDirectory wraps a Directory interface and provides extra logic +// to switch from BasicDirectory to HAMTDirectory and backwards based on +// size. +type DynamicDirectory struct { + Directory +} + +var _ Directory = (*DynamicDirectory)(nil) + +// AddChild implements the `Directory` interface. We check when adding new entries +// if we should switch to HAMTDirectory according to global option(s). +func (d *DynamicDirectory) AddChild(ctx context.Context, name string, nd ipld.Node) error { + hamtDir, ok := d.Directory.(*HAMTDirectory) + if ok { + // We evaluate a switch in the HAMTDirectory case even for an AddChild + // as it may overwrite an existing entry and end up actually reducing + // the directory size. + switchToBasic, err := hamtDir.needsToSwitchToBasicDir(ctx, name, nd) + if err != nil { + return err + } + + if switchToBasic { + basicDir, err := hamtDir.switchToBasic(ctx) + if err != nil { + return err + } + err = basicDir.AddChild(ctx, name, nd) + if err != nil { + return err + } + d.Directory = basicDir + return nil + } + + return d.Directory.AddChild(ctx, name, nd) + } + + // BasicDirectory + basicDir := d.Directory.(*BasicDirectory) + switchToHAMT, err := basicDir.needsToSwitchToHAMTDir(name, nd) + if err != nil { + return err + } + if !switchToHAMT { + return basicDir.AddChild(ctx, name, nd) + } + hamtDir, err = basicDir.switchToSharding(ctx) + if err != nil { + return err + } + err = hamtDir.AddChild(ctx, name, nd) + if err != nil { + return err + } + d.Directory = hamtDir + return nil +} + +// RemoveChild implements the `Directory` interface. Used in the case where we wrap +// a HAMTDirectory that might need to be downgraded to a BasicDirectory. The +// upgrade path is in AddChild. +func (d *DynamicDirectory) RemoveChild(ctx context.Context, name string) error { + hamtDir, ok := d.Directory.(*HAMTDirectory) + if !ok { + return d.Directory.RemoveChild(ctx, name) + } + + switchToBasic, err := hamtDir.needsToSwitchToBasicDir(ctx, name, nil) + if err != nil { + return err + } + + if !switchToBasic { + return hamtDir.RemoveChild(ctx, name) + } + + basicDir, err := hamtDir.switchToBasic(ctx) + if err != nil { + return err + } + err = basicDir.RemoveChild(ctx, name) + if err != nil { + return err + } + d.Directory = basicDir + return nil +} diff --git a/unixfs/io/directory_test.go b/unixfs/io/directory_test.go new file mode 100644 index 0000000000..4c16544eb6 --- /dev/null +++ b/unixfs/io/directory_test.go @@ -0,0 +1,638 @@ +package io + +import ( + "context" + "fmt" + "math" + "sort" + "strconv" + "strings" + "sync" + "testing" + "time" + + cid "github.com/ipfs/go-cid" + ds "github.com/ipfs/go-datastore" + dssync "github.com/ipfs/go-datastore/sync" + ipld "github.com/ipfs/go-ipld-format" + blocks "github.com/ipfs/go-libipfs/blocks" + bsrv "github.com/ipfs/go-libipfs/blockservice" + blockstore "github.com/ipfs/go-libipfs/blockstore" + offline "github.com/ipfs/go-libipfs/exchange/offline" + mdag "github.com/ipfs/go-merkledag" + mdtest "github.com/ipfs/go-merkledag/test" + + ft "github.com/ipfs/go-libipfs/unixfs" + "github.com/ipfs/go-libipfs/unixfs/hamt" + "github.com/ipfs/go-libipfs/unixfs/internal" + "github.com/ipfs/go-libipfs/unixfs/private/linksize" + + "github.com/stretchr/testify/assert" +) + +func TestEmptyNode(t *testing.T) { + n := ft.EmptyDirNode() + if len(n.Links()) != 0 { + t.Fatal("empty node should have 0 links") + } +} + +func TestDirectoryGrowth(t *testing.T) { + ds := mdtest.Mock() + dir := NewDirectory(ds) + ctx := context.Background() + + d := ft.EmptyDirNode() + ds.Add(ctx, d) + + nelems := 10000 + + for i := 0; i < nelems; i++ { + err := dir.AddChild(ctx, fmt.Sprintf("dir%d", i), d) + if err != nil { + t.Fatal(err) + } + } + + _, err := dir.GetNode() + if err != nil { + t.Fatal(err) + } + + links, err := dir.Links(ctx) + if err != nil { + t.Fatal(err) + } + + if len(links) != nelems { + t.Fatal("didnt get right number of elements") + } + + dirc := d.Cid() + + names := make(map[string]bool) + for _, l := range links { + names[l.Name] = true + if !l.Cid.Equals(dirc) { + t.Fatal("link wasnt correct") + } + } + + for i := 0; i < nelems; i++ { + dn := fmt.Sprintf("dir%d", i) + if !names[dn] { + t.Fatal("didnt find directory: ", dn) + } + + _, err := dir.Find(context.Background(), dn) + if err != nil { + t.Fatal(err) + } + } +} + +func TestDuplicateAddDir(t *testing.T) { + ds := mdtest.Mock() + dir := NewDirectory(ds) + ctx := context.Background() + nd := ft.EmptyDirNode() + + err := dir.AddChild(ctx, "test", nd) + if err != nil { + t.Fatal(err) + } + + err = dir.AddChild(ctx, "test", nd) + if err != nil { + t.Fatal(err) + } + + lnks, err := dir.Links(ctx) + if err != nil { + t.Fatal(err) + } + + if len(lnks) != 1 { + t.Fatal("expected only one link") + } +} + +func TestBasicDirectory_estimatedSize(t *testing.T) { + ds := mdtest.Mock() + basicDir := newEmptyBasicDirectory(ds) + + testDirectorySizeEstimation(t, basicDir, ds, func(dir Directory) int { + return dir.(*BasicDirectory).estimatedSize + }) +} + +func TestHAMTDirectory_sizeChange(t *testing.T) { + ds := mdtest.Mock() + hamtDir, err := newEmptyHAMTDirectory(ds, DefaultShardWidth) + assert.NoError(t, err) + + testDirectorySizeEstimation(t, hamtDir, ds, func(dir Directory) int { + // Since we created a HAMTDirectory from scratch with size 0 its + // internal sizeChange delta will in fact track the directory size + // throughout this run. + return dir.(*HAMTDirectory).sizeChange + }) +} + +func fullSizeEnumeration(dir Directory) int { + size := 0 + dir.ForEachLink(context.Background(), func(l *ipld.Link) error { + size += linksize.LinkSizeFunction(l.Name, l.Cid) + return nil + }) + return size +} + +func testDirectorySizeEstimation(t *testing.T, dir Directory, ds ipld.DAGService, size func(Directory) int) { + linksize.LinkSizeFunction = mockLinkSizeFunc(1) + defer func() { linksize.LinkSizeFunction = productionLinkSize }() + + ctx := context.Background() + child := ft.EmptyFileNode() + assert.NoError(t, ds.Add(ctx, child)) + + // Several overwrites should not corrupt the size estimation. + assert.NoError(t, dir.AddChild(ctx, "child", child)) + assert.NoError(t, dir.AddChild(ctx, "child", child)) + assert.NoError(t, dir.AddChild(ctx, "child", child)) + assert.NoError(t, dir.RemoveChild(ctx, "child")) + assert.NoError(t, dir.AddChild(ctx, "child", child)) + assert.NoError(t, dir.RemoveChild(ctx, "child")) + assert.Equal(t, 0, size(dir), "estimated size is not zero after removing all entries") + + dirEntries := 100 + for i := 0; i < dirEntries; i++ { + assert.NoError(t, dir.AddChild(ctx, fmt.Sprintf("child-%03d", i), child)) + } + assert.Equal(t, dirEntries, size(dir), "estimated size inaccurate after adding many entries") + + assert.NoError(t, dir.RemoveChild(ctx, "child-045")) // just random values + assert.NoError(t, dir.RemoveChild(ctx, "child-063")) + assert.NoError(t, dir.RemoveChild(ctx, "child-011")) + assert.NoError(t, dir.RemoveChild(ctx, "child-000")) + assert.NoError(t, dir.RemoveChild(ctx, "child-099")) + dirEntries -= 5 + assert.Equal(t, dirEntries, size(dir), "estimated size inaccurate after removing some entries") + + // All of the following remove operations will fail (won't impact dirEntries): + assert.Error(t, dir.RemoveChild(ctx, "nonexistent-name")) + assert.Error(t, dir.RemoveChild(ctx, "child-045")) // already removed + assert.Error(t, dir.RemoveChild(ctx, "child-100")) + assert.Equal(t, dirEntries, size(dir), "estimated size inaccurate after failed remove attempts") + + // Restore a directory from original's node and check estimated size consistency. + dirNode, err := dir.GetNode() + assert.NoError(t, err) + restoredDir, err := NewDirectoryFromNode(ds, dirNode.(*mdag.ProtoNode)) + assert.NoError(t, err) + assert.Equal(t, size(dir), fullSizeEnumeration(restoredDir), "restored directory's size doesn't match original's") + // We don't use the estimation size function for the restored directory + // because in the HAMT case this function depends on the sizeChange variable + // that will be cleared when loading the directory from the node. + // This also covers the case of comparing the size estimation `size()` with + // the full enumeration function `fullSizeEnumeration()` to make sure it's + // correct. +} + +// Any entry link size will have the fixedSize passed. +func mockLinkSizeFunc(fixedSize int) func(linkName string, linkCid cid.Cid) int { + return func(_ string, _ cid.Cid) int { + return fixedSize + } +} + +func checkBasicDirectory(t *testing.T, dir Directory, errorMessage string) { + if _, ok := dir.(*DynamicDirectory).Directory.(*BasicDirectory); !ok { + t.Fatal(errorMessage) + } +} + +func checkHAMTDirectory(t *testing.T, dir Directory, errorMessage string) { + if _, ok := dir.(*DynamicDirectory).Directory.(*HAMTDirectory); !ok { + t.Fatal(errorMessage) + } +} + +func TestProductionLinkSize(t *testing.T) { + link, err := ipld.MakeLink(ft.EmptyDirNode()) + assert.NoError(t, err) + link.Name = "directory_link_name" + assert.Equal(t, 53, productionLinkSize(link.Name, link.Cid)) + + link, err = ipld.MakeLink(ft.EmptyFileNode()) + assert.NoError(t, err) + link.Name = "file_link_name" + assert.Equal(t, 48, productionLinkSize(link.Name, link.Cid)) + + ds := mdtest.Mock() + basicDir := newEmptyBasicDirectory(ds) + assert.NoError(t, err) + for i := 0; i < 10; i++ { + basicDir.AddChild(context.Background(), strconv.FormatUint(uint64(i), 10), ft.EmptyFileNode()) + } + basicDirNode, err := basicDir.GetNode() + assert.NoError(t, err) + link, err = ipld.MakeLink(basicDirNode) + assert.NoError(t, err) + link.Name = "basic_dir" + assert.Equal(t, 43, productionLinkSize(link.Name, link.Cid)) +} + +// Test HAMTDirectory <-> BasicDirectory switch based on directory size. The +// switch is managed by the DynamicDirectory abstraction. +func TestDynamicDirectorySwitch(t *testing.T) { + oldHamtOption := HAMTShardingSize + defer func() { HAMTShardingSize = oldHamtOption }() + HAMTShardingSize = 0 // Disable automatic switch at the start. + linksize.LinkSizeFunction = mockLinkSizeFunc(1) + defer func() { linksize.LinkSizeFunction = productionLinkSize }() + + ds := mdtest.Mock() + dir := NewDirectory(ds) + checkBasicDirectory(t, dir, "new dir is not BasicDirectory") + + ctx := context.Background() + child := ft.EmptyDirNode() + err := ds.Add(ctx, child) + assert.NoError(t, err) + + err = dir.AddChild(ctx, "1", child) + assert.NoError(t, err) + checkBasicDirectory(t, dir, "added child, option still disabled") + + // Set a threshold so big a new entry won't trigger the change. + HAMTShardingSize = math.MaxInt32 + + err = dir.AddChild(ctx, "2", child) + assert.NoError(t, err) + checkBasicDirectory(t, dir, "added child, option now enabled but at max") + + // Now set it so low to make sure any new entry will trigger the upgrade. + HAMTShardingSize = 1 + + // We are already above the threshold, we trigger the switch with an overwrite + // (any AddChild() should reevaluate the size). + err = dir.AddChild(ctx, "2", child) + assert.NoError(t, err) + checkHAMTDirectory(t, dir, "added child, option at min, should switch up") + + // Set threshold at the number of current entries and delete the last one + // to trigger a switch and evaluate if the rest of the entries are conserved. + HAMTShardingSize = 2 + err = dir.RemoveChild(ctx, "2") + assert.NoError(t, err) + checkBasicDirectory(t, dir, "removed threshold entry, option at min, should switch down") +} + +func TestIntegrityOfDirectorySwitch(t *testing.T) { + ds := mdtest.Mock() + dir := NewDirectory(ds) + checkBasicDirectory(t, dir, "new dir is not BasicDirectory") + + ctx := context.Background() + child := ft.EmptyDirNode() + err := ds.Add(ctx, child) + assert.NoError(t, err) + + basicDir := newEmptyBasicDirectory(ds) + hamtDir, err := newEmptyHAMTDirectory(ds, DefaultShardWidth) + assert.NoError(t, err) + for i := 0; i < 1000; i++ { + basicDir.AddChild(ctx, strconv.FormatUint(uint64(i), 10), child) + hamtDir.AddChild(ctx, strconv.FormatUint(uint64(i), 10), child) + } + compareDirectoryEntries(t, basicDir, hamtDir) + + hamtDirFromSwitch, err := basicDir.switchToSharding(ctx) + assert.NoError(t, err) + basicDirFromSwitch, err := hamtDir.switchToBasic(ctx) + assert.NoError(t, err) + compareDirectoryEntries(t, basicDir, basicDirFromSwitch) + compareDirectoryEntries(t, hamtDir, hamtDirFromSwitch) +} + +// This is the value of concurrent fetches during dag.Walk. Used in +// test to better predict how many nodes will be fetched. +var defaultConcurrentFetch = 32 + +// FIXME: Taken from private github.com/ipfs/go-merkledag@v0.2.3/merkledag.go. +// (We can also pass an explicit concurrency value in `(*Shard).EnumLinksAsync()` +// and take ownership of this configuration, but departing from the more +// standard and reliable one in `go-merkledag`. + +// Test that we fetch as little nodes as needed to reach the HAMTShardingSize +// during the sizeBelowThreshold computation. +func TestHAMTEnumerationWhenComputingSize(t *testing.T) { + // Adjust HAMT global/static options for the test to simplify its logic. + // FIXME: These variables weren't designed to be modified and we should + // review in depth side effects. + + // Set all link sizes to a uniform 1 so the estimated directory size + // is just the count of its entry links (in HAMT/Shard terminology these + // are the "value" links pointing to anything that is *not* another Shard). + linksize.LinkSizeFunction = mockLinkSizeFunc(1) + defer func() { linksize.LinkSizeFunction = productionLinkSize }() + + // Use an identity hash function to ease the construction of "complete" HAMTs + // (see CreateCompleteHAMT below for more details). (Ideally this should be + // a parameter we pass and not a global option we modify in the caller.) + oldHashFunc := internal.HAMTHashFunction + defer func() { internal.HAMTHashFunction = oldHashFunc }() + internal.HAMTHashFunction = idHash + + oldHamtOption := HAMTShardingSize + defer func() { HAMTShardingSize = oldHamtOption }() + + // --- End of test static configuration adjustments. --- + + // Some arbitrary values below that make this test not that expensive. + treeHeight := 4 + // How many leaf shards nodes (with value links, + // i.e., directory entries) do we need to reach the threshold. + thresholdToWidthRatio := 4 + // Departing from DefaultShardWidth of 256 to reduce HAMT size in + // CreateCompleteHAMT. + shardWidth := 16 + HAMTShardingSize = shardWidth * thresholdToWidthRatio + + // We create a "complete" HAMT (see CreateCompleteHAMT for more details) + // with a regular structure to be able to predict how many Shard nodes we + // will need to fetch in order to reach the HAMTShardingSize threshold in + // sizeBelowThreshold (assuming a sequential DAG walk function). + + bstore := blockstore.NewBlockstore(dssync.MutexWrap(ds.NewMapDatastore())) + countGetsDS := newCountGetsDS(bstore) + dsrv := mdag.NewDAGService(bsrv.New(countGetsDS, offline.Exchange(countGetsDS))) + completeHAMTRoot, err := CreateCompleteHAMT(dsrv, treeHeight, shardWidth) + assert.NoError(t, err) + + // Calculate the optimal number of nodes to traverse + optimalNodesToFetch := 0 + nodesToProcess := HAMTShardingSize + for i := 0; i < treeHeight-1; i++ { + // divide by the shard width to get the parents and continue up the tree + parentNodes := int(math.Ceil(float64(nodesToProcess) / float64(shardWidth))) + optimalNodesToFetch += parentNodes + nodesToProcess = parentNodes + } + + // With this structure and a BFS traversal (from `parallelWalkDepth`) then + // we would roughly fetch the following nodes: + nodesToFetch := 0 + // * all layers up to (but not including) the last one with leaf nodes + // (because it's a BFS) + for i := 0; i < treeHeight-1; i++ { + nodesToFetch += int(math.Pow(float64(shardWidth), float64(i))) + } + // * `thresholdToWidthRatio` leaf Shards with enough value links to reach + // the HAMTShardingSize threshold. + nodesToFetch += thresholdToWidthRatio + + hamtDir, err := newHAMTDirectoryFromNode(dsrv, completeHAMTRoot) + assert.NoError(t, err) + + countGetsDS.resetCounter() + countGetsDS.setRequestDelay(10 * time.Millisecond) + // (Without the `setRequestDelay` above the number of nodes fetched + // drops dramatically and unpredictably as the BFS starts to behave + // more like a DFS because some search paths are fetched faster than + // others.) + below, err := hamtDir.sizeBelowThreshold(context.TODO(), 0) + assert.NoError(t, err) + assert.False(t, below) + t.Logf("fetched %d nodes (predicted range: %d-%d)", + countGetsDS.uniqueCidsFetched(), optimalNodesToFetch, nodesToFetch+defaultConcurrentFetch) + // Check that the actual number of nodes fetched is within the margin of the + // estimated `nodesToFetch` plus an extra of `defaultConcurrentFetch` since + // we are fetching in parallel. + assert.True(t, countGetsDS.uniqueCidsFetched() <= nodesToFetch+defaultConcurrentFetch) + assert.True(t, countGetsDS.uniqueCidsFetched() >= optimalNodesToFetch) +} + +// Compare entries in the leftDir against the rightDir and possibly +// missingEntries in the second. +func compareDirectoryEntries(t *testing.T, leftDir Directory, rightDir Directory) { + leftLinks, err := getAllLinksSortedByName(leftDir) + assert.NoError(t, err) + rightLinks, err := getAllLinksSortedByName(rightDir) + assert.NoError(t, err) + + assert.Equal(t, len(leftLinks), len(rightLinks)) + + for i, leftLink := range leftLinks { + assert.Equal(t, leftLink, rightLinks[i]) // FIXME: Can we just compare the entire struct? + } +} + +func getAllLinksSortedByName(d Directory) ([]*ipld.Link, error) { + entries, err := d.Links(context.Background()) + if err != nil { + return nil, err + } + sortLinksByName(entries) + return entries, nil +} + +func sortLinksByName(l []*ipld.Link) { + sort.SliceStable(l, func(i, j int) bool { + return strings.Compare(l[i].Name, l[j].Name) == -1 // FIXME: Is this correct? + }) +} + +func TestDirBuilder(t *testing.T) { + ds := mdtest.Mock() + dir := NewDirectory(ds) + ctx := context.Background() + + child := ft.EmptyDirNode() + err := ds.Add(ctx, child) + if err != nil { + t.Fatal(err) + } + + count := 5000 + + for i := 0; i < count; i++ { + err := dir.AddChild(ctx, fmt.Sprintf("entry %d", i), child) + if err != nil { + t.Fatal(err) + } + } + + dirnd, err := dir.GetNode() + if err != nil { + t.Fatal(err) + } + + links, err := dir.Links(ctx) + if err != nil { + t.Fatal(err) + } + + if len(links) != count { + t.Fatal("not enough links dawg", len(links), count) + } + + adir, err := NewDirectoryFromNode(ds, dirnd) + if err != nil { + t.Fatal(err) + } + + links, err = adir.Links(ctx) + if err != nil { + t.Fatal(err) + } + + names := make(map[string]bool) + for _, lnk := range links { + names[lnk.Name] = true + } + + for i := 0; i < count; i++ { + n := fmt.Sprintf("entry %d", i) + if !names[n] { + t.Fatal("COULDNT FIND: ", n) + } + } + + if len(links) != count { + t.Fatal("wrong number of links", len(links), count) + } + + linkResults := dir.EnumLinksAsync(ctx) + + asyncNames := make(map[string]bool) + var asyncLinks []*ipld.Link + + for linkResult := range linkResults { + if linkResult.Err != nil { + t.Fatal(linkResult.Err) + } + asyncNames[linkResult.Link.Name] = true + asyncLinks = append(asyncLinks, linkResult.Link) + } + + for i := 0; i < count; i++ { + n := fmt.Sprintf("entry %d", i) + if !asyncNames[n] { + t.Fatal("COULDNT FIND: ", n) + } + } + + if len(asyncLinks) != count { + t.Fatal("wrong number of links", len(asyncLinks), count) + } +} + +func newHAMTDirectoryFromNode(dserv ipld.DAGService, node ipld.Node) (*HAMTDirectory, error) { + shard, err := hamt.NewHamtFromDag(dserv, node) + if err != nil { + return nil, err + } + return &HAMTDirectory{ + dserv: dserv, + shard: shard, + }, nil +} + +func newEmptyHAMTDirectory(dserv ipld.DAGService, shardWidth int) (*HAMTDirectory, error) { + shard, err := hamt.NewShard(dserv, shardWidth) + if err != nil { + return nil, err + } + + return &HAMTDirectory{ + dserv: dserv, + shard: shard, + }, nil +} + +// countGetsDS is a DAG service that keeps track of the number of +// unique CIDs fetched. +type countGetsDS struct { + blockstore.Blockstore + + cidsFetched map[cid.Cid]struct{} + mapLock sync.Mutex + started bool + + getRequestDelay time.Duration +} + +var _ blockstore.Blockstore = (*countGetsDS)(nil) + +func newCountGetsDS(bs blockstore.Blockstore) *countGetsDS { + return &countGetsDS{ + bs, + make(map[cid.Cid]struct{}), + sync.Mutex{}, + false, + 0, + } +} + +func (d *countGetsDS) resetCounter() { + d.mapLock.Lock() + defer d.mapLock.Unlock() + d.cidsFetched = make(map[cid.Cid]struct{}) + d.started = true +} + +func (d *countGetsDS) uniqueCidsFetched() int { + d.mapLock.Lock() + defer d.mapLock.Unlock() + return len(d.cidsFetched) +} + +func (d *countGetsDS) setRequestDelay(timeout time.Duration) { + d.getRequestDelay = timeout +} + +func (d *countGetsDS) maybeSleep(c cid.Cid) { + d.mapLock.Lock() + _, cidRequestedBefore := d.cidsFetched[c] + d.cidsFetched[c] = struct{}{} + d.mapLock.Unlock() + + if d.getRequestDelay != 0 && !cidRequestedBefore { + // First request gets a timeout to simulate a network fetch. + // Subsequent requests get no timeout simulating an in-disk cache. + time.Sleep(d.getRequestDelay) + } +} + +func (d *countGetsDS) Has(ctx context.Context, c cid.Cid) (bool, error) { + if d.started { + panic("implement me") + } + return d.Blockstore.Has(ctx, c) +} + +func (d *countGetsDS) Get(ctx context.Context, c cid.Cid) (blocks.Block, error) { + blk, err := d.Blockstore.Get(ctx, c) + if err != nil { + return nil, err + } + + d.maybeSleep(c) + return blk, nil +} + +func (d *countGetsDS) GetSize(ctx context.Context, c cid.Cid) (int, error) { + if d.started { + panic("implement me") + } + return d.Blockstore.GetSize(ctx, c) +} + +func (d *countGetsDS) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) { + if d.started { + panic("implement me") + } + return d.Blockstore.AllKeysChan(ctx) +} diff --git a/unixfs/io/doc.go b/unixfs/io/doc.go new file mode 100644 index 0000000000..cf844bd23a --- /dev/null +++ b/unixfs/io/doc.go @@ -0,0 +1,3 @@ +// Package io implements convenience objects for working with the ipfs +// unixfs data format. +package io diff --git a/unixfs/io/resolve.go b/unixfs/io/resolve.go new file mode 100644 index 0000000000..4ec530bb3d --- /dev/null +++ b/unixfs/io/resolve.go @@ -0,0 +1,41 @@ +package io + +import ( + "context" + + ft "github.com/ipfs/go-libipfs/unixfs" + hamt "github.com/ipfs/go-libipfs/unixfs/hamt" + dag "github.com/ipfs/go-merkledag" + + ipld "github.com/ipfs/go-ipld-format" +) + +// ResolveUnixfsOnce resolves a single hop of a path through a graph in a +// unixfs context. This includes handling traversing sharded directories. +func ResolveUnixfsOnce(ctx context.Context, ds ipld.NodeGetter, nd ipld.Node, names []string) (*ipld.Link, []string, error) { + pn, ok := nd.(*dag.ProtoNode) + if ok { + fsn, err := ft.FSNodeFromBytes(pn.Data()) + if err != nil { + // Not a unixfs node, use standard object traversal code + return nd.ResolveLink(names) + } + + if fsn.Type() == ft.THAMTShard { + rods := dag.NewReadOnlyDagService(ds) + s, err := hamt.NewHamtFromDag(rods, nd) + if err != nil { + return nil, nil, err + } + + out, err := s.Find(ctx, names[0]) + if err != nil { + return nil, nil, err + } + + return out, names[1:], nil + } + } + + return nd.ResolveLink(names) +} diff --git a/unixfs/mod/dagmodifier.go b/unixfs/mod/dagmodifier.go new file mode 100644 index 0000000000..e5e7befbc4 --- /dev/null +++ b/unixfs/mod/dagmodifier.go @@ -0,0 +1,605 @@ +// Package mod provides DAG modification utilities to, for example, +// insert additional nodes in a unixfs DAG or truncate them. +package mod + +import ( + "bytes" + "context" + "errors" + "io" + + ft "github.com/ipfs/go-libipfs/unixfs" + help "github.com/ipfs/go-libipfs/unixfs/importer/helpers" + trickle "github.com/ipfs/go-libipfs/unixfs/importer/trickle" + uio "github.com/ipfs/go-libipfs/unixfs/io" + + cid "github.com/ipfs/go-cid" + ipld "github.com/ipfs/go-ipld-format" + chunker "github.com/ipfs/go-libipfs/chunker" + mdag "github.com/ipfs/go-merkledag" +) + +// Common errors +var ( + ErrSeekFail = errors.New("failed to seek properly") + ErrUnrecognizedWhence = errors.New("unrecognized whence") + ErrNotUnixfs = errors.New("dagmodifier only supports unixfs nodes (proto or raw)") +) + +// 2MB +var writebufferSize = 1 << 21 + +// DagModifier is the only struct licensed and able to correctly +// perform surgery on a DAG 'file' +// Dear god, please rename this to something more pleasant +type DagModifier struct { + dagserv ipld.DAGService + curNode ipld.Node + + splitter chunker.SplitterGen + ctx context.Context + readCancel func() + + writeStart uint64 + curWrOff uint64 + wrBuf *bytes.Buffer + + Prefix cid.Prefix + RawLeaves bool + + read uio.DagReader +} + +// NewDagModifier returns a new DagModifier, the Cid prefix for newly +// created nodes will be inhered from the passed in node. If the Cid +// version if not 0 raw leaves will also be enabled. The Prefix and +// RawLeaves options can be overridden by changing them after the call. +func NewDagModifier(ctx context.Context, from ipld.Node, serv ipld.DAGService, spl chunker.SplitterGen) (*DagModifier, error) { + switch from.(type) { + case *mdag.ProtoNode, *mdag.RawNode: + // ok + default: + return nil, ErrNotUnixfs + } + + prefix := from.Cid().Prefix() + prefix.Codec = cid.DagProtobuf + rawLeaves := false + if prefix.Version > 0 { + rawLeaves = true + } + + return &DagModifier{ + curNode: from.Copy(), + dagserv: serv, + splitter: spl, + ctx: ctx, + Prefix: prefix, + RawLeaves: rawLeaves, + }, nil +} + +// WriteAt will modify a dag file in place +func (dm *DagModifier) WriteAt(b []byte, offset int64) (int, error) { + // TODO: this is currently VERY inefficient + // each write that happens at an offset other than the current one causes a + // flush to disk, and dag rewrite + if offset == int64(dm.writeStart) && dm.wrBuf != nil { + // If we would overwrite the previous write + if len(b) >= dm.wrBuf.Len() { + dm.wrBuf.Reset() + } + } else if uint64(offset) != dm.curWrOff { + size, err := dm.Size() + if err != nil { + return 0, err + } + if offset > size { + err := dm.expandSparse(offset - size) + if err != nil { + return 0, err + } + } + + err = dm.Sync() + if err != nil { + return 0, err + } + dm.writeStart = uint64(offset) + } + + return dm.Write(b) +} + +// A reader that just returns zeros +type zeroReader struct{} + +func (zr zeroReader) Read(b []byte) (int, error) { + for i := range b { + b[i] = 0 + } + return len(b), nil +} + +// expandSparse grows the file with zero blocks of 4096 +// A small blocksize is chosen to aid in deduplication +func (dm *DagModifier) expandSparse(size int64) error { + r := io.LimitReader(zeroReader{}, size) + spl := chunker.NewSizeSplitter(r, 4096) + nnode, err := dm.appendData(dm.curNode, spl) + if err != nil { + return err + } + err = dm.dagserv.Add(dm.ctx, nnode) + return err +} + +// Write continues writing to the dag at the current offset +func (dm *DagModifier) Write(b []byte) (int, error) { + if dm.read != nil { + dm.read = nil + } + if dm.wrBuf == nil { + dm.wrBuf = new(bytes.Buffer) + } + + n, err := dm.wrBuf.Write(b) + if err != nil { + return n, err + } + dm.curWrOff += uint64(n) + if dm.wrBuf.Len() > writebufferSize { + err := dm.Sync() + if err != nil { + return n, err + } + } + return n, nil +} + +// Size returns the Filesize of the node +func (dm *DagModifier) Size() (int64, error) { + fileSize, err := fileSize(dm.curNode) + if err != nil { + return 0, err + } + if dm.wrBuf != nil && int64(dm.wrBuf.Len())+int64(dm.writeStart) > int64(fileSize) { + return int64(dm.wrBuf.Len()) + int64(dm.writeStart), nil + } + return int64(fileSize), nil +} + +func fileSize(n ipld.Node) (uint64, error) { + switch nd := n.(type) { + case *mdag.ProtoNode: + fsn, err := ft.FSNodeFromBytes(nd.Data()) + if err != nil { + return 0, err + } + return fsn.FileSize(), nil + case *mdag.RawNode: + return uint64(len(nd.RawData())), nil + default: + return 0, ErrNotUnixfs + } +} + +// Sync writes changes to this dag to disk +func (dm *DagModifier) Sync() error { + // No buffer? Nothing to do + if dm.wrBuf == nil { + return nil + } + + // If we have an active reader, kill it + if dm.read != nil { + dm.read = nil + dm.readCancel() + } + + // Number of bytes we're going to write + buflen := dm.wrBuf.Len() + + fs, err := fileSize(dm.curNode) + if err != nil { + return err + } + if fs < dm.writeStart { + if err := dm.expandSparse(int64(dm.writeStart - fs)); err != nil { + return err + } + } + + // overwrite existing dag nodes + thisc, err := dm.modifyDag(dm.curNode, dm.writeStart) + if err != nil { + return err + } + + dm.curNode, err = dm.dagserv.Get(dm.ctx, thisc) + if err != nil { + return err + } + + // need to write past end of current dag + if dm.wrBuf.Len() > 0 { + dm.curNode, err = dm.appendData(dm.curNode, dm.splitter(dm.wrBuf)) + if err != nil { + return err + } + + err = dm.dagserv.Add(dm.ctx, dm.curNode) + if err != nil { + return err + } + } + + dm.writeStart += uint64(buflen) + dm.wrBuf = nil + + return nil +} + +// modifyDag writes the data in 'dm.wrBuf' over the data in 'node' starting at 'offset' +// returns the new key of the passed in node. +func (dm *DagModifier) modifyDag(n ipld.Node, offset uint64) (cid.Cid, error) { + // If we've reached a leaf node. + if len(n.Links()) == 0 { + switch nd0 := n.(type) { + case *mdag.ProtoNode: + fsn, err := ft.FSNodeFromBytes(nd0.Data()) + if err != nil { + return cid.Cid{}, err + } + + _, err = dm.wrBuf.Read(fsn.Data()[offset:]) + if err != nil && err != io.EOF { + return cid.Cid{}, err + } + + // Update newly written node.. + b, err := fsn.GetBytes() + if err != nil { + return cid.Cid{}, err + } + + nd := new(mdag.ProtoNode) + nd.SetData(b) + nd.SetCidBuilder(nd0.CidBuilder()) + err = dm.dagserv.Add(dm.ctx, nd) + if err != nil { + return cid.Cid{}, err + } + + return nd.Cid(), nil + case *mdag.RawNode: + origData := nd0.RawData() + bytes := make([]byte, len(origData)) + + // copy orig data up to offset + copy(bytes, origData[:offset]) + + // copy in new data + n, err := dm.wrBuf.Read(bytes[offset:]) + if err != nil && err != io.EOF { + return cid.Cid{}, err + } + + // copy remaining data + offsetPlusN := int(offset) + n + if offsetPlusN < len(origData) { + copy(bytes[offsetPlusN:], origData[offsetPlusN:]) + } + + nd, err := mdag.NewRawNodeWPrefix(bytes, nd0.Cid().Prefix()) + if err != nil { + return cid.Cid{}, err + } + err = dm.dagserv.Add(dm.ctx, nd) + if err != nil { + return cid.Cid{}, err + } + + return nd.Cid(), nil + } + } + + node, ok := n.(*mdag.ProtoNode) + if !ok { + return cid.Cid{}, ErrNotUnixfs + } + + fsn, err := ft.FSNodeFromBytes(node.Data()) + if err != nil { + return cid.Cid{}, err + } + + var cur uint64 + for i, bs := range fsn.BlockSizes() { + // We found the correct child to write into + if cur+bs > offset { + child, err := node.Links()[i].GetNode(dm.ctx, dm.dagserv) + if err != nil { + return cid.Cid{}, err + } + + k, err := dm.modifyDag(child, offset-cur) + if err != nil { + return cid.Cid{}, err + } + + node.Links()[i].Cid = k + + // Recache serialized node + _, err = node.EncodeProtobuf(true) + if err != nil { + return cid.Cid{}, err + } + + if dm.wrBuf.Len() == 0 { + // No more bytes to write! + break + } + offset = cur + bs + } + cur += bs + } + + err = dm.dagserv.Add(dm.ctx, node) + return node.Cid(), err +} + +// appendData appends the blocks from the given chan to the end of this dag +func (dm *DagModifier) appendData(nd ipld.Node, spl chunker.Splitter) (ipld.Node, error) { + switch nd := nd.(type) { + case *mdag.ProtoNode, *mdag.RawNode: + dbp := &help.DagBuilderParams{ + Dagserv: dm.dagserv, + Maxlinks: help.DefaultLinksPerBlock, + CidBuilder: dm.Prefix, + RawLeaves: dm.RawLeaves, + } + db, err := dbp.New(spl) + if err != nil { + return nil, err + } + return trickle.Append(dm.ctx, nd, db) + default: + return nil, ErrNotUnixfs + } +} + +// Read data from this dag starting at the current offset +func (dm *DagModifier) Read(b []byte) (int, error) { + err := dm.readPrep() + if err != nil { + return 0, err + } + + n, err := dm.read.Read(b) + dm.curWrOff += uint64(n) + return n, err +} + +func (dm *DagModifier) readPrep() error { + err := dm.Sync() + if err != nil { + return err + } + + if dm.read == nil { + ctx, cancel := context.WithCancel(dm.ctx) + dr, err := uio.NewDagReader(ctx, dm.curNode, dm.dagserv) + if err != nil { + cancel() + return err + } + + i, err := dr.Seek(int64(dm.curWrOff), io.SeekStart) + if err != nil { + cancel() + return err + } + + if i != int64(dm.curWrOff) { + cancel() + return ErrSeekFail + } + + dm.readCancel = cancel + dm.read = dr + } + + return nil +} + +// CtxReadFull reads data from this dag starting at the current offset +func (dm *DagModifier) CtxReadFull(ctx context.Context, b []byte) (int, error) { + err := dm.readPrep() + if err != nil { + return 0, err + } + + n, err := dm.read.CtxReadFull(ctx, b) + dm.curWrOff += uint64(n) + return n, err +} + +// GetNode gets the modified DAG Node +func (dm *DagModifier) GetNode() (ipld.Node, error) { + err := dm.Sync() + if err != nil { + return nil, err + } + return dm.curNode.Copy(), nil +} + +// HasChanges returned whether or not there are unflushed changes to this dag +func (dm *DagModifier) HasChanges() bool { + return dm.wrBuf != nil +} + +// Seek modifies the offset according to whence. See unixfs/io for valid whence +// values. +func (dm *DagModifier) Seek(offset int64, whence int) (int64, error) { + err := dm.Sync() + if err != nil { + return 0, err + } + + fisize, err := dm.Size() + if err != nil { + return 0, err + } + + var newoffset uint64 + switch whence { + case io.SeekCurrent: + newoffset = dm.curWrOff + uint64(offset) + case io.SeekStart: + newoffset = uint64(offset) + case io.SeekEnd: + newoffset = uint64(fisize) - uint64(offset) + default: + return 0, ErrUnrecognizedWhence + } + + if int64(newoffset) > fisize { + if err := dm.expandSparse(int64(newoffset) - fisize); err != nil { + return 0, err + } + } + dm.curWrOff = newoffset + dm.writeStart = newoffset + + if dm.read != nil { + _, err = dm.read.Seek(offset, whence) + if err != nil { + return 0, err + } + } + + return int64(dm.curWrOff), nil +} + +// Truncate truncates the current Node to 'size' and replaces it with the +// new one. +func (dm *DagModifier) Truncate(size int64) error { + err := dm.Sync() + if err != nil { + return err + } + + realSize, err := dm.Size() + if err != nil { + return err + } + if size == int64(realSize) { + return nil + } + + // Truncate can also be used to expand the file + if size > int64(realSize) { + return dm.expandSparse(int64(size) - realSize) + } + + nnode, err := dm.dagTruncate(dm.ctx, dm.curNode, uint64(size)) + if err != nil { + return err + } + + err = dm.dagserv.Add(dm.ctx, nnode) + if err != nil { + return err + } + + dm.curNode = nnode + return nil +} + +// dagTruncate truncates the given node to 'size' and returns the modified Node +func (dm *DagModifier) dagTruncate(ctx context.Context, n ipld.Node, size uint64) (ipld.Node, error) { + if len(n.Links()) == 0 { + switch nd := n.(type) { + case *mdag.ProtoNode: + // TODO: this can likely be done without marshaling and remarshaling + fsn, err := ft.FSNodeFromBytes(nd.Data()) + if err != nil { + return nil, err + } + nd.SetData(ft.WrapData(fsn.Data()[:size])) + return nd, nil + case *mdag.RawNode: + return mdag.NewRawNodeWPrefix(nd.RawData()[:size], nd.Cid().Prefix()) + } + } + + nd, ok := n.(*mdag.ProtoNode) + if !ok { + return nil, ErrNotUnixfs + } + + var cur uint64 + end := 0 + var modified ipld.Node + ndata, err := ft.FSNodeFromBytes(nd.Data()) + if err != nil { + return nil, err + } + // Reset the block sizes of the node to adjust them + // with the new values of the truncated children. + ndata.RemoveAllBlockSizes() + for i, lnk := range nd.Links() { + child, err := lnk.GetNode(ctx, dm.dagserv) + if err != nil { + return nil, err + } + + childsize, err := fileSize(child) + if err != nil { + return nil, err + } + + // found the child we want to cut + if size < cur+childsize { + nchild, err := dm.dagTruncate(ctx, child, size-cur) + if err != nil { + return nil, err + } + + ndata.AddBlockSize(size - cur) + + modified = nchild + end = i + break + } + cur += childsize + ndata.AddBlockSize(childsize) + } + + err = dm.dagserv.Add(ctx, modified) + if err != nil { + return nil, err + } + + nd.SetLinks(nd.Links()[:end]) + err = nd.AddNodeLink("", modified) + if err != nil { + return nil, err + } + + d, err := ndata.GetBytes() + if err != nil { + return nil, err + } + // Save the new block sizes to the original node. + nd.SetData(d) + + // invalidate cache and recompute serialized data + _, err = nd.EncodeProtobuf(true) + if err != nil { + return nil, err + } + + return nd, nil +} diff --git a/unixfs/mod/dagmodifier_test.go b/unixfs/mod/dagmodifier_test.go new file mode 100644 index 0000000000..4fc7b66493 --- /dev/null +++ b/unixfs/mod/dagmodifier_test.go @@ -0,0 +1,859 @@ +package mod + +import ( + "context" + "fmt" + "io" + "testing" + + h "github.com/ipfs/go-libipfs/unixfs/importer/helpers" + trickle "github.com/ipfs/go-libipfs/unixfs/importer/trickle" + uio "github.com/ipfs/go-libipfs/unixfs/io" + testu "github.com/ipfs/go-libipfs/unixfs/test" + dag "github.com/ipfs/go-merkledag" + + "github.com/ipfs/go-libipfs/unixfs" + u "github.com/ipfs/go-libipfs/util" +) + +func testModWrite(t *testing.T, beg, size uint64, orig []byte, dm *DagModifier, opts testu.NodeOpts) []byte { + newdata := make([]byte, size) + r := u.NewTimeSeededRand() + r.Read(newdata) + + if size+beg > uint64(len(orig)) { + orig = append(orig, make([]byte, (size+beg)-uint64(len(orig)))...) + } + copy(orig[beg:], newdata) + + nmod, err := dm.WriteAt(newdata, int64(beg)) + if err != nil { + t.Fatal(err) + } + + if nmod != int(size) { + t.Fatalf("Mod length not correct! %d != %d", nmod, size) + } + + verifyNode(t, orig, dm, opts) + + return orig +} + +func verifyNode(t *testing.T, orig []byte, dm *DagModifier, opts testu.NodeOpts) { + nd, err := dm.GetNode() + if err != nil { + t.Fatal(err) + } + + err = trickle.VerifyTrickleDagStructure(nd, trickle.VerifyParams{ + Getter: dm.dagserv, + Direct: h.DefaultLinksPerBlock, + LayerRepeat: 4, + Prefix: &opts.Prefix, + RawLeaves: opts.RawLeavesUsed, + }) + if err != nil { + t.Fatal(err) + } + + rd, err := uio.NewDagReader(context.Background(), nd, dm.dagserv) + if err != nil { + t.Fatal(err) + } + + after, err := io.ReadAll(rd) + if err != nil { + t.Fatal(err) + } + + err = testu.ArrComp(after, orig) + if err != nil { + t.Fatal(err) + } +} + +func runAllSubtests(t *testing.T, tfunc func(*testing.T, testu.NodeOpts)) { + t.Run("opts=ProtoBufLeaves", func(t *testing.T) { tfunc(t, testu.UseProtoBufLeaves) }) + t.Run("opts=RawLeaves", func(t *testing.T) { tfunc(t, testu.UseRawLeaves) }) + t.Run("opts=CidV1", func(t *testing.T) { tfunc(t, testu.UseCidV1) }) + t.Run("opts=Blake2b256", func(t *testing.T) { tfunc(t, testu.UseBlake2b256) }) +} + +func TestDagModifierBasic(t *testing.T) { + runAllSubtests(t, testDagModifierBasic) +} +func testDagModifierBasic(t *testing.T, opts testu.NodeOpts) { + dserv := testu.GetDAGServ() + b, n := testu.GetRandomNode(t, dserv, 50000, opts) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + dagmod, err := NewDagModifier(ctx, n, dserv, testu.SizeSplitterGen(512)) + if err != nil { + t.Fatal(err) + } + if opts.ForceRawLeaves { + dagmod.RawLeaves = true + } + + // Within zero block + beg := uint64(15) + length := uint64(60) + + t.Log("Testing mod within zero block") + b = testModWrite(t, beg, length, b, dagmod, opts) + + // Within bounds of existing file + beg = 1000 + length = 4000 + t.Log("Testing mod within bounds of existing multiblock file.") + b = testModWrite(t, beg, length, b, dagmod, opts) + + // Extend bounds + beg = 49500 + length = 4000 + + t.Log("Testing mod that extends file.") + b = testModWrite(t, beg, length, b, dagmod, opts) + + // "Append" + beg = uint64(len(b)) + length = 3000 + t.Log("Testing pure append") + _ = testModWrite(t, beg, length, b, dagmod, opts) + + // Verify reported length + node, err := dagmod.GetNode() + if err != nil { + t.Fatal(err) + } + + size, err := fileSize(node) + if err != nil { + t.Fatal(err) + } + + expected := uint64(50000 + 3500 + 3000) + if size != expected { + t.Fatalf("Final reported size is incorrect [%d != %d]", size, expected) + } +} + +func TestMultiWrite(t *testing.T) { + runAllSubtests(t, testMultiWrite) +} +func testMultiWrite(t *testing.T, opts testu.NodeOpts) { + dserv := testu.GetDAGServ() + n := testu.GetEmptyNode(t, dserv, opts) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + dagmod, err := NewDagModifier(ctx, n, dserv, testu.SizeSplitterGen(512)) + if err != nil { + t.Fatal(err) + } + if opts.ForceRawLeaves { + dagmod.RawLeaves = true + } + + data := make([]byte, 4000) + u.NewTimeSeededRand().Read(data) + + for i := 0; i < len(data); i++ { + n, err := dagmod.WriteAt(data[i:i+1], int64(i)) + if err != nil { + t.Fatal(err) + } + if n != 1 { + t.Fatal("Somehow wrote the wrong number of bytes! (n != 1)") + } + + size, err := dagmod.Size() + if err != nil { + t.Fatal(err) + } + + if size != int64(i+1) { + t.Fatal("Size was reported incorrectly") + } + } + + verifyNode(t, data, dagmod, opts) +} + +func TestMultiWriteAndFlush(t *testing.T) { + runAllSubtests(t, testMultiWriteAndFlush) +} +func testMultiWriteAndFlush(t *testing.T, opts testu.NodeOpts) { + dserv := testu.GetDAGServ() + n := testu.GetEmptyNode(t, dserv, opts) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + dagmod, err := NewDagModifier(ctx, n, dserv, testu.SizeSplitterGen(512)) + if err != nil { + t.Fatal(err) + } + if opts.ForceRawLeaves { + dagmod.RawLeaves = true + } + + data := make([]byte, 20) + u.NewTimeSeededRand().Read(data) + + for i := 0; i < len(data); i++ { + n, err := dagmod.WriteAt(data[i:i+1], int64(i)) + if err != nil { + t.Fatal(err) + } + if n != 1 { + t.Fatal("Somehow wrote the wrong number of bytes! (n != 1)") + } + err = dagmod.Sync() + if err != nil { + t.Fatal(err) + } + } + + verifyNode(t, data, dagmod, opts) +} + +func TestWriteNewFile(t *testing.T) { + runAllSubtests(t, testWriteNewFile) +} +func testWriteNewFile(t *testing.T, opts testu.NodeOpts) { + dserv := testu.GetDAGServ() + n := testu.GetEmptyNode(t, dserv, opts) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + dagmod, err := NewDagModifier(ctx, n, dserv, testu.SizeSplitterGen(512)) + if err != nil { + t.Fatal(err) + } + if opts.ForceRawLeaves { + dagmod.RawLeaves = true + } + + towrite := make([]byte, 2000) + u.NewTimeSeededRand().Read(towrite) + + nw, err := dagmod.Write(towrite) + if err != nil { + t.Fatal(err) + } + if nw != len(towrite) { + t.Fatal("Wrote wrong amount") + } + + verifyNode(t, towrite, dagmod, opts) +} + +func TestMultiWriteCoal(t *testing.T) { + runAllSubtests(t, testMultiWriteCoal) +} +func testMultiWriteCoal(t *testing.T, opts testu.NodeOpts) { + dserv := testu.GetDAGServ() + n := testu.GetEmptyNode(t, dserv, opts) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + dagmod, err := NewDagModifier(ctx, n, dserv, testu.SizeSplitterGen(512)) + if err != nil { + t.Fatal(err) + } + if opts.ForceRawLeaves { + dagmod.RawLeaves = true + } + + data := make([]byte, 1000) + u.NewTimeSeededRand().Read(data) + + for i := 0; i < len(data); i++ { + n, err := dagmod.WriteAt(data[:i+1], 0) + if err != nil { + fmt.Println("FAIL AT ", i) + t.Fatal(err) + } + if n != i+1 { + t.Fatal("Somehow wrote the wrong number of bytes! (n != 1)") + } + + } + + verifyNode(t, data, dagmod, opts) +} + +func TestLargeWriteChunks(t *testing.T) { + runAllSubtests(t, testLargeWriteChunks) +} +func testLargeWriteChunks(t *testing.T, opts testu.NodeOpts) { + dserv := testu.GetDAGServ() + n := testu.GetEmptyNode(t, dserv, opts) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + dagmod, err := NewDagModifier(ctx, n, dserv, testu.SizeSplitterGen(512)) + if err != nil { + t.Fatal(err) + } + if opts.ForceRawLeaves { + dagmod.RawLeaves = true + } + + wrsize := 1000 + datasize := 10000000 + data := make([]byte, datasize) + + u.NewTimeSeededRand().Read(data) + + for i := 0; i < datasize/wrsize; i++ { + n, err := dagmod.WriteAt(data[i*wrsize:(i+1)*wrsize], int64(i*wrsize)) + if err != nil { + t.Fatal(err) + } + if n != wrsize { + t.Fatal("failed to write buffer") + } + } + + _, err = dagmod.Seek(0, io.SeekStart) + if err != nil { + t.Fatal(err) + } + + out, err := io.ReadAll(dagmod) + if err != nil { + t.Fatal(err) + } + + if err = testu.ArrComp(out, data); err != nil { + t.Fatal(err) + } +} + +func TestDagTruncate(t *testing.T) { + runAllSubtests(t, testDagTruncate) +} +func testDagTruncate(t *testing.T, opts testu.NodeOpts) { + dserv := testu.GetDAGServ() + b, n := testu.GetRandomNode(t, dserv, 50000, opts) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + dagmod, err := NewDagModifier(ctx, n, dserv, testu.SizeSplitterGen(512)) + if err != nil { + t.Fatal(err) + } + if opts.ForceRawLeaves { + dagmod.RawLeaves = true + } + + err = dagmod.Truncate(12345) + if err != nil { + t.Fatal(err) + } + size, err := dagmod.Size() + if err != nil { + t.Fatal(err) + } + + if size != 12345 { + t.Fatal("size was incorrect!") + } + + _, err = dagmod.Seek(0, io.SeekStart) + if err != nil { + t.Fatal(err) + } + + out, err := io.ReadAll(dagmod) + if err != nil { + t.Fatal(err) + } + + if err = testu.ArrComp(out, b[:12345]); err != nil { + t.Fatal(err) + } + + err = dagmod.Truncate(10) + if err != nil { + t.Fatal(err) + } + + size, err = dagmod.Size() + if err != nil { + t.Fatal(err) + } + + if size != 10 { + t.Fatal("size was incorrect!") + } + + err = dagmod.Truncate(0) + if err != nil { + t.Fatal(err) + } + + size, err = dagmod.Size() + if err != nil { + t.Fatal(err) + } + + if size != 0 { + t.Fatal("size was incorrect!") + } +} + +// TestDagSync tests that a DAG will expand sparse during sync +// if offset > curNode's size. +func TestDagSync(t *testing.T) { + dserv := testu.GetDAGServ() + nd := dag.NodeWithData(unixfs.FilePBData(nil, 0)) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + dagmod, err := NewDagModifier(ctx, nd, dserv, testu.SizeSplitterGen(512)) + if err != nil { + t.Fatal(err) + } + + _, err = dagmod.Write([]byte("test1")) + if err != nil { + t.Fatal(err) + } + + err = dagmod.Sync() + if err != nil { + t.Fatal(err) + } + + // Truncate leave the offset at 5 and filesize at 0 + err = dagmod.Truncate(0) + if err != nil { + t.Fatal(err) + } + + _, err = dagmod.Write([]byte("test2")) + if err != nil { + t.Fatal(err) + } + + // When Offset > filesize , Sync will call enpandSparse + err = dagmod.Sync() + if err != nil { + t.Fatal(err) + } + + _, err = dagmod.Seek(0, io.SeekStart) + if err != nil { + t.Fatal(err) + } + + out, err := io.ReadAll(dagmod) + if err != nil { + t.Fatal(err) + } + + if err = testu.ArrComp(out[5:], []byte("test2")); err != nil { + t.Fatal(err) + } +} + +// TestDagTruncateSameSize tests that a DAG truncated +// to the same size (i.e., doing nothing) doesn't modify +// the DAG (its hash). +func TestDagTruncateSameSize(t *testing.T) { + runAllSubtests(t, testDagTruncateSameSize) +} +func testDagTruncateSameSize(t *testing.T, opts testu.NodeOpts) { + dserv := testu.GetDAGServ() + _, n := testu.GetRandomNode(t, dserv, 50000, opts) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + dagmod, err := NewDagModifier(ctx, n, dserv, testu.SizeSplitterGen(512)) + if err != nil { + t.Fatal(err) + } + // Copied from `TestDagTruncate`. + + size, err := dagmod.Size() + if err != nil { + t.Fatal(err) + } + + err = dagmod.Truncate(size) + if err != nil { + t.Fatal(err) + } + + modifiedNode, err := dagmod.GetNode() + if err != nil { + t.Fatal(err) + } + + if modifiedNode.Cid().Equals(n.Cid()) == false { + t.Fatal("the node has been modified!") + } +} + +func TestSparseWrite(t *testing.T) { + runAllSubtests(t, testSparseWrite) +} +func testSparseWrite(t *testing.T, opts testu.NodeOpts) { + dserv := testu.GetDAGServ() + n := testu.GetEmptyNode(t, dserv, opts) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + dagmod, err := NewDagModifier(ctx, n, dserv, testu.SizeSplitterGen(512)) + if err != nil { + t.Fatal(err) + } + if opts.ForceRawLeaves { + dagmod.RawLeaves = true + } + + buf := make([]byte, 5000) + u.NewTimeSeededRand().Read(buf[2500:]) + + wrote, err := dagmod.WriteAt(buf[2500:], 2500) + if err != nil { + t.Fatal(err) + } + + if wrote != 2500 { + t.Fatal("incorrect write amount") + } + + _, err = dagmod.Seek(0, io.SeekStart) + if err != nil { + t.Fatal(err) + } + + out, err := io.ReadAll(dagmod) + if err != nil { + t.Fatal(err) + } + + if err = testu.ArrComp(out, buf); err != nil { + t.Fatal(err) + } +} + +func TestSeekPastEndWrite(t *testing.T) { + runAllSubtests(t, testSeekPastEndWrite) +} +func testSeekPastEndWrite(t *testing.T, opts testu.NodeOpts) { + dserv := testu.GetDAGServ() + n := testu.GetEmptyNode(t, dserv, opts) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + dagmod, err := NewDagModifier(ctx, n, dserv, testu.SizeSplitterGen(512)) + if err != nil { + t.Fatal(err) + } + if opts.ForceRawLeaves { + dagmod.RawLeaves = true + } + + buf := make([]byte, 5000) + u.NewTimeSeededRand().Read(buf[2500:]) + + nseek, err := dagmod.Seek(2500, io.SeekStart) + if err != nil { + t.Fatal(err) + } + + if nseek != 2500 { + t.Fatal("failed to seek") + } + + wrote, err := dagmod.Write(buf[2500:]) + if err != nil { + t.Fatal(err) + } + + if wrote != 2500 { + t.Fatal("incorrect write amount") + } + + _, err = dagmod.Seek(0, io.SeekStart) + if err != nil { + t.Fatal(err) + } + + out, err := io.ReadAll(dagmod) + if err != nil { + t.Fatal(err) + } + + if err = testu.ArrComp(out, buf); err != nil { + t.Fatal(err) + } +} + +func TestRelativeSeek(t *testing.T) { + runAllSubtests(t, testRelativeSeek) +} +func testRelativeSeek(t *testing.T, opts testu.NodeOpts) { + dserv := testu.GetDAGServ() + n := testu.GetEmptyNode(t, dserv, opts) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + dagmod, err := NewDagModifier(ctx, n, dserv, testu.SizeSplitterGen(512)) + if err != nil { + t.Fatal(err) + } + if opts.ForceRawLeaves { + dagmod.RawLeaves = true + } + + for i := 0; i < 64; i++ { + dagmod.Write([]byte{byte(i)}) + if _, err := dagmod.Seek(1, io.SeekCurrent); err != nil { + t.Fatal(err) + } + } + + out, err := io.ReadAll(dagmod) + if err != nil { + t.Fatal(err) + } + + for i, v := range out { + if v != 0 && i/2 != int(v) { + t.Errorf("expected %d, at index %d, got %d", i/2, i, v) + } + } +} + +func TestInvalidSeek(t *testing.T) { + runAllSubtests(t, testInvalidSeek) +} +func testInvalidSeek(t *testing.T, opts testu.NodeOpts) { + dserv := testu.GetDAGServ() + n := testu.GetEmptyNode(t, dserv, opts) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + dagmod, err := NewDagModifier(ctx, n, dserv, testu.SizeSplitterGen(512)) + if err != nil { + t.Fatal(err) + } + if opts.ForceRawLeaves { + dagmod.RawLeaves = true + } + + _, err = dagmod.Seek(10, -10) + + if err != ErrUnrecognizedWhence { + t.Fatal(err) + } +} + +func TestEndSeek(t *testing.T) { + runAllSubtests(t, testEndSeek) +} +func testEndSeek(t *testing.T, opts testu.NodeOpts) { + dserv := testu.GetDAGServ() + + n := testu.GetEmptyNode(t, dserv, opts) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + dagmod, err := NewDagModifier(ctx, n, dserv, testu.SizeSplitterGen(512)) + if err != nil { + t.Fatal(err) + } + if opts.ForceRawLeaves { + dagmod.RawLeaves = true + } + + _, err = dagmod.Write(make([]byte, 100)) + if err != nil { + t.Fatal(err) + } + + offset, err := dagmod.Seek(0, io.SeekCurrent) + if err != nil { + t.Fatal(err) + } + if offset != 100 { + t.Fatal("expected the relative seek 0 to return current location") + } + + offset, err = dagmod.Seek(0, io.SeekStart) + if err != nil { + t.Fatal(err) + } + if offset != 0 { + t.Fatal("expected the absolute seek to set offset at 0") + } + + offset, err = dagmod.Seek(0, io.SeekEnd) + if err != nil { + t.Fatal(err) + } + if offset != 100 { + t.Fatal("expected the end seek to set offset at end") + } +} + +func TestReadAndSeek(t *testing.T) { + runAllSubtests(t, testReadAndSeek) +} +func testReadAndSeek(t *testing.T, opts testu.NodeOpts) { + dserv := testu.GetDAGServ() + + n := testu.GetEmptyNode(t, dserv, opts) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + dagmod, err := NewDagModifier(ctx, n, dserv, testu.SizeSplitterGen(512)) + if err != nil { + t.Fatal(err) + } + if opts.ForceRawLeaves { + dagmod.RawLeaves = true + } + + writeBuf := []byte{0, 1, 2, 3, 4, 5, 6, 7} + dagmod.Write(writeBuf) + + if !dagmod.HasChanges() { + t.Fatal("there are changes, this should be true") + } + + readBuf := make([]byte, 4) + offset, err := dagmod.Seek(0, io.SeekStart) + if offset != 0 { + t.Fatal("expected offset to be 0") + } + if err != nil { + t.Fatal(err) + } + + // read 0,1,2,3 + c, err := dagmod.Read(readBuf) + if err != nil { + t.Fatal(err) + } + if c != 4 { + t.Fatalf("expected length of 4 got %d", c) + } + + for i := byte(0); i < 4; i++ { + if readBuf[i] != i { + t.Fatalf("wrong value %d [at index %d]", readBuf[i], i) + } + } + + // skip 4 + _, err = dagmod.Seek(1, io.SeekCurrent) + if err != nil { + t.Fatalf("error: %s, offset %d, reader offset %d", err, dagmod.curWrOff, getOffset(dagmod.read)) + } + + //read 5,6,7 + readBuf = make([]byte, 3) + c, err = dagmod.Read(readBuf) + if err != nil { + t.Fatal(err) + } + if c != 3 { + t.Fatalf("expected length of 3 got %d", c) + } + + for i := byte(0); i < 3; i++ { + if readBuf[i] != i+5 { + t.Fatalf("wrong value %d [at index %d]", readBuf[i], i) + } + + } + +} + +func TestCtxRead(t *testing.T) { + runAllSubtests(t, testCtxRead) +} +func testCtxRead(t *testing.T, opts testu.NodeOpts) { + dserv := testu.GetDAGServ() + + n := testu.GetEmptyNode(t, dserv, opts) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + dagmod, err := NewDagModifier(ctx, n, dserv, testu.SizeSplitterGen(512)) + if err != nil { + t.Fatal(err) + } + if opts.ForceRawLeaves { + dagmod.RawLeaves = true + } + + _, err = dagmod.Write([]byte{0, 1, 2, 3, 4, 5, 6, 7}) + if err != nil { + t.Fatal(err) + } + dagmod.Seek(0, io.SeekStart) + + readBuf := make([]byte, 4) + _, err = dagmod.CtxReadFull(ctx, readBuf) + if err != nil { + t.Fatal(err) + } + err = testu.ArrComp(readBuf, []byte{0, 1, 2, 3}) + if err != nil { + t.Fatal(err) + } + // TODO(Kubuxu): context cancel case, I will do it after I figure out dagreader tests, + // because this is exacelly the same. +} + +func BenchmarkDagmodWrite(b *testing.B) { + b.StopTimer() + dserv := testu.GetDAGServ() + n := testu.GetEmptyNode(b, dserv, testu.UseProtoBufLeaves) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + wrsize := 4096 + + dagmod, err := NewDagModifier(ctx, n, dserv, testu.SizeSplitterGen(512)) + if err != nil { + b.Fatal(err) + } + + buf := make([]byte, b.N*wrsize) + u.NewTimeSeededRand().Read(buf) + b.StartTimer() + b.SetBytes(int64(wrsize)) + for i := 0; i < b.N; i++ { + n, err := dagmod.Write(buf[i*wrsize : (i+1)*wrsize]) + if err != nil { + b.Fatal(err) + } + if n != wrsize { + b.Fatal("Wrote bad size") + } + } +} + +func getOffset(reader uio.DagReader) int64 { + offset, err := reader.Seek(0, io.SeekCurrent) + if err != nil { + panic("failed to retrieve offset: " + err.Error()) + } + return offset +} diff --git a/unixfs/pb/Makefile b/unixfs/pb/Makefile new file mode 100644 index 0000000000..51552a0961 --- /dev/null +++ b/unixfs/pb/Makefile @@ -0,0 +1,11 @@ +PB = $(wildcard *.proto) +GO = $(PB:.proto=.pb.go) + +all: $(GO) + +%.pb.go: %.proto + protoc --proto_path=$(GOPATH)/src:. --gogo_out=. $< + +clean: + rm -f *.pb.go + rm -f *.go diff --git a/unixfs/pb/unixfs.pb.go b/unixfs/pb/unixfs.pb.go new file mode 100644 index 0000000000..e523140075 --- /dev/null +++ b/unixfs/pb/unixfs.pb.go @@ -0,0 +1,218 @@ +// Code generated by protoc-gen-gogo. DO NOT EDIT. +// source: unixfs.proto + +package unixfs_pb + +import ( + fmt "fmt" + math "math" + + proto "github.com/gogo/protobuf/proto" +) + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package + +type Data_DataType int32 + +const ( + Data_Raw Data_DataType = 0 + Data_Directory Data_DataType = 1 + Data_File Data_DataType = 2 + Data_Metadata Data_DataType = 3 + Data_Symlink Data_DataType = 4 + Data_HAMTShard Data_DataType = 5 +) + +var Data_DataType_name = map[int32]string{ + 0: "Raw", + 1: "Directory", + 2: "File", + 3: "Metadata", + 4: "Symlink", + 5: "HAMTShard", +} + +var Data_DataType_value = map[string]int32{ + "Raw": 0, + "Directory": 1, + "File": 2, + "Metadata": 3, + "Symlink": 4, + "HAMTShard": 5, +} + +func (x Data_DataType) Enum() *Data_DataType { + p := new(Data_DataType) + *p = x + return p +} + +func (x Data_DataType) String() string { + return proto.EnumName(Data_DataType_name, int32(x)) +} + +func (x *Data_DataType) UnmarshalJSON(data []byte) error { + value, err := proto.UnmarshalJSONEnum(Data_DataType_value, data, "Data_DataType") + if err != nil { + return err + } + *x = Data_DataType(value) + return nil +} + +func (Data_DataType) EnumDescriptor() ([]byte, []int) { + return fileDescriptor_e2fd76cc44dfc7c3, []int{0, 0} +} + +type Data struct { + Type *Data_DataType `protobuf:"varint,1,req,name=Type,enum=unixfs.pb.Data_DataType" json:"Type,omitempty"` + Data []byte `protobuf:"bytes,2,opt,name=Data" json:"Data,omitempty"` + Filesize *uint64 `protobuf:"varint,3,opt,name=filesize" json:"filesize,omitempty"` + Blocksizes []uint64 `protobuf:"varint,4,rep,name=blocksizes" json:"blocksizes,omitempty"` + HashType *uint64 `protobuf:"varint,5,opt,name=hashType" json:"hashType,omitempty"` + Fanout *uint64 `protobuf:"varint,6,opt,name=fanout" json:"fanout,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *Data) Reset() { *m = Data{} } +func (m *Data) String() string { return proto.CompactTextString(m) } +func (*Data) ProtoMessage() {} +func (*Data) Descriptor() ([]byte, []int) { + return fileDescriptor_e2fd76cc44dfc7c3, []int{0} +} +func (m *Data) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_Data.Unmarshal(m, b) +} +func (m *Data) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_Data.Marshal(b, m, deterministic) +} +func (m *Data) XXX_Merge(src proto.Message) { + xxx_messageInfo_Data.Merge(m, src) +} +func (m *Data) XXX_Size() int { + return xxx_messageInfo_Data.Size(m) +} +func (m *Data) XXX_DiscardUnknown() { + xxx_messageInfo_Data.DiscardUnknown(m) +} + +var xxx_messageInfo_Data proto.InternalMessageInfo + +func (m *Data) GetType() Data_DataType { + if m != nil && m.Type != nil { + return *m.Type + } + return Data_Raw +} + +func (m *Data) GetData() []byte { + if m != nil { + return m.Data + } + return nil +} + +func (m *Data) GetFilesize() uint64 { + if m != nil && m.Filesize != nil { + return *m.Filesize + } + return 0 +} + +func (m *Data) GetBlocksizes() []uint64 { + if m != nil { + return m.Blocksizes + } + return nil +} + +func (m *Data) GetHashType() uint64 { + if m != nil && m.HashType != nil { + return *m.HashType + } + return 0 +} + +func (m *Data) GetFanout() uint64 { + if m != nil && m.Fanout != nil { + return *m.Fanout + } + return 0 +} + +type Metadata struct { + MimeType *string `protobuf:"bytes,1,opt,name=MimeType" json:"MimeType,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *Metadata) Reset() { *m = Metadata{} } +func (m *Metadata) String() string { return proto.CompactTextString(m) } +func (*Metadata) ProtoMessage() {} +func (*Metadata) Descriptor() ([]byte, []int) { + return fileDescriptor_e2fd76cc44dfc7c3, []int{1} +} +func (m *Metadata) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_Metadata.Unmarshal(m, b) +} +func (m *Metadata) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_Metadata.Marshal(b, m, deterministic) +} +func (m *Metadata) XXX_Merge(src proto.Message) { + xxx_messageInfo_Metadata.Merge(m, src) +} +func (m *Metadata) XXX_Size() int { + return xxx_messageInfo_Metadata.Size(m) +} +func (m *Metadata) XXX_DiscardUnknown() { + xxx_messageInfo_Metadata.DiscardUnknown(m) +} + +var xxx_messageInfo_Metadata proto.InternalMessageInfo + +func (m *Metadata) GetMimeType() string { + if m != nil && m.MimeType != nil { + return *m.MimeType + } + return "" +} + +func init() { + proto.RegisterEnum("unixfs.pb.Data_DataType", Data_DataType_name, Data_DataType_value) + proto.RegisterType((*Data)(nil), "unixfs.pb.Data") + proto.RegisterType((*Metadata)(nil), "unixfs.pb.Metadata") +} + +func init() { proto.RegisterFile("unixfs.proto", fileDescriptor_e2fd76cc44dfc7c3) } + +var fileDescriptor_e2fd76cc44dfc7c3 = []byte{ + // 254 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x4c, 0x90, 0xb1, 0x6a, 0xeb, 0x30, + 0x18, 0x85, 0xaf, 0x6c, 0x25, 0xb1, 0xff, 0xeb, 0x16, 0xf1, 0x0f, 0x45, 0x74, 0x28, 0xc6, 0x43, + 0xd1, 0x50, 0x3c, 0xf4, 0x0d, 0x0a, 0xa1, 0x74, 0xf1, 0xa2, 0x84, 0xee, 0x4a, 0x22, 0x63, 0x11, + 0xc7, 0x0a, 0xb6, 0x42, 0xeb, 0x3e, 0x45, 0x1f, 0xb9, 0xc8, 0x8e, 0xdd, 0x2e, 0x82, 0x4f, 0xe7, + 0x7c, 0xe2, 0x20, 0x48, 0x2e, 0x8d, 0xf9, 0x2c, 0xbb, 0xfc, 0xdc, 0x5a, 0x67, 0x31, 0x9e, 0x68, + 0x97, 0x7d, 0x07, 0x40, 0xd7, 0xca, 0x29, 0x7c, 0x02, 0xba, 0xed, 0xcf, 0x9a, 0x93, 0x34, 0x10, + 0xb7, 0xcf, 0x3c, 0x9f, 0x2b, 0xb9, 0x8f, 0x87, 0xc3, 0xe7, 0x72, 0x68, 0x21, 0x8e, 0x16, 0x0f, + 0x52, 0x22, 0x12, 0x39, 0xbe, 0x70, 0x0f, 0x51, 0x69, 0x6a, 0xdd, 0x99, 0x2f, 0xcd, 0xc3, 0x94, + 0x08, 0x2a, 0x67, 0xc6, 0x07, 0x80, 0x5d, 0x6d, 0xf7, 0x47, 0x0f, 0x1d, 0xa7, 0x69, 0x28, 0xa8, + 0xfc, 0x73, 0xe3, 0xdd, 0x4a, 0x75, 0xd5, 0xb0, 0x60, 0x31, 0xba, 0x13, 0xe3, 0x1d, 0x2c, 0x4b, + 0xd5, 0xd8, 0x8b, 0xe3, 0xcb, 0x21, 0xb9, 0x52, 0xf6, 0x0e, 0xd1, 0xb4, 0x0a, 0x57, 0x10, 0x4a, + 0xf5, 0xc1, 0xfe, 0xe1, 0x0d, 0xc4, 0x6b, 0xd3, 0xea, 0xbd, 0xb3, 0x6d, 0xcf, 0x08, 0x46, 0x40, + 0x5f, 0x4d, 0xad, 0x59, 0x80, 0x09, 0x44, 0x85, 0x76, 0xea, 0xa0, 0x9c, 0x62, 0x21, 0xfe, 0x87, + 0xd5, 0xa6, 0x3f, 0xd5, 0xa6, 0x39, 0x32, 0xea, 0x9d, 0xb7, 0x97, 0x62, 0xbb, 0xa9, 0x54, 0x7b, + 0x60, 0x8b, 0xec, 0xf1, 0xb7, 0xe9, 0x77, 0x15, 0xe6, 0xa4, 0xaf, 0x3f, 0x43, 0x44, 0x2c, 0x67, + 0xfe, 0x09, 0x00, 0x00, 0xff, 0xff, 0xe9, 0xa0, 0x51, 0x10, 0x54, 0x01, 0x00, 0x00, +} diff --git a/unixfs/pb/unixfs.proto b/unixfs/pb/unixfs.proto new file mode 100644 index 0000000000..ffc059e8be --- /dev/null +++ b/unixfs/pb/unixfs.proto @@ -0,0 +1,26 @@ +syntax = "proto2"; + +package unixfs.pb; + +message Data { + enum DataType { + Raw = 0; + Directory = 1; + File = 2; + Metadata = 3; + Symlink = 4; + HAMTShard = 5; + } + + required DataType Type = 1; + optional bytes Data = 2; + optional uint64 filesize = 3; + repeated uint64 blocksizes = 4; + + optional uint64 hashType = 5; + optional uint64 fanout = 6; +} + +message Metadata { + optional string MimeType = 1; +} diff --git a/unixfs/private/linksize/linksize.go b/unixfs/private/linksize/linksize.go new file mode 100644 index 0000000000..e7ae098b61 --- /dev/null +++ b/unixfs/private/linksize/linksize.go @@ -0,0 +1,5 @@ +package linksize + +import "github.com/ipfs/go-cid" + +var LinkSizeFunction func(linkName string, linkCid cid.Cid) int diff --git a/unixfs/test/utils.go b/unixfs/test/utils.go new file mode 100644 index 0000000000..b382e59334 --- /dev/null +++ b/unixfs/test/utils.go @@ -0,0 +1,138 @@ +package testu + +import ( + "bytes" + "context" + "fmt" + "io" + "testing" + + ft "github.com/ipfs/go-libipfs/unixfs" + h "github.com/ipfs/go-libipfs/unixfs/importer/helpers" + trickle "github.com/ipfs/go-libipfs/unixfs/importer/trickle" + + cid "github.com/ipfs/go-cid" + ipld "github.com/ipfs/go-ipld-format" + chunker "github.com/ipfs/go-libipfs/chunker" + u "github.com/ipfs/go-libipfs/util" + mdag "github.com/ipfs/go-merkledag" + mdagmock "github.com/ipfs/go-merkledag/test" + mh "github.com/multiformats/go-multihash" +) + +// SizeSplitterGen creates a generator. +func SizeSplitterGen(size int64) chunker.SplitterGen { + return func(r io.Reader) chunker.Splitter { + return chunker.NewSizeSplitter(r, size) + } +} + +// GetDAGServ returns a mock DAGService. +func GetDAGServ() ipld.DAGService { + return mdagmock.Mock() +} + +// NodeOpts is used by GetNode, GetEmptyNode and GetRandomNode +type NodeOpts struct { + Prefix cid.Prefix + // ForceRawLeaves if true will force the use of raw leaves + ForceRawLeaves bool + // RawLeavesUsed is true if raw leaves or either implicitly or explicitly enabled + RawLeavesUsed bool +} + +// Some shorthands for NodeOpts. +var ( + UseProtoBufLeaves = NodeOpts{Prefix: mdag.V0CidPrefix()} + UseRawLeaves = NodeOpts{Prefix: mdag.V0CidPrefix(), ForceRawLeaves: true, RawLeavesUsed: true} + UseCidV1 = NodeOpts{Prefix: mdag.V1CidPrefix(), RawLeavesUsed: true} + UseBlake2b256 NodeOpts +) + +func init() { + UseBlake2b256 = UseCidV1 + UseBlake2b256.Prefix.MhType = mh.Names["blake2b-256"] + UseBlake2b256.Prefix.MhLength = -1 +} + +// GetNode returns a unixfs file node with the specified data. +func GetNode(t testing.TB, dserv ipld.DAGService, data []byte, opts NodeOpts) ipld.Node { + in := bytes.NewReader(data) + + dbp := h.DagBuilderParams{ + Dagserv: dserv, + Maxlinks: h.DefaultLinksPerBlock, + CidBuilder: opts.Prefix, + RawLeaves: opts.RawLeavesUsed, + } + + db, err := dbp.New(SizeSplitterGen(500)(in)) + if err != nil { + t.Fatal(err) + } + node, err := trickle.Layout(db) + if err != nil { + t.Fatal(err) + } + + return node +} + +// GetEmptyNode returns an empty unixfs file node. +func GetEmptyNode(t testing.TB, dserv ipld.DAGService, opts NodeOpts) ipld.Node { + return GetNode(t, dserv, []byte{}, opts) +} + +// GetRandomNode returns a random unixfs file node. +func GetRandomNode(t testing.TB, dserv ipld.DAGService, size int64, opts NodeOpts) ([]byte, ipld.Node) { + in := io.LimitReader(u.NewTimeSeededRand(), size) + buf, err := io.ReadAll(in) + if err != nil { + t.Fatal(err) + } + + node := GetNode(t, dserv, buf, opts) + return buf, node +} + +// ArrComp checks if two byte slices are the same. +func ArrComp(a, b []byte) error { + if len(a) != len(b) { + return fmt.Errorf("arrays differ in length. %d != %d", len(a), len(b)) + } + for i, v := range a { + if v != b[i] { + return fmt.Errorf("arrays differ at index: %d", i) + } + } + return nil +} + +// PrintDag pretty-prints the given dag to stdout. +func PrintDag(nd *mdag.ProtoNode, ds ipld.DAGService, indent int) { + fsn, err := ft.FSNodeFromBytes(nd.Data()) + if err != nil { + panic(err) + } + + for i := 0; i < indent; i++ { + fmt.Print(" ") + } + fmt.Printf("{size = %d, type = %s, children = %d", fsn.FileSize(), fsn.Type().String(), fsn.NumChildren()) + if len(nd.Links()) > 0 { + fmt.Println() + } + for _, lnk := range nd.Links() { + child, err := lnk.GetNode(context.Background(), ds) + if err != nil { + panic(err) + } + PrintDag(child.(*mdag.ProtoNode), ds, indent+1) + } + if len(nd.Links()) > 0 { + for i := 0; i < indent; i++ { + fmt.Print(" ") + } + } + fmt.Println("}") +} diff --git a/unixfs/unixfs.go b/unixfs/unixfs.go new file mode 100644 index 0000000000..0117a310c9 --- /dev/null +++ b/unixfs/unixfs.go @@ -0,0 +1,418 @@ +// Package unixfs implements a data format for files in the IPFS filesystem It +// is not the only format in ipfs, but it is the one that the filesystem +// assumes +package unixfs + +import ( + "errors" + "fmt" + + proto "github.com/gogo/protobuf/proto" + dag "github.com/ipfs/go-merkledag" + + ipld "github.com/ipfs/go-ipld-format" + pb "github.com/ipfs/go-libipfs/unixfs/pb" +) + +// A LinkResult for any parallel enumeration of links +// TODO: Should this live in go-ipld-format? +type LinkResult struct { + Link *ipld.Link + Err error +} + +// Shorthands for protobuffer types +const ( + TRaw = pb.Data_Raw + TFile = pb.Data_File + TDirectory = pb.Data_Directory + TMetadata = pb.Data_Metadata + TSymlink = pb.Data_Symlink + THAMTShard = pb.Data_HAMTShard +) + +// Common errors +var ( + ErrMalformedFileFormat = errors.New("malformed data in file format") + ErrUnrecognizedType = errors.New("unrecognized node type") +) + +// FromBytes unmarshals a byte slice as protobuf Data. +// Deprecated: Use `FSNodeFromBytes` instead to avoid direct manipulation of `pb.Data`. +func FromBytes(data []byte) (*pb.Data, error) { + pbdata := new(pb.Data) + err := proto.Unmarshal(data, pbdata) + if err != nil { + return nil, err + } + return pbdata, nil +} + +// FilePBData creates a protobuf File with the given +// byte slice and returns the marshaled protobuf bytes representing it. +func FilePBData(data []byte, totalsize uint64) []byte { + pbfile := new(pb.Data) + typ := pb.Data_File + pbfile.Type = &typ + pbfile.Data = data + pbfile.Filesize = proto.Uint64(totalsize) + + data, err := proto.Marshal(pbfile) + if err != nil { + // This really shouldnt happen, i promise + // The only failure case for marshal is if required fields + // are not filled out, and they all are. If the proto object + // gets changed and nobody updates this function, the code + // should panic due to programmer error + panic(err) + } + return data +} + +// FolderPBData returns Bytes that represent a Directory. +func FolderPBData() []byte { + pbfile := new(pb.Data) + typ := pb.Data_Directory + pbfile.Type = &typ + + data, err := proto.Marshal(pbfile) + if err != nil { + //this really shouldnt happen, i promise + panic(err) + } + return data +} + +// WrapData marshals raw bytes into a `Data_Raw` type protobuf message. +func WrapData(b []byte) []byte { + pbdata := new(pb.Data) + typ := pb.Data_Raw + pbdata.Data = b + pbdata.Type = &typ + pbdata.Filesize = proto.Uint64(uint64(len(b))) + + out, err := proto.Marshal(pbdata) + if err != nil { + // This shouldnt happen. seriously. + panic(err) + } + + return out +} + +// SymlinkData returns a `Data_Symlink` protobuf message for the path you specify. +func SymlinkData(path string) ([]byte, error) { + pbdata := new(pb.Data) + typ := pb.Data_Symlink + pbdata.Data = []byte(path) + pbdata.Type = &typ + + out, err := proto.Marshal(pbdata) + if err != nil { + return nil, err + } + + return out, nil +} + +// HAMTShardData return a `Data_HAMTShard` protobuf message +func HAMTShardData(data []byte, fanout uint64, hashType uint64) ([]byte, error) { + pbdata := new(pb.Data) + typ := pb.Data_HAMTShard + pbdata.Type = &typ + pbdata.HashType = proto.Uint64(hashType) + pbdata.Data = data + pbdata.Fanout = proto.Uint64(fanout) + + out, err := proto.Marshal(pbdata) + if err != nil { + return nil, err + } + + return out, nil +} + +// UnwrapData unmarshals a protobuf messages and returns the contents. +func UnwrapData(data []byte) ([]byte, error) { + pbdata := new(pb.Data) + err := proto.Unmarshal(data, pbdata) + if err != nil { + return nil, err + } + return pbdata.GetData(), nil +} + +// DataSize returns the size of the contents in protobuf wrapped slice. +// For raw data it simply provides the length of it. For Data_Files, it +// will return the associated filesize. Note that Data_Directories will +// return an error. +func DataSize(data []byte) (uint64, error) { + pbdata := new(pb.Data) + err := proto.Unmarshal(data, pbdata) + if err != nil { + return 0, err + } + return size(pbdata) +} + +func size(pbdata *pb.Data) (uint64, error) { + switch pbdata.GetType() { + case pb.Data_Directory, pb.Data_HAMTShard: + return 0, errors.New("can't get data size of directory") + case pb.Data_File, pb.Data_Raw: + return pbdata.GetFilesize(), nil + case pb.Data_Symlink: + return uint64(len(pbdata.GetData())), nil + default: + return 0, errors.New("unrecognized node data type") + } +} + +// An FSNode represents a filesystem object using the UnixFS specification. +// +// The `NewFSNode` constructor should be used instead of just calling `new(FSNode)` +// to guarantee that the required (`Type` and `Filesize`) fields in the `format` +// structure are initialized before marshaling (in `GetBytes()`). +type FSNode struct { + + // UnixFS format defined as a protocol buffers message. + format pb.Data +} + +// FSNodeFromBytes unmarshal a protobuf message onto an FSNode. +func FSNodeFromBytes(b []byte) (*FSNode, error) { + n := new(FSNode) + err := proto.Unmarshal(b, &n.format) + if err != nil { + return nil, err + } + + return n, nil +} + +// NewFSNode creates a new FSNode structure with the given `dataType`. +// +// It initializes the (required) `Type` field (that doesn't have a `Set()` +// accessor so it must be specified at creation), otherwise the `Marshal()` +// method in `GetBytes()` would fail (`required field "Type" not set`). +// +// It also initializes the `Filesize` pointer field to ensure its value +// is never nil before marshaling, this is not a required field but it is +// done to be backwards compatible with previous `go-ipfs` versions hash. +// (If it wasn't initialized there could be cases where `Filesize` could +// have been left at nil, when the `FSNode` was created but no data or +// child nodes were set to adjust it, as is the case in `NewLeaf()`.) +func NewFSNode(dataType pb.Data_DataType) *FSNode { + n := new(FSNode) + n.format.Type = &dataType + + // Initialize by `Filesize` by updating it with a dummy (zero) value. + n.UpdateFilesize(0) + + return n +} + +// HashType gets hash type of format +func (n *FSNode) HashType() uint64 { + return n.format.GetHashType() +} + +// Fanout gets fanout of format +func (n *FSNode) Fanout() uint64 { + return n.format.GetFanout() +} + +// AddBlockSize adds the size of the next child block of this node +func (n *FSNode) AddBlockSize(s uint64) { + n.UpdateFilesize(int64(s)) + n.format.Blocksizes = append(n.format.Blocksizes, s) +} + +// RemoveBlockSize removes the given child block's size. +func (n *FSNode) RemoveBlockSize(i int) { + n.UpdateFilesize(-int64(n.format.Blocksizes[i])) + n.format.Blocksizes = append(n.format.Blocksizes[:i], n.format.Blocksizes[i+1:]...) +} + +// BlockSize returns the block size indexed by `i`. +// TODO: Evaluate if this function should be bounds checking. +func (n *FSNode) BlockSize(i int) uint64 { + return n.format.Blocksizes[i] +} + +// BlockSizes gets blocksizes of format +func (n *FSNode) BlockSizes() []uint64 { + return n.format.GetBlocksizes() +} + +// RemoveAllBlockSizes removes all the child block sizes of this node. +func (n *FSNode) RemoveAllBlockSizes() { + n.format.Blocksizes = []uint64{} + n.format.Filesize = proto.Uint64(uint64(len(n.Data()))) +} + +// GetBytes marshals this node as a protobuf message. +func (n *FSNode) GetBytes() ([]byte, error) { + return proto.Marshal(&n.format) +} + +// FileSize returns the size of the file. +func (n *FSNode) FileSize() uint64 { + // XXX: This needs to be able to return an error when we don't know the + // size. + size, _ := size(&n.format) + return size +} + +// NumChildren returns the number of child blocks of this node +func (n *FSNode) NumChildren() int { + return len(n.format.Blocksizes) +} + +// Data retrieves the `Data` field from the internal `format`. +func (n *FSNode) Data() []byte { + return n.format.GetData() +} + +// SetData sets the `Data` field from the internal `format` +// updating its `Filesize`. +func (n *FSNode) SetData(newData []byte) { + n.UpdateFilesize(int64(len(newData) - len(n.Data()))) + n.format.Data = newData +} + +// UpdateFilesize updates the `Filesize` field from the internal `format` +// by a signed difference (`filesizeDiff`). +// TODO: Add assert to check for `Filesize` > 0? +func (n *FSNode) UpdateFilesize(filesizeDiff int64) { + n.format.Filesize = proto.Uint64(uint64( + int64(n.format.GetFilesize()) + filesizeDiff)) +} + +// Type retrieves the `Type` field from the internal `format`. +func (n *FSNode) Type() pb.Data_DataType { + return n.format.GetType() +} + +// IsDir checks whether the node represents a directory +func (n *FSNode) IsDir() bool { + switch n.Type() { + case pb.Data_Directory, pb.Data_HAMTShard: + return true + default: + return false + } +} + +// Metadata is used to store additional FSNode information. +type Metadata struct { + MimeType string + Size uint64 +} + +// MetadataFromBytes Unmarshals a protobuf Data message into Metadata. +// The provided slice should have been encoded with BytesForMetadata(). +func MetadataFromBytes(b []byte) (*Metadata, error) { + pbd := new(pb.Data) + err := proto.Unmarshal(b, pbd) + if err != nil { + return nil, err + } + if pbd.GetType() != pb.Data_Metadata { + return nil, errors.New("incorrect node type") + } + + pbm := new(pb.Metadata) + err = proto.Unmarshal(pbd.Data, pbm) + if err != nil { + return nil, err + } + md := new(Metadata) + md.MimeType = pbm.GetMimeType() + return md, nil +} + +// Bytes marshals Metadata as a protobuf message of Metadata type. +func (m *Metadata) Bytes() ([]byte, error) { + pbm := new(pb.Metadata) + pbm.MimeType = &m.MimeType + return proto.Marshal(pbm) +} + +// BytesForMetadata wraps the given Metadata as a profobuf message of Data type, +// setting the DataType to Metadata. The wrapped bytes are itself the +// result of calling m.Bytes(). +func BytesForMetadata(m *Metadata) ([]byte, error) { + pbd := new(pb.Data) + pbd.Filesize = proto.Uint64(m.Size) + typ := pb.Data_Metadata + pbd.Type = &typ + mdd, err := m.Bytes() + if err != nil { + return nil, err + } + + pbd.Data = mdd + return proto.Marshal(pbd) +} + +// EmptyDirNode creates an empty folder Protonode. +func EmptyDirNode() *dag.ProtoNode { + return dag.NodeWithData(FolderPBData()) +} + +// EmptyFileNode creates an empty file Protonode. +func EmptyFileNode() *dag.ProtoNode { + return dag.NodeWithData(FilePBData(nil, 0)) +} + +// ReadUnixFSNodeData extracts the UnixFS data from an IPLD node. +// Raw nodes are (also) processed because they are used as leaf +// nodes containing (only) UnixFS data. +func ReadUnixFSNodeData(node ipld.Node) (data []byte, err error) { + switch node := node.(type) { + + case *dag.ProtoNode: + fsNode, err := FSNodeFromBytes(node.Data()) + if err != nil { + return nil, fmt.Errorf("incorrectly formatted protobuf: %s", err) + } + + switch fsNode.Type() { + case pb.Data_File, pb.Data_Raw: + return fsNode.Data(), nil + // Only leaf nodes (of type `Data_Raw`) contain data but due to a + // bug the `Data_File` type (normally used for internal nodes) is + // also used for leaf nodes, so both types are accepted here + // (see the `balanced` package for more details). + default: + return nil, fmt.Errorf("found %s node in unexpected place", + fsNode.Type().String()) + } + + case *dag.RawNode: + return node.RawData(), nil + + default: + return nil, ErrUnrecognizedType + // TODO: To avoid rewriting the error message, but a different error from + // `unixfs.ErrUnrecognizedType` should be used (defining it in the + // `merkledag` or `go-ipld-format` packages). + } +} + +// Extract the `unixfs.FSNode` from the `ipld.Node` (assuming this +// was implemented by a `mdag.ProtoNode`). +func ExtractFSNode(node ipld.Node) (*FSNode, error) { + protoNode, ok := node.(*dag.ProtoNode) + if !ok { + return nil, errors.New("expected a ProtoNode as internal node") + } + + fsNode, err := FSNodeFromBytes(protoNode.Data()) + if err != nil { + return nil, err + } + + return fsNode, nil +} diff --git a/unixfs/unixfs_test.go b/unixfs/unixfs_test.go new file mode 100644 index 0000000000..6fdd7e5a12 --- /dev/null +++ b/unixfs/unixfs_test.go @@ -0,0 +1,186 @@ +package unixfs + +import ( + "bytes" + "testing" + + proto "github.com/gogo/protobuf/proto" + + pb "github.com/ipfs/go-libipfs/unixfs/pb" +) + +func TestFSNode(t *testing.T) { + fsn := NewFSNode(TFile) + for i := 0; i < 16; i++ { + fsn.AddBlockSize(100) + } + fsn.RemoveBlockSize(15) + + fsn.SetData(make([]byte, 128)) + + b, err := fsn.GetBytes() + if err != nil { + t.Fatal(err) + } + + pbn := new(pb.Data) + err = proto.Unmarshal(b, pbn) + if err != nil { + t.Fatal(err) + } + + ds, err := DataSize(b) + if err != nil { + t.Fatal(err) + } + nKids := fsn.NumChildren() + if nKids != 15 { + t.Fatal("Wrong number of child nodes") + } + + if ds != (100*15)+128 { + t.Fatal("Datasize calculations incorrect!") + } + + nfsn, err := FSNodeFromBytes(b) + if err != nil { + t.Fatal(err) + } + + if nfsn.FileSize() != (100*15)+128 { + t.Fatal("fsNode FileSize calculations incorrect") + } +} + +func TestPBdataTools(t *testing.T) { + raw := []byte{0x00, 0x01, 0x02, 0x17, 0xA1} + rawPB := WrapData(raw) + + pbDataSize, err := DataSize(rawPB) + if err != nil { + t.Fatal(err) + } + + same := len(raw) == int(pbDataSize) + if !same { + t.Fatal("WrapData changes the size of data.") + } + + rawPBBytes, err := UnwrapData(rawPB) + if err != nil { + t.Fatal(err) + } + + same = bytes.Equal(raw, rawPBBytes) + if !same { + t.Fatal("Unwrap failed to produce the correct wrapped data.") + } + + rawPBdata, err := FSNodeFromBytes(rawPB) + if err != nil { + t.Fatal(err) + } + + isRaw := rawPBdata.Type() == TRaw + if !isRaw { + t.Fatal("WrapData does not create pb.Data_Raw!") + } + + catFile := []byte("Mr_Meowgie.gif") + catPBfile := FilePBData(catFile, 17) + catSize, err := DataSize(catPBfile) + if catSize != 17 { + t.Fatal("FilePBData is the wrong size.") + } + if err != nil { + t.Fatal(err) + } + + dirPB := FolderPBData() + dir, err := FSNodeFromBytes(dirPB) + isDir := dir.Type() == TDirectory + if !isDir { + t.Fatal("FolderPBData does not create a directory!") + } + if err != nil { + t.Fatal(err) + } + _, dirErr := DataSize(dirPB) + if dirErr == nil { + t.Fatal("DataSize didn't throw an error when taking the size of a directory.") + } + + catSym, err := SymlinkData("/ipfs/adad123123/meowgie.gif") + if err != nil { + t.Fatal(err) + } + + catSymPB, err := FSNodeFromBytes(catSym) + isSym := catSymPB.Type() == TSymlink + if !isSym { + t.Fatal("Failed to make a Symlink.") + } + if err != nil { + t.Fatal(err) + } +} + +func TestSymlinkFilesize(t *testing.T) { + path := "/ipfs/adad123123/meowgie.gif" + sym, err := SymlinkData(path) + if err != nil { + t.Fatal(err) + } + size, err := DataSize(sym) + if err != nil { + t.Fatal(err) + } + if int(size) != len(path) { + t.Fatalf("size mismatch: %d != %d", size, len(path)) + } +} + +func TestMetadata(t *testing.T) { + meta := &Metadata{ + MimeType: "audio/aiff", + Size: 12345, + } + + _, err := meta.Bytes() + if err != nil { + t.Fatal(err) + } + + metaPB, err := BytesForMetadata(meta) + if err != nil { + t.Fatal(err) + } + + meta, err = MetadataFromBytes(metaPB) + if err != nil { + t.Fatal(err) + } + + mimeAiff := meta.MimeType == "audio/aiff" + if !mimeAiff { + t.Fatal("Metadata does not Marshal and Unmarshal properly!") + } + +} + +func TestIsDir(t *testing.T) { + prepares := map[pb.Data_DataType]bool{ + TDirectory: true, + THAMTShard: true, + TFile: false, + TMetadata: false, + TRaw: false, + TSymlink: false, + } + for typ, v := range prepares { + fsn := NewFSNode(typ) + if fsn.IsDir() != v { + t.Fatalf("type %v, IsDir() should be %v, but %v", typ, v, fsn.IsDir()) + } + } +} diff --git a/util/.gitignore b/util/.gitignore new file mode 100644 index 0000000000..1377554ebe --- /dev/null +++ b/util/.gitignore @@ -0,0 +1 @@ +*.swp diff --git a/util/file.go b/util/file.go new file mode 100644 index 0000000000..e6e30df4d3 --- /dev/null +++ b/util/file.go @@ -0,0 +1,12 @@ +package util + +import "os" + +// FileExists check if the file with the given path exits. +func FileExists(filename string) bool { + fi, err := os.Lstat(filename) + if fi != nil || (err != nil && !os.IsNotExist(err)) { + return true + } + return false +} diff --git a/util/file_test.go b/util/file_test.go new file mode 100644 index 0000000000..040b229270 --- /dev/null +++ b/util/file_test.go @@ -0,0 +1,10 @@ +package util + +import "testing" + +func TestFileDoesNotExist(t *testing.T) { + t.Parallel() + if FileExists("i would be surprised to discover that this file exists") { + t.Fail() + } +} diff --git a/util/time.go b/util/time.go new file mode 100644 index 0000000000..37d720fb1b --- /dev/null +++ b/util/time.go @@ -0,0 +1,22 @@ +package util + +import "time" + +// TimeFormatIpfs is the format ipfs uses to represent time in string form. +var TimeFormatIpfs = time.RFC3339Nano + +// ParseRFC3339 parses an RFC3339Nano-formatted time stamp and +// returns the UTC time. +func ParseRFC3339(s string) (time.Time, error) { + t, err := time.Parse(TimeFormatIpfs, s) + if err != nil { + return time.Time{}, err + } + return t.UTC(), nil +} + +// FormatRFC3339 returns the string representation of the +// UTC value of the given time in RFC3339Nano format. +func FormatRFC3339(t time.Time) string { + return t.UTC().Format(TimeFormatIpfs) +} diff --git a/util/time_test.go b/util/time_test.go new file mode 100644 index 0000000000..b5a98caa62 --- /dev/null +++ b/util/time_test.go @@ -0,0 +1,16 @@ +package util + +import ( + "testing" + "time" +) + +func TestTimeFormatParseInversion(t *testing.T) { + v, err := ParseRFC3339(FormatRFC3339(time.Now())) + if err != nil { + t.Fatal(err) + } + if v.Location() != time.UTC { + t.Fatal("Time should be UTC") + } +} diff --git a/util/util.go b/util/util.go new file mode 100644 index 0000000000..ffcab2f33d --- /dev/null +++ b/util/util.go @@ -0,0 +1,158 @@ +// Package util implements various utility functions used within ipfs +// that do not currently have a better place to live. +package util + +import ( + "errors" + "io" + "math/rand" + "os" + "path/filepath" + "runtime/debug" + "strings" + "time" + + b58 "github.com/mr-tron/base58/base58" + mh "github.com/multiformats/go-multihash" +) + +// DefaultIpfsHash is the current default hash function used by IPFS. +const DefaultIpfsHash = mh.SHA2_256 + +// Debug is a global flag for debugging. +var Debug bool + +// ErrNotImplemented signifies a function has not been implemented yet. +var ErrNotImplemented = errors.New("error: not implemented yet") + +// ErrTimeout implies that a timeout has been triggered +var ErrTimeout = errors.New("error: call timed out") + +// ErrSearchIncomplete implies that a search type operation didn't +// find the expected node, but did find 'a' node. +var ErrSearchIncomplete = errors.New("error: search incomplete") + +// ErrCast is returned when a cast fails AND the program should not panic. +func ErrCast() error { + debug.PrintStack() + return errCast +} + +var errCast = errors.New("cast error") + +// ExpandPathnames takes a set of paths and turns them into absolute paths +func ExpandPathnames(paths []string) ([]string, error) { + var out []string + for _, p := range paths { + abspath, err := filepath.Abs(p) + if err != nil { + return nil, err + } + out = append(out, abspath) + } + return out, nil +} + +type randGen struct { + rand.Rand +} + +// NewTimeSeededRand returns a random bytes reader +// which has been initialized with the current time. +func NewTimeSeededRand() io.Reader { + src := rand.NewSource(time.Now().UnixNano()) + return &randGen{ + Rand: *rand.New(src), + } +} + +// NewSeededRand returns a random bytes reader +// initialized with the given seed. +func NewSeededRand(seed int64) io.Reader { + src := rand.NewSource(seed) + return &randGen{ + Rand: *rand.New(src), + } +} + +func (r *randGen) Read(p []byte) (n int, err error) { + for i := 0; i < len(p); i++ { + p[i] = byte(r.Rand.Intn(255)) + } + return len(p), nil +} + +// GetenvBool is the way to check an env var as a boolean +func GetenvBool(name string) bool { + v := strings.ToLower(os.Getenv(name)) + return v == "true" || v == "t" || v == "1" +} + +// MultiErr is a util to return multiple errors +type MultiErr []error + +func (m MultiErr) Error() string { + if len(m) == 0 { + return "no errors" + } + + s := "Multiple errors: " + for i, e := range m { + if i != 0 { + s += ", " + } + s += e.Error() + } + return s +} + +// Partition splits a subject 3 parts: prefix, separator, suffix. +// The first occurrence of the separator will be matched. +// ie. Partition("Ready, steady, go!", ", ") -> ["Ready", ", ", "steady, go!"] +func Partition(subject string, sep string) (string, string, string) { + if i := strings.Index(subject, sep); i != -1 { + return subject[:i], subject[i : i+len(sep)], subject[i+len(sep):] + } + return subject, "", "" +} + +// RPartition splits a subject 3 parts: prefix, separator, suffix. +// The last occurrence of the separator will be matched. +// ie. RPartition("Ready, steady, go!", ", ") -> ["Ready, steady", ", ", "go!"] +func RPartition(subject string, sep string) (string, string, string) { + if i := strings.LastIndex(subject, sep); i != -1 { + return subject[:i], subject[i : i+len(sep)], subject[i+len(sep):] + } + return subject, "", "" +} + +// Hash is the global IPFS hash function. uses multihash SHA2_256, 256 bits +func Hash(data []byte) mh.Multihash { + h, err := mh.Sum(data, DefaultIpfsHash, -1) + if err != nil { + // this error can be safely ignored (panic) because multihash only fails + // from the selection of hash function. If the fn + length are valid, it + // won't error. + panic("multihash failed to hash using SHA2_256.") + } + return h +} + +// IsValidHash checks whether a given hash is valid (b58 decodable, len > 0) +func IsValidHash(s string) bool { + out, err := b58.Decode(s) + if err != nil { + return false + } + _, err = mh.Cast(out) + return err == nil +} + +// XOR takes two byte slices, XORs them together, returns the resulting slice. +func XOR(a, b []byte) []byte { + c := make([]byte, len(a)) + for i := 0; i < len(a); i++ { + c[i] = a[i] ^ b[i] + } + return c +} diff --git a/util/util_test.go b/util/util_test.go new file mode 100644 index 0000000000..70747ad902 --- /dev/null +++ b/util/util_test.go @@ -0,0 +1,63 @@ +package util + +import ( + "bytes" + "testing" +) + +func TestXOR(t *testing.T) { + cases := [][3][]byte{ + { + {0xFF, 0xFF, 0xFF}, + {0xFF, 0xFF, 0xFF}, + {0x00, 0x00, 0x00}, + }, + { + {0x00, 0xFF, 0x00}, + {0xFF, 0xFF, 0xFF}, + {0xFF, 0x00, 0xFF}, + }, + { + {0x55, 0x55, 0x55}, + {0x55, 0xFF, 0xAA}, + {0x00, 0xAA, 0xFF}, + }, + } + + for _, c := range cases { + r := XOR(c[0], c[1]) + if !bytes.Equal(r, c[2]) { + t.Error("XOR failed") + } + } +} + +func BenchmarkHash256K(b *testing.B) { + buf := make([]byte, 256*1024) + NewTimeSeededRand().Read(buf) + b.SetBytes(int64(256 * 1024)) + b.ResetTimer() + for i := 0; i < b.N; i++ { + Hash(buf) + } +} + +func BenchmarkHash512K(b *testing.B) { + buf := make([]byte, 512*1024) + NewTimeSeededRand().Read(buf) + b.SetBytes(int64(512 * 1024)) + b.ResetTimer() + for i := 0; i < b.N; i++ { + Hash(buf) + } +} + +func BenchmarkHash1M(b *testing.B) { + buf := make([]byte, 1024*1024) + NewTimeSeededRand().Read(buf) + b.SetBytes(int64(1024 * 1024)) + b.ResetTimer() + for i := 0; i < b.N; i++ { + Hash(buf) + } +} diff --git a/verifcid/validate.go b/verifcid/validate.go new file mode 100644 index 0000000000..9164d6d85e --- /dev/null +++ b/verifcid/validate.go @@ -0,0 +1,72 @@ +package verifcid + +import ( + "fmt" + + cid "github.com/ipfs/go-cid" + mh "github.com/multiformats/go-multihash" +) + +var ErrPossiblyInsecureHashFunction = fmt.Errorf("potentially insecure hash functions not allowed") +var ErrBelowMinimumHashLength = fmt.Errorf("hashes must be at least %d bytes long", minimumHashLength) +var ErrAboveMaximumHashLength = fmt.Errorf("hashes must be at most %d bytes long", maximumHashLength) + +const minimumHashLength = 20 +const maximumHashLength = 128 + +var goodset = map[uint64]bool{ + mh.SHA2_256: true, + mh.SHA2_512: true, + mh.SHA3_224: true, + mh.SHA3_256: true, + mh.SHA3_384: true, + mh.SHA3_512: true, + mh.SHAKE_256: true, + mh.DBL_SHA2_256: true, + mh.KECCAK_224: true, + mh.KECCAK_256: true, + mh.KECCAK_384: true, + mh.KECCAK_512: true, + mh.BLAKE3: true, + mh.IDENTITY: true, + mh.POSEIDON_BLS12_381_A1_FC1: true, + mh.SHA2_256_TRUNC254_PADDED: true, + mh.X11: true, + + mh.SHA1: true, // not really secure but still useful +} + +func IsGoodHash(code uint64) bool { + good, found := goodset[code] + if good { + return true + } + + if !found { + if code >= mh.BLAKE2B_MIN+19 && code <= mh.BLAKE2B_MAX { + return true + } + if code >= mh.BLAKE2S_MIN+19 && code <= mh.BLAKE2S_MAX { + return true + } + } + + return false +} + +func ValidateCid(c cid.Cid) error { + pref := c.Prefix() + if !IsGoodHash(pref.MhType) { + return ErrPossiblyInsecureHashFunction + } + + if pref.MhType != mh.IDENTITY && pref.MhLength < minimumHashLength { + return ErrBelowMinimumHashLength + } + + if pref.MhType != mh.IDENTITY && pref.MhLength > maximumHashLength { + return ErrAboveMaximumHashLength + } + + return nil +} diff --git a/verifcid/validate_test.go b/verifcid/validate_test.go new file mode 100644 index 0000000000..5129b861af --- /dev/null +++ b/verifcid/validate_test.go @@ -0,0 +1,70 @@ +package verifcid + +import ( + "testing" + + mh "github.com/multiformats/go-multihash" + + cid "github.com/ipfs/go-cid" +) + +func TestValidateCids(t *testing.T) { + assertTrue := func(v bool) { + t.Helper() + if !v { + t.Fatal("expected success") + } + } + assertFalse := func(v bool) { + t.Helper() + if v { + t.Fatal("expected failure") + } + } + + assertTrue(IsGoodHash(mh.SHA2_256)) + assertTrue(IsGoodHash(mh.BLAKE2B_MIN + 32)) + assertTrue(IsGoodHash(mh.DBL_SHA2_256)) + assertTrue(IsGoodHash(mh.KECCAK_256)) + assertTrue(IsGoodHash(mh.SHA3)) + + assertTrue(IsGoodHash(mh.SHA1)) + + assertFalse(IsGoodHash(mh.BLAKE2B_MIN + 5)) + + mhcid := func(code uint64, length int) cid.Cid { + mhash, err := mh.Sum([]byte{}, code, length) + if err != nil { + t.Fatalf("%v: code: %x length: %d", err, code, length) + } + return cid.NewCidV1(cid.DagCBOR, mhash) + } + + cases := []struct { + cid cid.Cid + err error + }{ + {mhcid(mh.SHA2_256, 32), nil}, + {mhcid(mh.SHA2_256, 16), ErrBelowMinimumHashLength}, + {mhcid(mh.MURMUR3X64_64, 4), ErrPossiblyInsecureHashFunction}, + {mhcid(mh.BLAKE3, 32), nil}, + {mhcid(mh.BLAKE3, 69), nil}, + {mhcid(mh.BLAKE3, 128), nil}, + } + + for i, cas := range cases { + if ValidateCid(cas.cid) != cas.err { + t.Errorf("wrong result in case of %s (index %d). Expected: %s, got %s", + cas.cid, i, cas.err, ValidateCid(cas.cid)) + } + } + + longBlake3Hex := "1e810104e0bb39f30b1a3feb89f536c93be15055482df748674b00d26e5a75777702e9791074b7511b59d31c71c62f5a745689fa6c9497f68bdf1061fe07f518d410c0b0c27f41b3cf083f8a7fdc67a877e21790515762a754a45dcb8a356722698a7af5ed2bb608983d5aa75d4d61691ef132efe8631ce0afc15553a08fffc60ee9369b" + longBlake3Mh, err := mh.FromHexString(longBlake3Hex) + if err != nil { + t.Fatalf("failed to produce a multihash from the long blake3 hash: %v", err) + } + if ValidateCid(cid.NewCidV1(cid.DagCBOR, longBlake3Mh)) != ErrAboveMaximumHashLength { + t.Errorf("a CID that was longer than the maximum hash length did not error with ErrAboveMaximumHashLength") + } +}