diff --git a/.circleci/config.yml b/.circleci/config.yml index 35dd7716d..f1a6b27bd 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -325,17 +325,21 @@ workflows: suite: itest-markets_v1_deal target: "./itests/markets_v1_deal_test.go" - - test: - name: test-itest-markets_v1_retrieval - suite: itest-markets_v1_retrieval - target: "./itests/markets_v1_retrieval_test.go" + name: test-itest-markets_v1_identity_cid + suite: itest-markets_v1_identity_cid + target: "./itests/markets_v1_identity_cid_test.go" - test: name: test-itest-markets_v1_offline_deal suite: itest-markets_v1_offline_deal target: "./itests/markets_v1_offline_deal_test.go" + - test: + name: test-itest-markets_v1_retrieval + suite: itest-markets_v1_retrieval + target: "./itests/markets_v1_retrieval_test.go" + - test: name: test-itest-disabled_markets_v1_deal suite: itest-disabled_markets_v1_deal diff --git a/go.mod b/go.mod index e25cbbd09..f3387fefc 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ replace github.com/filecoin-project/filecoin-ffi => ./extern/filecoin-ffi replace github.com/filecoin-project/boostd-data => ./extern/boostd-data -// replace github.com/filcoin-project/boost-graphsync => ../boost-graphsync +// replace github.com/filecoin-project/boost-graphsync => ../boost-graphsync // replace github.com/filecoin-project/go-data-transfer => ../go-data-transfer @@ -51,7 +51,7 @@ require ( github.com/ipfs/go-cid v0.4.1 github.com/ipfs/go-cidutil v0.1.0 github.com/ipfs/go-datastore v0.6.0 - github.com/ipfs/go-graphsync v0.14.8 + github.com/ipfs/go-graphsync v0.14.10 github.com/ipfs/go-ipfs-blockstore v1.3.0 // indirect github.com/ipfs/go-ipfs-blocksutil v0.0.1 github.com/ipfs/go-ipfs-chunker v0.0.5 // indirect @@ -334,7 +334,7 @@ require ( require ( github.com/filecoin-project/boost-gfm v1.26.7 - github.com/filecoin-project/boost-graphsync v0.13.8 + github.com/filecoin-project/boost-graphsync v0.13.9 github.com/filecoin-project/go-data-transfer/v2 v2.0.0-rc7 github.com/filecoin-project/go-fil-markets v1.28.3 github.com/filecoin-project/lotus v1.23.2-0.20230622154405-168d022018ce diff --git a/go.sum b/go.sum index 7478462ea..519f4196f 100644 --- a/go.sum +++ b/go.sum @@ -309,8 +309,8 @@ github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYF github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk= github.com/filecoin-project/boost-gfm v1.26.7 h1:ENJEqx1OzY072QnUP37YrGVmUiCewRwHAjbtTxyW74Y= github.com/filecoin-project/boost-gfm v1.26.7/go.mod h1:OhG2y7WeDx3KU9DPjgWllS+3/ospPjm8/XDrvN6uOfk= -github.com/filecoin-project/boost-graphsync v0.13.8 h1:QqH15/qe4IUnQj5pc1WIuBDe2gq0kzWQ6l4dUirbmfs= -github.com/filecoin-project/boost-graphsync v0.13.8/go.mod h1:bc2M5ZLZJtXHl8kjnqtn4L1MsdEqpJErDaIeY0bJ9wk= +github.com/filecoin-project/boost-graphsync v0.13.9 h1:RQepfTlffLGUmp3Ff7VosYrWUKPLiz++GGV2D/gIfuw= +github.com/filecoin-project/boost-graphsync v0.13.9/go.mod h1:bc2M5ZLZJtXHl8kjnqtn4L1MsdEqpJErDaIeY0bJ9wk= github.com/filecoin-project/dagstore v0.7.0 h1:IS0R+69za8dguYWeqz/MI+nb7ONpk03tAkxPCBXEKm0= github.com/filecoin-project/dagstore v0.7.0/go.mod h1:YKn4qXih+/2xQWpfJsaKGOi4POw5vH5grDmfPCCnx8g= github.com/filecoin-project/go-address v0.0.3/go.mod h1:jr8JxKsYx+lQlQZmF5i2U0Z+cGQ59wMIps/8YW/lDj8= @@ -772,8 +772,8 @@ github.com/ipfs/go-ds-measure v0.2.0/go.mod h1:SEUD/rE2PwRa4IQEC5FuNAmjJCyYObZr9 github.com/ipfs/go-fs-lock v0.0.6/go.mod h1:OTR+Rj9sHiRubJh3dRhD15Juhd/+w6VPOY28L7zESmM= github.com/ipfs/go-fs-lock v0.0.7 h1:6BR3dajORFrFTkb5EpCUFIAypsoxpGpDSVUdFwzgL9U= github.com/ipfs/go-fs-lock v0.0.7/go.mod h1:Js8ka+FNYmgQRLrRXzU3CB/+Csr1BwrRilEcvYrHhhc= -github.com/ipfs/go-graphsync v0.14.8 h1:NFFHquTNnwPi05tJhdpPj4CJMnqRBLxpZd+IfPRauf4= -github.com/ipfs/go-graphsync v0.14.8/go.mod h1:qyHjUvHey6EfKUDMQPwCuVkMOurRG3hcjRm+FaVP6bE= +github.com/ipfs/go-graphsync v0.14.10 h1:2slsmWcHxXGS/T6HR5hilo2sSAX1VsZVxjoOzA3XCAQ= +github.com/ipfs/go-graphsync v0.14.10/go.mod h1:qyHjUvHey6EfKUDMQPwCuVkMOurRG3hcjRm+FaVP6bE= github.com/ipfs/go-ipfs-blockstore v0.0.1/go.mod h1:d3WClOmRQKFnJ0Jz/jj/zmksX0ma1gROTlovZKBmN08= github.com/ipfs/go-ipfs-blockstore v0.1.0/go.mod h1:5aD0AvHPi7mZc6Ci1WCAhiBQu2IsfTduLl+422H6Rqw= github.com/ipfs/go-ipfs-blockstore v0.2.1/go.mod h1:jGesd8EtCM3/zPgx+qr0/feTXGUeRai6adgwC+Q+JvE= diff --git a/itests/markets_v1_identity_cid_test.go b/itests/markets_v1_identity_cid_test.go new file mode 100644 index 000000000..d1f0376e7 --- /dev/null +++ b/itests/markets_v1_identity_cid_test.go @@ -0,0 +1,141 @@ +package itests + +import ( + "context" + "fmt" + "math/rand" + "os" + "testing" + + gstestutil "github.com/filecoin-project/boost-graphsync/testutil" + "github.com/filecoin-project/boost/itests/framework" + "github.com/filecoin-project/boost/testutil" + lapi "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/itests/kit" + "github.com/ipfs/go-cid" + "github.com/ipld/go-car/v2" + "github.com/ipld/go-car/v2/storage" + "github.com/ipld/go-ipld-prime" + "github.com/ipld/go-ipld-prime/datamodel" + "github.com/ipld/go-ipld-prime/fluent/qp" + cidlink "github.com/ipld/go-ipld-prime/linking/cid" + "github.com/ipld/go-ipld-prime/node/basicnode" + selectorparse "github.com/ipld/go-ipld-prime/traversal/selector/parse" + multihash "github.com/multiformats/go-multihash" + "github.com/stretchr/testify/require" +) + +func TestMarketsV1DealAndRetrievalWithIdentityCID(t *testing.T) { + req := require.New(t) + ctx := context.Background() + log := framework.Log + + kit.QuietMiningLogs() + framework.SetLogLevel() + var opts []framework.FrameworkOpts + opts = append(opts, framework.EnableLegacyDeals(true)) + f := framework.NewTestFramework(ctx, t, opts...) + err := f.Start() + req.NoError(err) + defer f.Stop() + + // Create a CAR file + carPath := f.HomeDir + "/testfile.car" + log.Debugf("using test car %s", carPath) + carFile, err := os.Create(carPath) + req.NoError(err) + + carStore, err := storage.NewReadableWritable(carFile, []cid.Cid{cid.MustParse("bafyreiaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")}, car.WriteAsCarV1(true)) + req.NoError(err) + + lsys := cidlink.DefaultLinkSystem() + lsys.SetWriteStorage(carStore) + lsys.SetReadStorage(carStore) + + // Setup our DAG with an identity CID in the middle + testDag := gstestutil.SetupIdentityDAG(ctx, t, lsys) + expectedCids := make([]cid.Cid, 0) + + // Fill out the deal with a bunch of random data so we're not trying to do a + // tiny storage deal. + rseed := 1 + rnd := rand.New(rand.NewSource(int64(rseed))) + n, err := qp.BuildList(basicnode.Prototype.List, 8, func(la ipld.ListAssembler) { + for ii := 0; ii < 7; ii++ { + rndBytes := make([]byte, 1<<20) + _, err := rnd.Read(rndBytes) + req.NoError(err) + l, err := lsys.Store(ipld.LinkContext{}, rawlp, basicnode.NewBytes(rndBytes)) + req.NoError(err) + qp.ListEntry(la, qp.Link(l)) + expectedCids = append(expectedCids, l.(cidlink.Link).Cid) + } + qp.ListEntry(la, qp.Link(testDag.RootLink)) + }) + req.NoError(err) + expectedCids = append(expectedCids, toCids(testDag.AllLinksNoIdent)...) // all blocks BUT the identity block + rootLink, err := lsys.Store(ipld.LinkContext{}, dclp, n) + req.NoError(err) + expectedCids = append([]cid.Cid{rootLink.(cidlink.Link).Cid}, expectedCids...) + + req.NoError(carFile.Close()) + req.NoError(car.ReplaceRootsInFile(carPath, []cid.Cid{rootLink.(cidlink.Link).Cid})) + log.Debugw("filled car, replaced root with correct root", "root", rootLink.String()) + + // Import and make a deal to store + + res, err := f.FullNode.ClientImport(ctx, lapi.FileRef{Path: carPath, IsCAR: true}) + req.NoError(err) + + log.Debugw("imported data for deal") + + dp := f.DefaultMarketsV1DealParams() + dp.Data.Root = res.Root + + log.Debugw("starting deal", "root", res.Root) + dealProposalCid, err := f.FullNode.ClientStartDeal(ctx, &dp) + req.NoError(err) + + log.Debugw("got deal proposal cid", "cid", dealProposalCid) + + err = f.WaitDealSealed(ctx, dealProposalCid) + req.NoError(err) + + // Deal is stored and sealed, attempt different retrieval forms + + log.Debugw("deal is sealed, starting retrieval", "cid", dealProposalCid, "root", res.Root) + outPath := f.Retrieve(ctx, t, dealProposalCid, rootLink.(cidlink.Link).Cid, false, selectorparse.CommonSelector_ExploreAllRecursively) + + // Inspect what we got + gotCids, err := testutil.CidsInCar(outPath) + req.NoError(err) + req.Equal(toStr(expectedCids), toStr(gotCids)) + log.Debugw("successfully retrieved %d blocks, not including identity block", len(expectedCids)) +} + +func toCids(links []datamodel.Link) []cid.Cid { + var cids []cid.Cid + for _, link := range links { + cids = append(cids, link.(cidlink.Link).Cid) + fmt.Println("link.(cidlink.Link).Cid", link.(cidlink.Link).Cid) + } + return cids +} + +var rawlp = cidlink.LinkPrototype{ + Prefix: cid.Prefix{ + Version: 1, + Codec: cid.Raw, + MhType: multihash.SHA2_256, + MhLength: 32, + }, +} + +var dclp = cidlink.LinkPrototype{ + Prefix: cid.Prefix{ + Version: 1, + Codec: cid.DagCBOR, + MhType: multihash.SHA2_256, + MhLength: 32, + }, +} diff --git a/itests/markets_v1_offline_deal_test.go b/itests/markets_v1_offline_deal_test.go index 32859a223..45c21cbf2 100644 --- a/itests/markets_v1_offline_deal_test.go +++ b/itests/markets_v1_offline_deal_test.go @@ -10,7 +10,6 @@ import ( "github.com/filecoin-project/boost-gfm/storagemarket" "github.com/filecoin-project/boost/itests/framework" "github.com/filecoin-project/boost/testutil" - "github.com/filecoin-project/lotus/api" lapi "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/itests/kit" "github.com/stretchr/testify/require" @@ -69,7 +68,7 @@ func TestMarketsV1OfflineDeal(t *testing.T) { // Create a CAR file from the raw file log.Debugw("generate out.car for miner") carFilePath := filepath.Join(f.HomeDir, "out.car") - err = f.FullNode.ClientGenCar(ctx, api.FileRef{Path: inPath}, carFilePath) + err = f.FullNode.ClientGenCar(ctx, lapi.FileRef{Path: inPath}, carFilePath) require.NoError(t, err) // Import the CAR file on the miner - this is the equivalent to diff --git a/retrievalmarket/server/gsunpaidretrieval_test.go b/retrievalmarket/server/gsunpaidretrieval_test.go index 42b5acfb2..ca92f338c 100644 --- a/retrievalmarket/server/gsunpaidretrieval_test.go +++ b/retrievalmarket/server/gsunpaidretrieval_test.go @@ -62,8 +62,7 @@ func TestGS(t *testing.T) { _ = logging.SetLogLevel("testgs", "info") //_ = logging.SetLogLevel("dt-impl", "debug") - missingCid, err := cid.Parse("bafkqaaa") - require.NoError(t, err) + missingCid := cid.MustParse("baguqeeraaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa") testCases := []testCase{{ name: "happy path",