Skip to content

Commit edab3de

Browse files
LexLuthrLexLuthr
authored
fix: allow client to set fast-retrieval (#1004)
* allow client to set fast-retrieval * cbor gen * db migration * fix db tests, add check in prov test * fix db label test
1 parent f3fd05a commit edab3de

File tree

18 files changed

+161
-51
lines changed

18 files changed

+161
-51
lines changed

‎api/api_full.go‎

Lines changed: 0 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@ package api
22

33
import (
44
"context"
5-
"encoding/json"
65
"fmt"
76
"time"
87

@@ -297,34 +296,6 @@ type MethodCall struct {
297296
Error string
298297
}
299298

300-
type StartDealParams struct {
301-
Data *storagemarket.DataRef
302-
Wallet address.Address
303-
Miner address.Address
304-
EpochPrice types.BigInt
305-
MinBlocksDuration uint64
306-
ProviderCollateral big.Int
307-
DealStartEpoch abi.ChainEpoch
308-
FastRetrieval bool
309-
VerifiedDeal bool
310-
}
311-
312-
func (s *StartDealParams) UnmarshalJSON(raw []byte) (err error) {
313-
type sdpAlias StartDealParams
314-
315-
sdp := sdpAlias{
316-
FastRetrieval: true,
317-
}
318-
319-
if err := json.Unmarshal(raw, &sdp); err != nil {
320-
return err
321-
}
322-
323-
*s = StartDealParams(sdp)
324-
325-
return nil
326-
}
327-
328299
type IpldObject struct {
329300
Cid cid.Cid
330301
Obj interface{}

‎build/openrpc/boost.json.gz‎

11 Bytes
Binary file not shown.

‎cmd/boost/deal_cmd.go‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -54,8 +54,8 @@ var dealFlags = []cli.Flag{
5454
Required: true,
5555
},
5656
&cli.IntFlag{
57-
Name: "start-epoch",
58-
Usage: "start epoch by when the deal should be proved by provider on-chain",
57+
Name: "start-epoch",
58+
Usage: "start epoch by when the deal should be proved by provider on-chain",
5959
DefaultText: "current chain head + 2 days",
6060
},
6161
&cli.IntFlag{
@@ -77,6 +77,11 @@ var dealFlags = []cli.Flag{
7777
Usage: "whether the deal funds should come from verified client data-cap",
7878
Value: true,
7979
},
80+
&cli.BoolFlag{
81+
Name: "fast-retrieval",
82+
Usage: "indicates that data should be available for fast retrieval",
83+
Value: true,
84+
},
8085
&cli.StringFlag{
8186
Name: "wallet",
8287
Usage: "wallet address to be used to initiate the deal",
@@ -251,6 +256,7 @@ func dealCmdAction(cctx *cli.Context, isOnline bool) error {
251256
DealDataRoot: rootCid,
252257
IsOffline: !isOnline,
253258
Transfer: transfer,
259+
FastRetrieval: cctx.Bool("fast-retrieval"),
254260
}
255261

256262
log.Debugw("about to submit deal proposal", "uuid", dealUuid.String())

‎db/deals.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,7 @@ func newDealAccessor(db *sql.DB, deal *types.ProviderDealState) *dealAccessor {
7878
"CheckpointAt": &fielddef.FieldDef{F: &deal.CheckpointAt},
7979
"Error": &fielddef.FieldDef{F: &deal.Err},
8080
"Retry": &fielddef.FieldDef{F: &deal.Retry},
81+
"FastRetrieval": &fielddef.FieldDef{F: &deal.FastRetrieval},
8182

8283
// Needed so the deal can be looked up by signed proposal cid
8384
"SignedProposalCID": &fielddef.SignedPropFieldDef{Prop: deal.ClientDealProposal},

‎db/fixtures.go‎

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -84,14 +84,15 @@ func GenerateNDeals(count int) ([]types.ProviderDealState, error) {
8484
Params: []byte(fmt.Sprintf(`{"url":"http://files.org/file%d.car"}`, rand.Intn(1000))),
8585
Size: uint64(rand.Intn(10000)),
8686
},
87-
ChainDealID: abi.DealID(rand.Intn(10000)),
88-
PublishCID: &publishCid,
89-
SectorID: abi.SectorNumber(rand.Intn(10000)),
90-
Offset: abi.PaddedPieceSize(rand.Intn(1000000)),
91-
Length: abi.PaddedPieceSize(rand.Intn(1000000)),
92-
Checkpoint: dealcheckpoints.Accepted,
93-
Retry: types.DealRetryAuto,
94-
Err: dealErr,
87+
ChainDealID: abi.DealID(rand.Intn(10000)),
88+
PublishCID: &publishCid,
89+
SectorID: abi.SectorNumber(rand.Intn(10000)),
90+
Offset: abi.PaddedPieceSize(rand.Intn(1000000)),
91+
Length: abi.PaddedPieceSize(rand.Intn(1000000)),
92+
Checkpoint: dealcheckpoints.Accepted,
93+
Retry: types.DealRetryAuto,
94+
Err: dealErr,
95+
FastRetrieval: false,
9596
}
9697

9798
deals = append(deals, deal)

‎db/migrations/20221124191002_deals_add_fast_retrieval.sql‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
-- +goose Up
2+
-- +goose StatementBegin
3+
ALTER TABLE Deals
4+
ADD FastRetrieval BOOL;
5+
-- +goose StatementEnd
6+
7+
-- +goose Down
8+
-- +goose StatementBegin
9+
SELECT 'down SQL query';
10+
-- +goose StatementEnd

‎db/migrations/20221124191256_deals_set_fast_retrieval.go‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
package migrations
2+
3+
import (
4+
"database/sql"
5+
6+
"github.com/pressly/goose/v3"
7+
)
8+
9+
func init() {
10+
goose.AddMigration(upSetdealsfastretrieval, downSetdealsfastretrieval)
11+
}
12+
13+
func upSetdealsfastretrieval(tx *sql.Tx) error {
14+
_, err := tx.Exec("UPDATE Deals SET FastRetrieval=?;", true)
15+
if err != nil {
16+
return err
17+
}
18+
return nil
19+
}
20+
21+
func downSetdealsfastretrieval(tx *sql.Tx) error {
22+
// This code is executed when the migration is rolled back.
23+
return nil
24+
}

‎db/migrations_tests/deals_fast_retrieval_test.go‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
package migrations_tests
2+
3+
import (
4+
"context"
5+
"testing"
6+
7+
"github.com/filecoin-project/boost/db"
8+
"github.com/filecoin-project/boost/db/migrations"
9+
"github.com/pressly/goose/v3"
10+
"github.com/stretchr/testify/require"
11+
)
12+
13+
func TestDealFastRetrieval(t *testing.T) {
14+
req := require.New(t)
15+
ctx := context.Background()
16+
17+
sqldb := db.CreateTestTmpDB(t)
18+
req.NoError(db.CreateAllBoostTables(ctx, sqldb, sqldb))
19+
20+
// Run migrations up to the one that adds the FastRetrieval field to Deals
21+
goose.SetBaseFS(migrations.EmbedMigrations)
22+
req.NoError(goose.SetDialect("sqlite3"))
23+
req.NoError(goose.UpTo(sqldb, ".", 20221124191002))
24+
25+
// Generate 2 deals
26+
dealsDB := db.NewDealsDB(sqldb)
27+
deals, err := db.GenerateNDeals(1)
28+
req.NoError(err)
29+
30+
// Insert the deals in DB
31+
err = dealsDB.Insert(ctx, &deals[0])
32+
require.NoError(t, err)
33+
34+
// Get deal state
35+
dealState, err := dealsDB.ByID(ctx, deals[0].DealUuid)
36+
require.NoError(t, err)
37+
require.False(t, dealState.FastRetrieval)
38+
39+
//Run migration
40+
req.NoError(goose.UpByOne(sqldb, "."))
41+
42+
// Check the deal state again
43+
dealState, err = dealsDB.ByID(ctx, deals[0].DealUuid)
44+
require.NoError(t, err)
45+
require.True(t, dealState.FastRetrieval)
46+
}

‎db/migrations_tests/storagetagged_set_host_test.go‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,12 +3,13 @@ package migrations_tests
33
import (
44
"context"
55
"fmt"
6+
"testing"
7+
68
"github.com/filecoin-project/boost/db"
79
"github.com/filecoin-project/boost/db/migrations"
810
"github.com/filecoin-project/boost/storagemarket/types"
911
"github.com/pressly/goose/v3"
1012
"github.com/stretchr/testify/require"
11-
"testing"
1213
)
1314

1415
func TestStorageTaggedSetHost(t *testing.T) {
@@ -25,6 +26,11 @@ func TestStorageTaggedSetHost(t *testing.T) {
2526

2627
// Generate 2 deals
2728
dealsDB := db.NewDealsDB(sqldb)
29+
30+
// Add FastRetrieval to allow tests to works
31+
_, err := sqldb.Exec(`ALTER TABLE Deals ADD FastRetrieval BOOL`)
32+
require.NoError(t, err)
33+
2834
deals, err := db.GenerateNDeals(2)
2935
req.NoError(err)
3036

‎documentation/en/api-v1-methods.md‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -397,7 +397,8 @@ Response:
397397
"CheckpointAt": "0001-01-01T00:00:00Z",
398398
"Err": "string value",
399399
"Retry": "auto",
400-
"NBytesReceived": 9
400+
"NBytesReceived": 9,
401+
"FastRetrieval": true
401402
}
402403
```
403404

@@ -462,7 +463,8 @@ Response:
462463
"CheckpointAt": "0001-01-01T00:00:00Z",
463464
"Err": "string value",
464465
"Retry": "auto",
465-
"NBytesReceived": 9
466+
"NBytesReceived": 9,
467+
"FastRetrieval": true
466468
}
467469
```
468470

@@ -506,7 +508,8 @@ Inputs:
506508
"ClientID": "string value",
507509
"Params": "Ynl0ZSBhcnJheQ==",
508510
"Size": 42
509-
}
511+
},
512+
"FastRetrieval": true
510513
}
511514
]
512515
```

0 commit comments

Comments
 (0)