diff --git a/cfg.go b/cfg.go index 56e2b73..98d509e 100644 --- a/cfg.go +++ b/cfg.go @@ -32,7 +32,7 @@ type Option func(cfg *Config) error // NewSDK constructs a new client from the Config. // // This function consumes the config. Do not reuse it (really!). -func (cfg *Config) NewSDK(ctx context.Context) (*chain.ChainClient, error) { +func (cfg *Config) NewSDK(ctx context.Context) (chain.Chainer, error) { if cfg.Name == "" { cfg.Name = config.CharacterName_Default } diff --git a/chain/chain.go b/chain/chain.go index 1eeadcf..588e546 100755 --- a/chain/chain.go +++ b/chain/chain.go @@ -60,7 +60,7 @@ var _ Chainer = (*ChainClient)(nil) // Return: // - *ChainClient: chain client // - error: error message -func NewChainClientUnconnectedRpc(ctx context.Context, name string, rpcs []string, mnemonic string, t time.Duration) (*ChainClient, error) { +func NewChainClientUnconnectedRpc(ctx context.Context, name string, rpcs []string, mnemonic string, t time.Duration) (Chainer, error) { var err error var chainClient = &ChainClient{ lock: new(sync.Mutex), @@ -95,7 +95,7 @@ func NewChainClientUnconnectedRpc(ctx context.Context, name string, rpcs []strin // Return: // - *ChainClient: chain client // - error: error message -func NewChainClient(ctx context.Context, name string, rpcs []string, mnemonic string, t time.Duration) (*ChainClient, error) { +func NewChainClient(ctx context.Context, name string, rpcs []string, mnemonic string, t time.Duration) (Chainer, error) { var ( err error chainClient = &ChainClient{ diff --git a/chain/chainer.go b/chain/chainer.go index a0089d6..03dd05c 100755 --- a/chain/chainer.go +++ b/chain/chainer.go @@ -20,6 +20,8 @@ type Chainer interface { QueryCountedServiceFailed(accountID []byte, block int32) (uint32, error) SubmitIdleProof(idleProof []types.U8) (string, error) SubmitServiceProof(serviceProof []types.U8) (string, error) + SubmitVerifyIdleResult(totalProofHash []types.U8, front, rear types.U64, accumulator Accumulator, result types.Bool, sig types.Bytes, teePuk WorkerPublicKey) (string, error) + SubmitVerifyServiceResult(result types.Bool, sign types.Bytes, bloomFilter BloomFilter, teePuk WorkerPublicKey) (string, error) // Babe QueryAuthorities(block int32) ([]ConsensusRrscAppPublic, error) @@ -92,10 +94,10 @@ type Chainer interface { MinerWithdraw() (string, error) ReceiveReward() (string, string, error) RegisterPoisKey(poisKey PoISKeyInfo, teeSignWithAcc, teeSign types.Bytes, teePuk WorkerPublicKey) (string, error) - RegnstkSminer(earnings string, peerId []byte, staking uint64, tibCount uint32) (string, error) + RegnstkSminer(earnings string, addr []byte, staking uint64, tibCount uint32) (string, error) RegnstkAssignStaking(earnings string, peerId []byte, stakingAcc string, tibCount uint32) (string, error) UpdateBeneficiary(earnings string) (string, error) - UpdateSminerPeerId(peerid PeerId) (string, error) + UpdateSminerAddr(addr []byte) (string, error) // Staking QueryCounterForValidators(block int32) (uint32, error) diff --git a/chain/deoss.go b/chain/deoss.go index fd5567a..caeed58 100755 --- a/chain/deoss.go +++ b/chain/deoss.go @@ -13,7 +13,6 @@ import ( "strings" "time" - "github.com/CESSProject/cess-go-sdk/config" "github.com/CESSProject/cess-go-sdk/utils" "github.com/centrifuge/go-substrate-rpc-client/v4/types" "github.com/centrifuge/go-substrate-rpc-client/v4/types/codec" @@ -523,8 +522,8 @@ func (c *ChainClient) RegisterOss(peerId []byte, domain string) (string, error) peerid[i] = types.U8(peerId[i]) } - if len(domain) > config.MaxDomainNameLength { - return blockhash, fmt.Errorf("register deoss: Domain name length cannot exceed %v characters", config.MaxDomainNameLength) + if len(domain) > int(MaxDomainNameLength) { + return blockhash, fmt.Errorf("register deoss: Domain name length cannot exceed %v characters", MaxDomainNameLength) } err := utils.CheckDomain(domain) @@ -658,8 +657,8 @@ func (c *ChainClient) UpdateOss(peerId string, domain string) (string, error) { peerid[i] = types.U8(peerId[i]) } - if len(domain) > config.MaxDomainNameLength { - return blockhash, fmt.Errorf("update oss: domain name length cannot exceed %v", config.MaxDomainNameLength) + if len(domain) > int(MaxDomainNameLength) { + return blockhash, fmt.Errorf("update oss: domain name length cannot exceed %v", MaxDomainNameLength) } err := utils.CheckDomain(domain) diff --git a/chain/extrinsic_name.go b/chain/extrinsic_name.go index f10224e..c44f7de 100644 --- a/chain/extrinsic_name.go +++ b/chain/extrinsic_name.go @@ -199,12 +199,13 @@ const ( ExtName_Multisig_cancel_as_multi = "Multisig.cancel_as_multi" // Oss - ExtName_Oss_authorize = "Oss.authorize" - ExtName_Oss_cancel_authorize = "Oss.cancel_authorize" - ExtName_Oss_destroy = "Oss.destroy" - ExtName_Oss_proxy_authorzie = "Oss.proxy_authorzie" - ExtName_Oss_register = "Oss.register" - ExtName_Oss_update = "Oss.update" + ExtName_Oss_authorize = "Oss.authorize" + ExtName_Oss_cancel_authorize = "Oss.cancel_authorize" + ExtName_Oss_destroy = "Oss.destroy" + ExtName_Oss_evm_proxy_authorzie = "Oss.evm_proxy_authorzie" + ExtName_Oss_proxy_authorzie = "Oss.proxy_authorzie" + ExtName_Oss_register = "Oss.register" + ExtName_Oss_update = "Oss.update" // Parameters ExtName_Parameters_set_parameter = "Parameters.set_parameter" @@ -1213,6 +1214,11 @@ func (c *ChainClient) InitExtrinsicsName() error { } else { return err } + if callIndex, err := c.GetMetadata().FindCallIndex(ExtName_Oss_evm_proxy_authorzie); err == nil { + ExtrinsicsName[callIndex] = ExtName_Oss_evm_proxy_authorzie + } else { + return err + } if callIndex, err := c.GetMetadata().FindCallIndex(ExtName_Oss_proxy_authorzie); err == nil { ExtrinsicsName[callIndex] = ExtName_Oss_proxy_authorzie } else { diff --git a/chain/file_bank.go b/chain/file_bank.go index 8a60205..f29482f 100755 --- a/chain/file_bank.go +++ b/chain/file_bank.go @@ -15,7 +15,6 @@ import ( "strings" "time" - "github.com/CESSProject/cess-go-sdk/config" "github.com/CESSProject/cess-go-sdk/utils" "github.com/centrifuge/go-substrate-rpc-client/v4/types" "github.com/centrifuge/go-substrate-rpc-client/v4/types/codec" @@ -1201,7 +1200,7 @@ func (c *ChainClient) TransferReport(index uint8, fid string) (string, error) { accountInfo types.AccountInfo ) - if index <= 0 || int(index) > (config.DataShards+config.ParShards) { + if index <= 0 || int(index) > (DataShards+ParShards) { return "", errors.New("invalid index") } diff --git a/chain/pattern.go b/chain/pattern.go index 091f9ba..ae37a38 100644 --- a/chain/pattern.go +++ b/chain/pattern.go @@ -34,6 +34,28 @@ const BlockInterval = time.Second * time.Duration(BlockIntervalSec) const TreasuryAccount = "cXhT9Xh3DhrBMDmXcGeMPDmTzDm1J8vDxBtKvogV33pPshnWS" +const ( + SIZE_1KiB = 1024 + SIZE_1MiB = 1024 * SIZE_1KiB + SIZE_1GiB = 1024 * SIZE_1MiB + SIZE_1TiB = 1024 * SIZE_1GiB + + NumberOfDataCopies = 3 + + SegmentSize = 32 * SIZE_1MiB + FragmentSize = 8 * SIZE_1MiB + DataShards = 4 + ParShards = 8 +) + +const ( + MinBucketNameLength uint8 = 3 + MaxBucketNameLength uint8 = 63 + MaxDomainNameLength uint8 = 100 +) + +const ZeroFileHash_8M string = "2daeb1f36095b44b318410b3f4e8b5d989dcc7bb023d1426c492dab0a3053e74" + // pallet names const ( // Audit diff --git a/chain/sminer.go b/chain/sminer.go index 4352cf4..e02745b 100755 --- a/chain/sminer.go +++ b/chain/sminer.go @@ -1431,14 +1431,14 @@ func (c *ChainClient) RegisterPoisKey(poisKey PoISKeyInfo, teeSignWithAcc, teeSi // which is the first stage of storage miner registration. // // - earnings: earnings account -// - peerId: peer id +// - addr: address // - staking: number of staking, the unit is CESS // - tibCount: the size of declaration space, in TiB // // Return: // - string: block hash // - error: error message -func (c *ChainClient) RegnstkSminer(earnings string, peerId []byte, staking uint64, tibCount uint32) (string, error) { +func (c *ChainClient) RegnstkSminer(earnings string, addr []byte, staking uint64, tibCount uint32) (string, error) { <-c.tradeCh defer func() { c.tradeCh <- true @@ -1452,13 +1452,13 @@ func (c *ChainClient) RegnstkSminer(earnings string, peerId []byte, staking uint accountInfo types.AccountInfo ) - var peerid PeerId - if len(peerId) != PeerIdPublicKeyLen { - return blockhash, fmt.Errorf("[RegnstkSminer] invalid peerid: %v", peerId) + var pid PeerId + if len(addr) > PeerIdPublicKeyLen { + return blockhash, fmt.Errorf("[RegnstkSminer] addr exceeds %d bytes", PeerIdPublicKeyLen) } - for i := 0; i < len(peerid); i++ { - peerid[i] = types.U8(peerId[i]) + for i := 0; i < len(addr); i++ { + pid[i] = types.U8(addr[i]) } pubkey, err := utils.ParsingPublickey(earnings) @@ -1473,7 +1473,7 @@ func (c *ChainClient) RegnstkSminer(earnings string, peerId []byte, staking uint if !ok { return blockhash, errors.New("[big.Int.SetString]") } - call, err := types.NewCall(c.metadata, ExtName_Sminer_regnstk, *acc, peerid, types.NewU128(*realTokens), types.U32(tibCount)) + call, err := types.NewCall(c.metadata, ExtName_Sminer_regnstk, *acc, pid, types.NewU128(*realTokens), types.U32(tibCount)) if err != nil { err = fmt.Errorf("rpc err: [%s] [tx] [%s] NewCall: %v", c.GetCurrentRpcAddr(), ExtName_Sminer_regnstk, err) return blockhash, err @@ -1847,14 +1847,14 @@ func (c *ChainClient) UpdateBeneficiary(earnings string) (string, error) { } } -// UpdateSminerPeerId update peer id for storage miner +// UpdateSminerAddr update address for storage miner // -// - peerid: peer id +// - addr: address // // Return: // - string: block hash // - error: error message -func (c *ChainClient) UpdateSminerPeerId(peerid PeerId) (string, error) { +func (c *ChainClient) UpdateSminerAddr(addr []byte) (string, error) { <-c.tradeCh defer func() { c.tradeCh <- true @@ -1868,7 +1868,17 @@ func (c *ChainClient) UpdateSminerPeerId(peerid PeerId) (string, error) { accountInfo types.AccountInfo ) - call, err := types.NewCall(c.metadata, ExtName_Sminer_update_peer_id, peerid) + var pid PeerId + + if len(addr) > PeerIdPublicKeyLen { + return blockhash, fmt.Errorf("addr exceeds %d bytes", PeerIdPublicKeyLen) + } + + for i := 0; i < len(addr); i++ { + pid[i] = types.U8(addr[i]) + } + + call, err := types.NewCall(c.metadata, ExtName_Sminer_update_peer_id, pid) if err != nil { err = fmt.Errorf("rpc err: [%s] [tx] [%s] NewCall: %v", c.GetCurrentRpcAddr(), ExtName_Sminer_update_peer_id, err) return blockhash, err diff --git a/chain/util.go b/chain/util.go index 27808ae..7bcc050 100644 --- a/chain/util.go +++ b/chain/util.go @@ -9,8 +9,10 @@ package chain import ( "encoding/hex" + "regexp" "strings" + "github.com/CESSProject/cess-go-sdk/utils" "github.com/centrifuge/go-substrate-rpc-client/v4/types" "github.com/ethereum/go-ethereum/log" "github.com/pkg/errors" @@ -103,3 +105,42 @@ func H160ToSS58(origin string, chain_id uint16) (string, error) { return new_acc, nil } + +var re = regexp.MustCompile(`^[a-zA-Z0-9._-]+$`) + +func CheckBucketName(name string) bool { + if len(name) < int(MinBucketNameLength) || len(name) > int(MaxBucketNameLength) { + return false + } + + if !re.MatchString(name) { + return false + } + + if strings.Contains(name, " ") { + return false + } + + if strings.Count(name, ".") > 2 { + return false + } + + if byte(name[0]) == byte('.') || + byte(name[0]) == byte('-') || + byte(name[0]) == byte('_') || + byte(name[len(name)-1]) == byte('.') || + byte(name[len(name)-1]) == byte('-') || + byte(name[len(name)-1]) == byte('_') { + return false + } + + if utils.IsIPv4(name) { + return false + } + + if utils.IsIPv6(name) { + return false + } + + return true +} diff --git a/config/config.go b/config/config.go index ce81aac..d66b581 100755 --- a/config/config.go +++ b/config/config.go @@ -7,30 +7,7 @@ package config -const ( - SIZE_1KiB = 1024 - SIZE_1MiB = 1024 * SIZE_1KiB - SIZE_1GiB = 1024 * SIZE_1MiB - SIZE_1TiB = 1024 * SIZE_1GiB - - SegmentSize = 32 * SIZE_1MiB - FragmentSize = 8 * SIZE_1MiB - DataShards = 4 - ParShards = 8 -) - -const ( - MinBucketNameLength = 3 - MaxBucketNameLength = 63 - MaxDomainNameLength = 100 -) - const ( // default name CharacterName_Default = "cess-sdk-go" - - // offcial gateway address - PublicGatewayAddr = "https://deoss-pub-gateway.cess.network/" - // offcial gateway account - PublicGatewayAccount = "cXhwBytXqrZLr1qM5NHJhCzEMckSTzNKw17ci2aHft6ETSQm9" ) diff --git a/core/erasure/rs.go b/core/erasure/rs.go index 0c8a3c0..3e7d387 100644 --- a/core/erasure/rs.go +++ b/core/erasure/rs.go @@ -13,7 +13,7 @@ import ( "os" "path/filepath" - "github.com/CESSProject/cess-go-sdk/config" + "github.com/CESSProject/cess-go-sdk/chain" "github.com/CESSProject/cess-go-sdk/utils" "github.com/klauspost/reedsolomon" ) @@ -36,11 +36,11 @@ func ReedSolomon(file string, saveDir string) ([]string, error) { if fstat.IsDir() { return nil, errors.New("not a file") } - if fstat.Size() != config.SegmentSize { + if fstat.Size() != chain.SegmentSize { return nil, errors.New("invalid size") } - enc, err := reedsolomon.New(config.DataShards, config.ParShards) + enc, err := reedsolomon.New(chain.DataShards, chain.ParShards) if err != nil { return shardspath, err } @@ -93,12 +93,12 @@ func RSRestore(outpath string, shardspath []string) error { return nil } - enc, err := reedsolomon.New(config.DataShards, config.ParShards) + enc, err := reedsolomon.New(chain.DataShards, chain.ParShards) if err != nil { return err } - shards := make([][]byte, config.DataShards+config.ParShards) + shards := make([][]byte, chain.DataShards+chain.ParShards) for k, v := range shardspath { shards[k], err = os.ReadFile(v) if err != nil { @@ -127,7 +127,7 @@ func RSRestore(outpath string, shardspath []string) error { return err } defer f.Close() - err = enc.Join(f, shards, len(shards[0])*config.DataShards) + err = enc.Join(f, shards, len(shards[0])*chain.DataShards) return err } @@ -145,7 +145,7 @@ func RSRestoreData(outpath string, sharddata [][]byte) error { return nil } - datashards, parshards := config.DataShards, config.ParShards + datashards, parshards := chain.DataShards, chain.ParShards enc, err := reedsolomon.New(datashards, parshards) if err != nil { diff --git a/core/process/common.go b/core/process/common.go new file mode 100644 index 0000000..a6d124b --- /dev/null +++ b/core/process/common.go @@ -0,0 +1,14 @@ +/* + Copyright (C) CESS. All rights reserved. + Copyright (C) Cumulus Encrypted Storage System. All rights reserved. + + SPDX-License-Identifier: Apache-2.0 +*/ + +package process + +type RespType struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data any `json:"data"` +} diff --git a/core/process/file_chunk.go b/core/process/file_chunk.go index 0dd0d67..7424fc1 100644 --- a/core/process/file_chunk.go +++ b/core/process/file_chunk.go @@ -20,7 +20,7 @@ import ( "path/filepath" "strings" - "github.com/CESSProject/cess-go-sdk/config" + "github.com/CESSProject/cess-go-sdk/chain" "github.com/CESSProject/cess-go-sdk/utils" "github.com/btcsuite/btcutil/base58" "github.com/centrifuge/go-substrate-rpc-client/v4/signature" @@ -182,7 +182,7 @@ func UploadFileChunk(url, mnemonic, file, territory, bucket string, addExtendHea return "", errors.New("empty file") } - if !utils.CheckBucketName(bucket) { + if !chain.CheckBucketName(bucket) { return "", errors.New("invalid bucket name") } @@ -271,7 +271,7 @@ func UploadFileChunk(url, mnemonic, file, territory, bucket string, addExtendHea // - int: number of file chunks. // - error: error message. func SplitFileWithstandardSize(fpath, chunksDir string) (int64, int, error) { - return SplitFile(fpath, chunksDir, config.SegmentSize, true) + return SplitFile(fpath, chunksDir, chain.SegmentSize, true) } // Split File into Chunks. diff --git a/core/process/file.go b/core/process/gateway.go similarity index 95% rename from core/process/file.go rename to core/process/gateway.go index 0c483d9..fa7bf09 100644 --- a/core/process/file.go +++ b/core/process/gateway.go @@ -17,6 +17,7 @@ import ( "path/filepath" "strings" + "github.com/CESSProject/cess-go-sdk/chain" "github.com/CESSProject/cess-go-sdk/utils" "github.com/btcsuite/btcutil/base58" "github.com/centrifuge/go-substrate-rpc-client/v4/signature" @@ -48,8 +49,6 @@ var globalTransport = &http.Transport{ // // Explanation: // - Account refers to the account where you configured mnemonic when creating an SDK. -// - CESS public gateway address: [https://deoss-pub-gateway.cess.network/] -// - CESS public gateway account: [cXhwBytXqrZLr1qM5NHJhCzEMckSTzNKw17ci2aHft6ETSQm9] func StoreFile(url, file, bucket, territory, mnemonic string) (string, error) { fstat, err := os.Stat(file) if err != nil { @@ -65,7 +64,7 @@ func StoreFile(url, file, bucket, territory, mnemonic string) (string, error) { return "", errors.New("empty file") } - if !utils.CheckBucketName(bucket) { + if !chain.CheckBucketName(bucket) { return "", errors.New("invalid bucket name") } @@ -165,10 +164,8 @@ func StoreFile(url, file, bucket, territory, mnemonic string) (string, error) { // // Explanation: // - Account refers to the account where you configured mnemonic when creating an SDK. -// - CESS public gateway address: [https://deoss-pub-gateway.cess.network/] -// - CESS public gateway account: [cXhwBytXqrZLr1qM5NHJhCzEMckSTzNKw17ci2aHft6ETSQm9] func StoreObject(url string, bucket, territory, mnemonic string, reader io.Reader) (string, error) { - if !utils.CheckBucketName(bucket) { + if !chain.CheckBucketName(bucket) { return "", errors.New("invalid bucket name") } diff --git a/core/process/process.go b/core/process/process.go index 3b89164..8fba7e8 100755 --- a/core/process/process.go +++ b/core/process/process.go @@ -13,7 +13,6 @@ import ( "path/filepath" "github.com/CESSProject/cess-go-sdk/chain" - "github.com/CESSProject/cess-go-sdk/config" "github.com/CESSProject/cess-go-sdk/core/crypte" "github.com/CESSProject/cess-go-sdk/core/erasure" "github.com/CESSProject/cess-go-sdk/core/hashtree" @@ -46,13 +45,13 @@ func FillAndCut(file string, saveDir string) ([]string, error) { return nil, errors.Wrap(err, "FillingAndCutting") } - segmentCount := fstat.Size() / config.SegmentSize - if fstat.Size()%int64(config.SegmentSize) != 0 { + segmentCount := fstat.Size() / chain.SegmentSize + if fstat.Size()%int64(chain.SegmentSize) != 0 { segmentCount++ } segment := make([]string, segmentCount) - buf := make([]byte, config.SegmentSize) + buf := make([]byte, chain.SegmentSize) f, err := os.Open(file) if err != nil { return segment, errors.Wrap(err, "FillingAndCutting") @@ -61,7 +60,7 @@ func FillAndCut(file string, saveDir string) ([]string, error) { var num int for i := int64(0); i < segmentCount; i++ { - f.Seek(config.SegmentSize*i, 0) + f.Seek(chain.SegmentSize*i, 0) num, err = f.Read(buf) if err != nil && err != io.EOF { return segment, err @@ -69,11 +68,11 @@ func FillAndCut(file string, saveDir string) ([]string, error) { if num == 0 { return segment, errors.New("read file is empty") } - if num < config.SegmentSize { + if num < chain.SegmentSize { if i+1 != segmentCount { return segment, errors.New("read file failed") } - copy(buf[num:], make([]byte, config.SegmentSize-num)) + copy(buf[num:], make([]byte, chain.SegmentSize-num)) } hash, err := utils.CalcSHA256(buf) if err != nil { @@ -114,7 +113,7 @@ func FillAndCutWithAESEncryption(file string, cipher string, saveDir string) ([] return nil, errors.Wrap(err, "FillAndCutWithEncryption") } - segmentSize := config.SegmentSize - 16 + segmentSize := chain.SegmentSize - 16 segmentCount := fstat.Size() / int64(segmentSize) if fstat.Size()%int64(segmentSize) != 0 { segmentCount++ diff --git a/core/process/storage.go b/core/process/storage.go new file mode 100644 index 0000000..0c5dcba --- /dev/null +++ b/core/process/storage.go @@ -0,0 +1,536 @@ +/* + Copyright (C) CESS. All rights reserved. + Copyright (C) Cumulus Encrypted Storage System. All rights reserved. + + SPDX-License-Identifier: Apache-2.0 +*/ + +package process + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "mime/multipart" + "net/http" + "os" + "path/filepath" + "strings" + "time" + + "github.com/CESSProject/cess-go-sdk/chain" + "github.com/CESSProject/cess-go-sdk/core/crypte" + "github.com/CESSProject/cess-go-sdk/core/erasure" + "github.com/CESSProject/cess-go-sdk/utils" + "github.com/centrifuge/go-substrate-rpc-client/v4/types" +) + +// StoreFileToMiners store a file to some miners +// +// Receive parameter: +// - file: stored file +// - mnemonic: account mnemonic +// - territory: territory name +// - bucket: bucket name +// - timeout: timeout for waiting for block transaction to complete +// - rpcs: rpc address list +// - wantMiner: the wallet account of the miner you want to store. if it is empty, will be randomly selected. +// +// Return parameter: +// - string: [fid] unique identifier for the file +// - error: error message +// +// Preconditions: +// 1. your account needs to have money, and will be automatically created if the territory you specify does not exist. +// 2. if the number of miners you specify is less than 12, file storage will be exited if even one fails. +// 3. if the number of miners you specify is greater than 11, no other miners will be found for storage. +func StoreFileToMiners(file string, mnemonic string, territory string, bucket string, timeout time.Duration, rpcs []string, wantMiner []string) (string, error) { + size, err := CheckFile(file) + if err != nil { + return "", err + } + + if !chain.CheckBucketName(bucket) { + return "", errors.New("invalid bucket name") + } + + cacheDir := filepath.Join(filepath.Dir(file), fmt.Sprintf("%v", time.Now().UnixMilli())) + err = os.MkdirAll(cacheDir, 0755) + if err != nil { + return "", err + } + defer func() { + os.RemoveAll(cacheDir) + }() + + segmentInfo, fid, err := FullProcessing(file, "", cacheDir) + if err != nil { + return "", err + } + + if mnemonic == "" { + return fid, errors.New("empty mnemonic") + } + + cli, err := chain.NewChainClient(context.Background(), "", rpcs, mnemonic, timeout) + if err != nil { + return fid, err + } + defer cli.Close() + + err = CheckAccount(cli, territory, size) + if err != nil { + return fid, err + } + + _, err = cli.PlaceStorageOrder(fid, filepath.Base(file), bucket, territory, segmentInfo, cli.GetSignatureAccPulickey(), uint64(size)) + if err != nil { + return fid, err + } + + segmentlength := len(segmentInfo) + var fragmentGroup = make([][]string, chain.DataShards+chain.ParShards) + for j := 0; j < chain.DataShards+chain.ParShards; j++ { + fragmentGroup[j] = make([]string, segmentlength) + for i := 0; i < segmentlength; i++ { + fragmentGroup[j][i] = segmentInfo[i].FragmentHash[j] + } + } + + if len(wantMiner) >= (chain.DataShards + chain.ParShards) { + return fid, StoreToAllDesignatedMiners(cli, fragmentGroup, fid, wantMiner) + } + err = StorageToMiners(cli, fragmentGroup, fid, wantMiner) + return fid, err +} + +// RetrieveFileFromMiners Retrieve a storaged file from storage miners +// +// Receive parameter: +// - rpcs: rpc address list +// - fid: [fid] unique identifier for the file +// - cipher: decryption password, if any +// - savedir: file save directory, final save location: / +// +// Return parameter: +// - error: error message +// +// Preconditions: +// 1. the file to be downloaded needs to have been stored in the miner +func RetrieveFileFromMiners(rpcs []string, fid string, cipher, savedir string) error { + cli, err := chain.NewChainClient(context.Background(), "", rpcs, "", 0) + if err != nil { + return err + } + defer cli.Close() + + metaInfo, err := cli.QueryFile(fid, -1) + if err != nil { + if errors.Is(err, chain.ERR_RPC_EMPTY_VALUE) { + return errors.New("not found") + } + return err + } + _, err = Retrievefile(cli, metaInfo, fid, savedir, cipher) + return err +} + +func Retrievefile(cli chain.Chainer, fmeta chain.FileMetadata, fid, savedir, cipher string) (string, error) { + userfile := filepath.Join(savedir, fid) + fstat, err := os.Stat(userfile) + if err == nil { + if fstat.Size() > 0 { + return userfile, nil + } + } + err = os.MkdirAll(savedir, 0755) + if err != nil { + return userfile, err + } + + defer func(basedir string) { + for _, segment := range fmeta.SegmentList { + os.Remove(filepath.Join(basedir, string(segment.Hash[:]))) + for _, fragment := range segment.FragmentList { + os.Remove(filepath.Join(basedir, string(fragment.Hash[:]))) + } + } + }(savedir) + + var segmentspath = make([]string, 0) + fragmentpaths := make([]string, chain.DataShards+chain.ParShards) + + for _, segment := range fmeta.SegmentList { + for k, fragment := range segment.FragmentList { + fragmentpath := filepath.Join(savedir, string(fragment.Hash[:])) + fragmentpaths[k] = fragmentpath + if string(fragment.Hash[:]) != chain.ZeroFileHash_8M { + account, err := utils.EncodePublicKeyAsCessAccount(fragment.Miner[:]) + if err != nil { + return userfile, err + } + buf, err := DownloadFragmentFromMiner(cli, fragment.Miner[:], fid, string(fragment.Hash[:])) + if err != nil { + return userfile, fmt.Errorf("download from [%s] failed: %v", account, err) + } + err = utils.WriteBufToFile(buf, fragmentpath) + if err != nil { + return userfile, err + } + } else { + _, err = os.Stat(fragmentpath) + if err != nil { + ff, err := os.Create(fragmentpath) + if err != nil { + return userfile, err + } + _, err = ff.Write(make([]byte, chain.FragmentSize)) + if err != nil { + return userfile, err + } + err = ff.Sync() + if err != nil { + return userfile, err + } + err = ff.Close() + if err != nil { + return userfile, err + } + } + } + } + segmentpath := filepath.Join(savedir, string(segment.Hash[:])) + err = erasure.RSRestore(segmentpath, fragmentpaths) + if err != nil { + return "", err + } + segmentspath = append(segmentspath, segmentpath) + } + + if len(segmentspath) != len(fmeta.SegmentList) { + return "", errors.New("download failed") + } + + fd, err := os.Create(userfile) + if err != nil { + return "", err + } + defer fd.Close() + + var writecount = 0 + for i := 0; i < len(segmentspath); i++ { + buf, err := os.ReadFile(segmentspath[i]) + if err != nil { + return "", err + } + if cipher != "" { + buffer, err := crypte.AesCbcDecrypt(buf, []byte(cipher)) + if err != nil { + return "", err + } + if (writecount + 1) >= len(fmeta.SegmentList) { + fd.Write(buffer[:(fmeta.FileSize.Uint64() - uint64(writecount*chain.SegmentSize))]) + } else { + fd.Write(buffer) + } + } else { + if (writecount + 1) >= len(fmeta.SegmentList) { + fd.Write(buf[:(fmeta.FileSize.Uint64() - uint64(writecount*chain.SegmentSize))]) + } else { + fd.Write(buf) + } + } + writecount++ + } + if writecount != len(fmeta.SegmentList) { + return "", errors.New("write failed") + } + err = fd.Sync() + return userfile, err +} + +func DownloadFragmentFromMiner(cli chain.Chainer, minerpuk []byte, fid, fragment string) ([]byte, error) { + minerInfo, err := cli.QueryMinerItems(minerpuk, -1) + if err != nil { + return nil, err + } + + url := string(minerInfo.PeerId[:]) + + if strings.HasSuffix(url, "/") { + url = url + "fragment" + } else { + url = url + "/fragment" + } + + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + return nil, err + } + + message := utils.GetRandomcode(16) + sig, err := utils.SignedSR25519WithMnemonic(cli.GetURI(), message) + if err != nil { + return nil, fmt.Errorf("[SignedSR25519WithMnemonic] %v", err) + } + req.Header.Set("Fid", fid) + req.Header.Set("Fragment", fragment) + req.Header.Set("Account", cli.GetSignatureAcc()) + req.Header.Set("Message", message) + req.Header.Set("Signature", string(sig)) + req.Header.Set("Content-Type", "application/json") + + client := &http.Client{} + client.Transport = globalTransport + resp, err := client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + respbody, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("failed code: %d", resp.StatusCode) + } + return respbody, nil +} + +func StorageToMiners(cli chain.Chainer, fragmentGroup [][]string, fid string, wantMiner []string) error { + var ok bool + var err error + var sucMiner = make(map[string]struct{}, 0) + for i := 0; i < len(wantMiner); i++ { + _, ok = sucMiner[wantMiner[i]] + if ok { + continue + } + err = StoreBatchFragmentsToMiner(cli, fragmentGroup[i], fid, wantMiner[i]) + if err != nil { + return fmt.Errorf("[%s] failed: %v\n", wantMiner[i], err) + } + sucMiner[wantMiner[i]] = struct{}{} + if (i + 1) >= (chain.DataShards + chain.ParShards) { + return nil + } + } + fragmentGroup = fragmentGroup[len(wantMiner):] + + allminers, err := cli.QueryAllMiner(-1) + if err != nil { + return err + } + length := len(allminers) + account := "" + for i := 0; i < length; i++ { + account, err = utils.EncodePublicKeyAsCessAccount(allminers[i][:]) + if err != nil { + continue + } + _, ok = sucMiner[account] + if ok { + continue + } + err = StoreBatchFragmentsToMiner(cli, fragmentGroup[0], fid, account) + if err != nil { + continue + } + if len(fragmentGroup) == 1 { + return nil + } + fragmentGroup = fragmentGroup[1:] + } + return errors.New("storage failed") +} + +func StoreToAllDesignatedMiners(cli chain.Chainer, fragmentGroup [][]string, fid string, wantMiner []string) error { + var ok bool + var err error + var rntMsg string + var sucMiner = make(map[string]struct{}, 0) + minerlength := len(wantMiner) + for i := 0; i < chain.DataShards+chain.ParShards; i++ { + for j := 0; j < minerlength; j++ { + _, ok = sucMiner[wantMiner[j]] + if ok { + continue + } + err = StoreBatchFragmentsToMiner(cli, fragmentGroup[i], fid, wantMiner[j]) + if err != nil { + rntMsg += fmt.Sprintf("[%s] failed: %v\n", wantMiner[j], err) + } else { + sucMiner[wantMiner[j]] = struct{}{} + rntMsg += fmt.Sprintf("[%s] suc\n", wantMiner[j]) + } + } + } + if len(sucMiner) == chain.DataShards+chain.ParShards { + return nil + } + return errors.New(rntMsg) +} + +func StoreBatchFragmentsToMiner(cli chain.Chainer, fragments []string, fid, account string) error { + puk, err := utils.ParsingPublickey(account) + if err != nil { + return err + } + minerInfo, err := cli.QueryMinerItems(puk, -1) + if err != nil { + return err + } + + if string(minerInfo.State) != chain.MINER_STATE_POSITIVE { + return errors.New("not positive state") + } + + if minerInfo.IdleSpace.Uint64() < uint64(len(fragments)*chain.FragmentSize) { + return errors.New("insufficient space") + } + + length := len(fragments) + for i := 0; i < length; i++ { + err = UploadFragmentToMiner(cli, string(minerInfo.PeerId[:]), fid, fragments[i]) + if err != nil { + return err + } + } + return nil +} + +func UploadFragmentToMiner(cli chain.Chainer, ip string, fid string, file string) error { + message := utils.GetRandomcode(16) + sig, err := utils.SignedSR25519WithMnemonic(cli.GetURI(), message) + if err != nil { + return fmt.Errorf("[SignedSR25519WithMnemonic] %v", err) + } + + body := new(bytes.Buffer) + writer := multipart.NewWriter(body) + formFile, err := writer.CreateFormFile("file", filepath.Base(file)) + if err != nil { + return err + } + + fd, err := os.Open(file) + if err != nil { + return err + } + defer fd.Close() + + _, err = io.Copy(formFile, fd) + if err != nil { + return err + } + err = writer.Close() + if err != nil { + return err + } + url := ip + if strings.HasSuffix(url, "/") { + url = url + "fragment" + } else { + url = url + "/fragment" + } + + req, err := http.NewRequest(http.MethodPut, url, body) + if err != nil { + return err + } + + req.Header.Set("Fid", fid) + req.Header.Set("Account", cli.GetSignatureAcc()) + req.Header.Set("Message", message) + req.Header.Set("Signature", string(sig)) + req.Header.Set("Content-Type", writer.FormDataContentType()) + + client := &http.Client{} + client.Transport = globalTransport + resp, err := client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + respbody, err := io.ReadAll(resp.Body) + if err != nil { + return err + } + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("failed code: %d", resp.StatusCode) + } + var respinfo RespType + err = json.Unmarshal(respbody, &respinfo) + if err != nil { + return errors.New("server returns invalid data") + } + if respinfo.Code != 200 { + return fmt.Errorf("server returns code: %d", respinfo.Code) + } + return nil +} + +func CheckAccount(cli chain.Chainer, territory string, size int64) error { + useSpace := CalcUsedSpace(size) + territoryInfo, err := cli.QueryTerritory(cli.GetSignatureAccPulickey(), territory, -1) + if err != nil { + if !errors.Is(err, chain.ERR_RPC_EMPTY_VALUE) { + return err + } + gibs := useSpace / chain.SIZE_1GiB + if useSpace%chain.SIZE_1GiB != 0 { + gibs += 1 + } + _, err = cli.MintTerritory(uint32(gibs), territory, uint32(30)) + if err != nil { + return err + } + time.Sleep(chain.BlockInterval) + territoryInfo, err = cli.QueryTerritory(cli.GetSignatureAccPulickey(), territory, -1) + if err != nil { + return err + } + } + header, err := cli.GetSubstrateAPI().RPC.Chain.GetHeaderLatest() + if err != nil { + return err + } + + if territoryInfo.Deadline <= types.U32(header.Number) { + return errors.New("expired territory") + } + + if territoryInfo.RemainingSpace.Uint64() < useSpace { + return errors.New("insufficient territorial space") + } + return nil +} + +func CheckFile(file string) (int64, error) { + fstat, err := os.Stat(file) + if err != nil { + return 0, err + } + if fstat.IsDir() { + return 0, errors.New("not a file") + } + if fstat.Size() <= 0 { + return 0, errors.New("empty file") + } + return fstat.Size(), nil +} + +func CalcUsedSpace(size int64) uint64 { + count := size / chain.SegmentSize + if size%chain.SegmentSize != 0 { + count += 1 + } + return uint64(count*chain.SegmentSize) * chain.NumberOfDataCopies +} diff --git a/example/evm/evm.go b/example/evm/evm.go index a65a219..cdb0953 100644 --- a/example/evm/evm.go +++ b/example/evm/evm.go @@ -69,7 +69,7 @@ func main() { fmt.Printf("%s", block_hash) } -func NewSDK() (*chain.ChainClient, error) { +func NewSDK() (chain.Chainer, error) { return cess.New( context.Background(), cess.ConnectRpcAddrs(RPC_ADDRS), diff --git a/sdk.go b/sdk.go index 512bbca..f7e0c9b 100755 --- a/sdk.go +++ b/sdk.go @@ -23,7 +23,7 @@ import ( // timeout: time.Duration(time.Second * 6) // // - The serviceName is used to specify the name of your service -func New(ctx context.Context, opts ...Option) (*chain.ChainClient, error) { +func New(ctx context.Context, opts ...Option) (chain.Chainer, error) { return NewWithoutDefaults(ctx, append(opts, FallbackDefaults)...) } @@ -33,7 +33,7 @@ func New(ctx context.Context, opts ...Option) (*chain.ChainClient, error) { // Warning: This function should not be considered a stable interface. We may // choose to add required services at any time and, by using this function, you // opt-out of any defaults we may provide. -func NewWithoutDefaults(ctx context.Context, opts ...Option) (*chain.ChainClient, error) { +func NewWithoutDefaults(ctx context.Context, opts ...Option) (chain.Chainer, error) { var cfg Config if err := cfg.Apply(opts...); err != nil { return nil, err diff --git a/utils/bucket.go b/utils/bucket.go index 49701fd..23a7bd8 100755 --- a/utils/bucket.go +++ b/utils/bucket.go @@ -6,49 +6,3 @@ */ package utils - -import ( - "regexp" - "strings" - - "github.com/CESSProject/cess-go-sdk/config" -) - -var re = regexp.MustCompile(`^[a-zA-Z0-9._-]+$`) - -func CheckBucketName(name string) bool { - if len(name) < config.MinBucketNameLength || len(name) > config.MaxBucketNameLength { - return false - } - - if !re.MatchString(name) { - return false - } - - if strings.Contains(name, " ") { - return false - } - - if strings.Count(name, ".") > 2 { - return false - } - - if byte(name[0]) == byte('.') || - byte(name[0]) == byte('-') || - byte(name[0]) == byte('_') || - byte(name[len(name)-1]) == byte('.') || - byte(name[len(name)-1]) == byte('-') || - byte(name[len(name)-1]) == byte('_') { - return false - } - - if IsIPv4(name) { - return false - } - - if IsIPv6(name) { - return false - } - - return true -}