add getfile (#22)

CESSProject · Apr 12, 2023 · c7246c9 · c7246c9
1 parent 7f2fa6b
commit c7246c9
Show file tree

Hide file tree

Showing 5 changed files with 101 additions and 89 deletions.
diff --git a/core/client/client.go b/core/client/client.go
@@ -29,6 +29,7 @@ type Client interface {
 	CreateBucket(owner []byte, bucketname string) (string, error)
 	ProcessingData(path string) ([]SegmentInfo, string, error)
 	PutFile(owner []byte, segmentInfo []SegmentInfo, roothash, filename, bucketname string) (string, error)
+	GetFile(roothash, dir string) (string, error)
 	DeleteFile(owner []byte, roothash string) (string, chain.FileHash, error)
 	DeleteBucket(owner []byte, bucketName string) (string, error)
 	Update(name string) (string, error)

diff --git a/core/client/getfile.go b/core/client/getfile.go
@@ -0,0 +1,82 @@
+package client
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/CESSProject/sdk-go/core/erasure"
+	"github.com/CESSProject/sdk-go/core/rule"
+)
+
+func (c *Cli) GetFile(roothash, dir string) (string, error) {
+	var (
+		segmentspath = make([]string, 0)
+	)
+	userfile := filepath.Join(dir, roothash)
+	_, err := os.Stat(userfile)
+	if err == nil {
+		return userfile, nil
+	}
+	os.MkdirAll(dir, rule.DirMode)
+	f, err := os.Create(userfile)
+	if err != nil {
+		return "", err
+	}
+	defer f.Close()
+
+	fmeta, err := c.Chain.GetFileMetaInfo(roothash)
+	if err != nil {
+		return "", err
+	}
+	for _, segment := range fmeta.SegmentList {
+		for _, fragment := range segment.FragmentList {
+			fragmentpaths := make([]string, 0)
+			miner, err := c.Chain.QueryStorageMiner(fragment.Miner[:])
+			if err != nil {
+				return "", err
+			}
+			peerid, err := c.AddMultiaddrToPearstore(string(miner.Ip), time.Hour)
+			if err != nil {
+				return "", err
+			}
+			fragmentpath := filepath.Join(dir, string(fragment.Hash[:]))
+			err = c.Protocol.ReadFileAction(peerid, roothash, string(fragment.Hash[:]), fragmentpath, rule.FragmentSize)
+			if err != nil {
+				continue
+			}
+			fragmentpaths = append(fragmentpaths, fragmentpath)
+			segmentpath := filepath.Join(dir, string(segment.Hash[:]))
+			if len(fragmentpaths) >= rule.DataShards {
+				err = erasure.ReedSolomon_Restore(segmentpath, fragmentpaths)
+				if err != nil {
+					return "", err
+				}
+				segmentspath = append(segmentspath, segmentpath)
+			}
+		}
+	}
+
+	if len(segmentspath) != len(fmeta.SegmentList) {
+		return "", fmt.Errorf("Download failed")
+	}
+	var writecount = 0
+	for i := 0; i < len(fmeta.SegmentList); i++ {
+		for j := 0; j < len(segmentspath); j++ {
+			if string(fmeta.SegmentList[i].Hash[:]) == filepath.Base(segmentspath[j]) {
+				buf, err := os.ReadFile(segmentspath[j])
+				if err != nil {
+					return "", err
+				}
+				f.Write(buf)
+				writecount++
+				break
+			}
+		}
+	}
+	if writecount != len(fmeta.SegmentList) {
+		return "", fmt.Errorf("Write failed")
+	}
+	return userfile, nil
+}
diff --git a/core/erasure/rs.go b/core/erasure/rs.go
@@ -148,120 +148,47 @@ func ReedSolomon(path string) ([]string, error) {
 	// return shardspath, nil
 }
 
-func ReedSolomon_Restore(dir, fid string) error {
-	outfn := filepath.Join(dir, fid)
-
-	_, err := os.Stat(outfn)
+func ReedSolomon_Restore(outpath string, shardspath []string) error {
+	_, err := os.Stat(outpath)
 	if err == nil {
 		return nil
 	}
 
 	datashards, parshards := rule.DataShards, rule.ParShards
 
-	if datashards+parshards <= 6 {
-		enc, err := reedsolomon.New(datashards, parshards)
-		if err != nil {
-			return err
-		}
-		shards := make([][]byte, datashards+parshards)
-		for i := range shards {
-			infn := fmt.Sprintf("%s.00%d", outfn, i)
-			shards[i], err = ioutil.ReadFile(infn)
-			if err != nil {
-				shards[i] = nil
-			}
-		}
-
-		// Verify the shards
-		ok, _ := enc.Verify(shards)
-		if !ok {
-			err = enc.Reconstruct(shards)
-			if err != nil {
-				return err
-			}
-			ok, err = enc.Verify(shards)
-			if !ok {
-				return err
-			}
-		}
-		f, err := os.Create(outfn)
-		if err != nil {
-			return err
-		}
-		defer f.Close()
-		err = enc.Join(f, shards, len(shards[0])*datashards)
-		return err
-	}
-
-	enc, err := reedsolomon.NewStream(datashards, parshards)
+	enc, err := reedsolomon.New(datashards, parshards)
 	if err != nil {
 		return err
 	}
-
-	// Open the inputs
-	shards, size, err := openInput(datashards, parshards, outfn)
-	if err != nil {
-		return err
+	shards := make([][]byte, datashards+parshards)
+	for k, v := range shardspath {
+		//infn := fmt.Sprintf("%s.00%d", outfn, i)
+		shards[k], err = ioutil.ReadFile(v)
+		if err != nil {
+			shards[k] = nil
+		}
 	}
 
 	// Verify the shards
-	ok, err := enc.Verify(shards)
+	ok, _ := enc.Verify(shards)
 	if !ok {
-		shards, size, err = openInput(datashards, parshards, outfn)
+		err = enc.Reconstruct(shards)
 		if err != nil {
 			return err
 		}
-
-		out := make([]io.Writer, len(shards))
-		for i := range out {
-			if shards[i] == nil {
-				var outfn string
-				if i < 10 {
-					outfn = fmt.Sprintf("%s.00%d", outfn, i)
-				} else {
-					outfn = fmt.Sprintf("%s.0%d", outfn, i)
-				}
-				out[i], err = os.Create(outfn)
-				if err != nil {
-					return err
-				}
-			}
-		}
-		err = enc.Reconstruct(shards, out)
-		if err != nil {
-			return err
-		}
-
-		for i := range out {
-			if out[i] != nil {
-				err := out[i].(*os.File).Close()
-				if err != nil {
-					return err
-				}
-			}
-		}
-		shards, size, err = openInput(datashards, parshards, outfn)
 		ok, err = enc.Verify(shards)
 		if !ok {
 			return err
 		}
-		if err != nil {
-			return err
-		}
 	}
-
-	f, err := os.Create(outfn)
+	f, err := os.Create(outpath)
 	if err != nil {
 		return err
 	}
 	defer f.Close()
-	shards, size, err = openInput(datashards, parshards, outfn)
-	if err != nil {
-		return err
-	}
-
-	err = enc.Join(f, shards, int64(datashards)*size)
+	err = enc.Join(f, shards, len(shards[0])*datashards)
 	return err
+
 }
 
 func openInput(dataShards, parShards int, fname string) (r []io.Reader, size int64, err error) {

diff --git a/go.mod b/go.mod
@@ -4,7 +4,7 @@ go 1.19
 
 require (
 	github.com/CESSProject/cess-oss v0.1.2
-	github.com/CESSProject/p2p-go v0.0.6
+	github.com/CESSProject/p2p-go v0.0.7
 	github.com/btcsuite/btcutil v1.0.3-0.20201208143702-a53e38424cce
 	github.com/cbergoon/merkletree v0.2.0
 	github.com/centrifuge/go-substrate-rpc-client v2.0.0+incompatible

diff --git a/go.sum b/go.sum
@@ -53,6 +53,8 @@ github.com/CESSProject/p2p-go v0.0.5 h1:zbvFQdAwLYQ4pPyATc3kZBvw+1b/wiVoo1YpD1TY
 github.com/CESSProject/p2p-go v0.0.5/go.mod h1:GnUmdPBYVcyRrjijKDDMPj8OJ3QaZemob+kKaiP9ALQ=
 github.com/CESSProject/p2p-go v0.0.6 h1:fdvl6Q2gi+QVfX3LFELaSfHJZhL4D2XykARLVTfFmzM=
 github.com/CESSProject/p2p-go v0.0.6/go.mod h1:GnUmdPBYVcyRrjijKDDMPj8OJ3QaZemob+kKaiP9ALQ=
+github.com/CESSProject/p2p-go v0.0.7 h1:oW2UGYPNz1AVuj5smj6ApT0m/XJsKB5PecRgtQTY6DA=
+github.com/CESSProject/p2p-go v0.0.7/go.mod h1:GnUmdPBYVcyRrjijKDDMPj8OJ3QaZemob+kKaiP9ALQ=
 github.com/ChainSafe/go-schnorrkel v1.0.0 h1:3aDA67lAykLaG1y3AOjs88dMxC88PgUuHRrLeDnvGIM=
 github.com/ChainSafe/go-schnorrkel v1.0.0/go.mod h1:dpzHYVxLZcp8pjlV+O+UR8K0Hp/z7vcchBSbMBEhCw4=
 github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM=