diff --git a/chain/chain.go b/chain/chain.go index 6f4c8df3e..a81ecfade 100644 --- a/chain/chain.go +++ b/chain/chain.go @@ -5,12 +5,13 @@ import ( "encoding/hex" "errors" "fmt" - "github.com/bittorrent/go-btfs/chain/tokencfg" "io" "math/big" "strings" "time" + "github.com/bittorrent/go-btfs/chain/tokencfg" + "github.com/bittorrent/go-btfs/accounting" "github.com/bittorrent/go-btfs/chain/config" "github.com/bittorrent/go-btfs/settlement" @@ -348,7 +349,7 @@ func initSwap( priceOracle := priceoracle.New(currentPriceOracleAddress, transactionService) _, err := priceOracle.CheckNewPrice(tokencfg.GetWbttToken()) // CheckNewPrice when node starts if err != nil { - return nil, nil, errors.New("CheckNewPrice " + err.Error()) + return nil, nil, errors.New("CheckNewPrice error, it may happens when contract call failed if bttc chain rpc is down, please try again") } swapProtocol := swapprotocol.New(overlayEthAddress, priceOracle) diff --git a/cmd/btfs/init.go b/cmd/btfs/init.go index 1261778a2..63950b5c9 100644 --- a/cmd/btfs/init.go +++ b/cmd/btfs/init.go @@ -10,6 +10,7 @@ import ( "path/filepath" "strconv" "strings" + "time" "github.com/bittorrent/go-btfs/assets" "github.com/bittorrent/go-btfs/chain" @@ -36,6 +37,7 @@ const ( rmOnUnpinOptionName = "rm-on-unpin" seedOptionName = "seed" simpleMode = "simple-mode" + recoveryOptionName = "recovery" /* passWordOptionName = "password" passwordFileoptionName = "password-file" @@ -72,6 +74,7 @@ environment variable: cmds.BoolOption(rmOnUnpinOptionName, "r", "Remove unpinned files.").WithDefault(false), cmds.StringOption(seedOptionName, "s", "Import seed phrase"), cmds.BoolOption(simpleMode, "sm", "init with simple mode or not."), + cmds.StringOption(recoveryOptionName, "Recovery data from a backup"), /* cmds.StringOption(passWordOptionName, "", "password for decrypting keys."), cmds.StringOption(passwordFileoptionName, "", "path to a file that contains password for decrypting keys"), @@ -140,7 +143,30 @@ environment variable: password, _ := req.Options[passWordOptionName].(string) passwordFile, _ := req.Options[passwordFileoptionName].(string) */ + backupPath, ok := req.Options[recoveryOptionName].(string) + if ok { + btfsPath := env.(*oldcmds.Context).ConfigRoot + dstPath := filepath.Dir(btfsPath) + if fsrepo.IsInitialized(btfsPath) { + newPath := filepath.Join(dstPath, fmt.Sprintf(".btfs_backup_%d", time.Now().Unix())) + // newPath := filepath.Join(filepath.Dir(btfsPath), backup) + err := os.Rename(btfsPath, newPath) + if err != nil { + return err + } + fmt.Println("btfs configuration file already exists!") + fmt.Println("We have renamed it to ", newPath) + } + if err := commands.UnTar(backupPath, dstPath); err != nil { + err = commands.UnZip(backupPath, dstPath) + if err != nil { + return errors.New("your file format is not tar.gz or zip, please check again") + } + } + fmt.Println("Recovery successful!") + return nil + } return doInit(os.Stdout, cctx.ConfigRoot, empty, nBitsForKeypair, profile, conf, keyType, importKey, seedPhrase, rmOnUnpin, simpleModeIn) }, } diff --git a/core/commands/backup.go b/core/commands/backup.go new file mode 100644 index 000000000..1de94bf16 --- /dev/null +++ b/core/commands/backup.go @@ -0,0 +1,339 @@ +package commands + +import ( + "archive/tar" + "archive/zip" + "compress/gzip" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "time" + + cmds "github.com/bittorrent/go-btfs-cmds" + commands "github.com/bittorrent/go-btfs/commands" + fsrepo "github.com/bittorrent/go-btfs/repo/fsrepo" +) + +const ( + outputFileOption = "o" + compressOption = "a" + backupPathOption = "r" + excludeOption = "exclude" +) + +var BackupCmd = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "Back up BTFS's data", + LongDescription: ` +This command will create a backup of the data from the current BTFS node. +`, + }, + Arguments: []cmds.Argument{ + cmds.FileArg("file", true, false, "data to encode").EnableStdin(), + }, + Options: []cmds.Option{ + cmds.StringOption(outputFileOption, "backup output file path"), + cmds.StringOption(compressOption, "gz or zip").WithDefault("gz"), + cmds.StringsOption(excludeOption, "exclude backup output file path"), + }, + Run: func(req *cmds.Request, resp cmds.ResponseEmitter, env cmds.Environment) error { + r, err := fsrepo.Open(env.(*commands.Context).ConfigRoot) + if err != nil { + return err + } + defer r.Close() + + var fileName = fmt.Sprintf("btfs_backup_%d", time.Now().Unix()) + + outputName, ok := req.Options[outputFileOption].(string) + if ok { + fileName = outputName + } + btfsPath, err := fsrepo.BestKnownPath() + if err != nil { + return err + } + + excludePath, _ := req.Options[excludeOption].([]string) + for _, v := range excludePath { + // TODO + if v != "config" && v != "statestore" && v != "datastore" { + return errors.New("-exclude only support config, statestore or datastore") + } + } + // exclude the repo.lock to avoid dead lock + excludePath = append(excludePath, "repo.lock") + compressWay, _ := req.Options[compressOption].(string) + // TODO + if compressWay != "gz" && compressWay != "zip" { + return errors.New("-a only support zip or gz, gz is default") + } + absPath, err := filepath.Abs(fileName) + if err != nil { + return err + } + if compressWay == "zip" { + absPath += ".zip" + err = Zip(btfsPath, absPath, excludePath) + } else { + absPath += ".tar.gz" + err = Tar(btfsPath, absPath, excludePath) + } + if err != nil { + return err + } + fmt.Printf("Backup successful! The backup path is %s\n", absPath) + return nil + }, +} + +var RecoveryCmd = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "Recover BTFS's data from a archived file of backup", + LongDescription: `This command will recover data from a previously created backup file`, + }, + Options: []cmds.Option{ + cmds.StringOption(backupPathOption, "backup output file path"), + }, + Run: func(req *cmds.Request, resp cmds.ResponseEmitter, env cmds.Environment) error { + backupPath, ok := req.Options[backupPathOption].(string) + if !ok { + return errors.New("you need to specify -r to indicate the path you want to recover") + } + btfsPath := env.(*commands.Context).ConfigRoot + dstPath := filepath.Dir(btfsPath) + if fsrepo.IsInitialized(btfsPath) { + newPath := filepath.Join(dstPath, fmt.Sprintf(".btfs_backup_%d", time.Now().Unix())) + // newPath := filepath.Join(filepath.Dir(btfsPath), backup) + err := os.Rename(btfsPath, newPath) + if err != nil { + return err + } + fmt.Println("btfs configuration file already exists!") + fmt.Println("We have renamed it to ", newPath) + } + + if err := UnTar(backupPath, dstPath); err != nil { + err = UnZip(backupPath, dstPath) + if err != nil { + return errors.New("your file is not exists or your file format is not tar.gz or zip, please check again") + } + } + fmt.Println("Recovery successful!") + return nil + }, +} + +func Tar(src, dst string, excludePath []string) (err error) { + fw, err := os.Create(dst) + if err != nil { + return + } + defer fw.Close() + + gw := gzip.NewWriter(fw) + defer gw.Close() + + tw := tar.NewWriter(gw) + defer tw.Close() + + basePath := filepath.Dir(src) + filepath.Walk(src, func(fileAbsPath string, fi os.FileInfo, err error) error { + if err != nil { + return err + } + for _, v := range excludePath { + excludeAbsPath := filepath.Join(src, v) + if v != "" && strings.HasPrefix(fileAbsPath, excludeAbsPath) { + return nil + } + } + rel, err := filepath.Rel(basePath, fileAbsPath) + if err != nil { + return err + } + hdr, err := tar.FileInfoHeader(fi, "") + if err != nil { + return err + } + hdr.Name = rel + + // 写入文件信息 + if err = tw.WriteHeader(hdr); err != nil { + return err + } + + if fi.IsDir() { + return nil + } + + fr, err := os.Open(fileAbsPath) + if err != nil { + return err + } + defer fr.Close() + + // copy 文件数据到 tw + _, err = io.Copy(tw, fr) + if err != nil { + return err + } + return nil + }) + tw.Flush() + gw.Flush() + return +} + +func UnTar(src, dst string) (err error) { + fr, err := os.Open(src) + if err != nil { + return err + } + defer fr.Close() + gr, err := gzip.NewReader(fr) + if err != nil { + return err + } + defer gr.Close() + // tar read + tr := tar.NewReader(gr) + // 读取文件 + for { + h, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return err + } + if h.FileInfo().IsDir() { + err = os.MkdirAll(filepath.Join(dst, h.Name), h.FileInfo().Mode()) + if err != nil { + return err + } + continue + } + + fw, err := os.OpenFile(filepath.Join(dst, h.Name), os.O_CREATE|os.O_RDWR|os.O_TRUNC, os.FileMode(h.Mode)) + if err != nil { + return err + } + defer fw.Close() + // 写文件 + _, err = io.Copy(fw, tr) + if err != nil { + return err + } + } + return +} + +func Zip(src, dst string, excludePath []string) (err error) { + fw, err := os.Create(dst) + if err != nil { + return err + } + defer fw.Close() + + zw := zip.NewWriter(fw) + defer func() { + if err := zw.Close(); err != nil { + log.Fatalln(err) + } + }() + basePath := filepath.Dir(src) + filepath.Walk(src, func(fileAbsPath string, fi os.FileInfo, errBack error) (err error) { + if errBack != nil { + return errBack + } + for _, v := range excludePath { + excludeAbsPath := filepath.Join(src, v) + if v != "" && strings.HasPrefix(fileAbsPath, excludeAbsPath) { + return nil + } + } + fh, err := zip.FileInfoHeader(fi) + if err != nil { + return + } + + rel, err := filepath.Rel(basePath, fileAbsPath) + if err != nil { + return err + } + fh.Name = rel + + if fi.IsDir() { + fh.Name += "/" + } + + w, err := zw.CreateHeader(fh) + if err != nil { + return + } + + if !fh.Mode().IsRegular() { + return nil + } + + fr, err := os.Open(fileAbsPath) + if err != nil { + return + } + defer fr.Close() + + _, err = io.Copy(w, fr) + if err != nil { + return + } + return nil + }) + zw.Flush() + return +} + +func UnZip(src, dst string) (err error) { + zr, err := zip.OpenReader(src) + if err != nil { + return + } + defer zr.Close() + + for _, file := range zr.File { + err = persistZipFile(dst, file) + if err != nil { + return + } + } + return nil +} + +func persistZipFile(dst string, file *zip.File) (err error) { + path := filepath.Join(dst, file.Name) + + if file.FileInfo().IsDir() { + return os.MkdirAll(path, file.Mode()) + } + + fr, err := file.Open() + if err != nil { + return err + } + defer fr.Close() + + fw, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR|os.O_TRUNC, file.Mode()) + if err != nil { + return err + } + defer fw.Close() + + _, err = io.Copy(fw, fr) + if err != nil { + return err + } + return +} diff --git a/core/commands/commands.go b/core/commands/commands.go index 4f92bd252..3d1d6ecc1 100644 --- a/core/commands/commands.go +++ b/core/commands/commands.go @@ -12,7 +12,7 @@ import ( "sort" "strings" - "github.com/bittorrent/go-btfs-cmds" + cmds "github.com/bittorrent/go-btfs-cmds" ) type commandEncoder struct { diff --git a/core/commands/commands_test.go b/core/commands/commands_test.go index 6870f21d7..0a56d2d1b 100644 --- a/core/commands/commands_test.go +++ b/core/commands/commands_test.go @@ -350,6 +350,13 @@ func TestCommands(t *testing.T) { "/accesskey/delete", "/accesskey/get", "/accesskey/list", + "/multibase", + "/multibase/encode", + "/multibase/decode", + "/multibase/transcode", + "/multibase/list", + "/backup", + "/recovery", } cmdSet := make(map[string]struct{}) diff --git a/core/commands/multibase.go b/core/commands/multibase.go new file mode 100644 index 000000000..bdba34b0d --- /dev/null +++ b/core/commands/multibase.go @@ -0,0 +1,171 @@ +package commands + +import ( + "bytes" + "fmt" + "io" + "strings" + + cmds "github.com/bittorrent/go-btfs-cmds" + cmdenv "github.com/bittorrent/go-btfs/core/commands/cmdenv" + mbase "github.com/multiformats/go-multibase" +) + +var MbaseCmd = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "Encode and decode files or stdin with multibase format", + }, + Subcommands: map[string]*cmds.Command{ + "encode": mbaseEncodeCmd, + "decode": mbaseDecodeCmd, + "transcode": mbaseTranscodeCmd, + "list": basesCmd, + }, + Extra: CreateCmdExtras(SetDoesNotUseRepo(true)), +} + +const ( + mbaseOptionName = "b" +) + +var mbaseEncodeCmd = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "Encode data into multibase string", + LongDescription: ` +This command expects a file name or data provided via stdin. + +By default it will use URL-safe base64url encoding, +but one can customize used base with -b: + + > echo hello | btfs multibase encode -b base16 > output_file + > cat output_file + f68656c6c6f0a + + > echo hello > input_file + > btfs multibase encode -b base16 input_file + f68656c6c6f0a + `, + }, + Arguments: []cmds.Argument{ + cmds.FileArg("file", true, false, "data to encode").EnableStdin(), + }, + Options: []cmds.Option{ + cmds.StringOption(mbaseOptionName, "multibase encoding").WithDefault("base64url"), + }, + Run: func(req *cmds.Request, resp cmds.ResponseEmitter, env cmds.Environment) error { + if err := req.ParseBodyArgs(); err != nil { + return err + } + encoderName, _ := req.Options[mbaseOptionName].(string) + encoder, err := mbase.EncoderByName(encoderName) + if err != nil { + return err + } + files := req.Files.Entries() + file, err := cmdenv.GetFileArg(files) + if err != nil { + return fmt.Errorf("failed to access file: %w", err) + } + buf, err := io.ReadAll(file) + if err != nil { + return fmt.Errorf("failed to read file contents: %w", err) + } + encoded := encoder.Encode(buf) + reader := strings.NewReader(encoded) + return resp.Emit(reader) + }, +} + +var mbaseDecodeCmd = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "Decode multibase string", + LongDescription: ` +This command expects multibase inside of a file or via stdin: + + > echo -n hello | btfs multibase encode -b base16 > file + > cat file + f68656c6c6f + + > btfs multibase decode file + hello + + > cat file | btfs multibase decode + hello +`, + }, + Arguments: []cmds.Argument{ + cmds.FileArg("encoded_file", true, false, "encoded data to decode").EnableStdin(), + }, + Run: func(req *cmds.Request, resp cmds.ResponseEmitter, env cmds.Environment) error { + if err := req.ParseBodyArgs(); err != nil { + return err + } + files := req.Files.Entries() + file, err := cmdenv.GetFileArg(files) + if err != nil { + return fmt.Errorf("failed to access file: %w", err) + } + encodedData, err := io.ReadAll(file) + if err != nil { + return fmt.Errorf("failed to read file contents: %w", err) + } + _, data, err := mbase.Decode(string(encodedData)) + if err != nil { + return fmt.Errorf("failed to decode multibase: %w", err) + } + reader := bytes.NewReader(data) + return resp.Emit(reader) + }, +} + +var mbaseTranscodeCmd = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "Transcode multibase string between bases", + LongDescription: ` +This command expects multibase inside of a file or via stdin. + +By default it will use URL-safe base64url encoding, +but one can customize used base with -b: + + > echo -n hello | btfs multibase encode > file + > cat file + uaGVsbG8 + + > btfs multibase transcode file -b base16 > transcoded_file + > cat transcoded_file + f68656c6c6f +`, + }, + Arguments: []cmds.Argument{ + cmds.FileArg("encoded_file", true, false, "encoded data to decode").EnableStdin(), + }, + Options: []cmds.Option{ + cmds.StringOption(mbaseOptionName, "multibase encoding").WithDefault("base64url"), + }, + Run: func(req *cmds.Request, resp cmds.ResponseEmitter, env cmds.Environment) error { + if err := req.ParseBodyArgs(); err != nil { + return err + } + encoderName, _ := req.Options[mbaseOptionName].(string) + encoder, err := mbase.EncoderByName(encoderName) + if err != nil { + return err + } + files := req.Files.Entries() + file, err := cmdenv.GetFileArg(files) + if err != nil { + return fmt.Errorf("failed to access file: %w", err) + } + encodedData, err := io.ReadAll(file) + if err != nil { + return fmt.Errorf("failed to read file contents: %w", err) + } + _, data, err := mbase.Decode(string(encodedData)) + if err != nil { + return fmt.Errorf("failed to decode multibase: %w", err) + } + encoded := encoder.Encode(data) + reader := strings.NewReader(encoded) + return resp.Emit(reader) + }, +} diff --git a/core/commands/root.go b/core/commands/root.go index 049aa661b..0c5bb9a46 100644 --- a/core/commands/root.go +++ b/core/commands/root.go @@ -179,6 +179,9 @@ var rootSubcommands = map[string]*cmds.Command{ "network": NetworkCmd, "statuscontract": StatusContractCmd, "bittorrent": bittorrentCmd, + "multibase": MbaseCmd, + "backup": BackupCmd, + "recovery": RecoveryCmd, "accesskey": AccessKeyCmd, } diff --git a/core/commands/storage/upload/upload/upload.go b/core/commands/storage/upload/upload/upload.go index 7a8c7b248..390b3e41e 100644 --- a/core/commands/storage/upload/upload/upload.go +++ b/core/commands/storage/upload/upload/upload.go @@ -4,15 +4,18 @@ import ( "context" "errors" "fmt" - "github.com/bittorrent/go-btfs/chain/tokencfg" - "github.com/bittorrent/go-btfs/utils" "strconv" "strings" "time" + "github.com/bittorrent/go-btfs/chain/tokencfg" + "github.com/bittorrent/go-btfs/utils" + coreiface "github.com/bittorrent/interface-go-btfs-core" + "github.com/bittorrent/go-btfs/settlement/swap/swapprotocol" "github.com/bittorrent/go-btfs/chain" + "github.com/bittorrent/go-btfs/core/commands/cmdenv" "github.com/bittorrent/go-btfs/core/commands/storage/hosts" "github.com/bittorrent/go-btfs/core/commands/storage/upload/helper" "github.com/bittorrent/go-btfs/core/commands/storage/upload/offline" @@ -111,7 +114,15 @@ Use status command to check for completion: }, RunTimeout: 15 * time.Minute, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { - err := utils.CheckSimpleMode(env) + nd, err := cmdenv.GetNode(env) + if err != nil { + return err + } + + if !nd.IsOnline { + return coreiface.ErrOffline + } + err = utils.CheckSimpleMode(env) if err != nil { return err }