From ee7e1a6990800f612bae42cbc1479673e16767cf Mon Sep 17 00:00:00 2001 From: cody Date: Tue, 19 Nov 2024 11:18:25 +0800 Subject: [PATCH 01/10] feat: todo, add mode and mtime for file --- core/commands/add.go | 13 +++++++++++++ core/coreapi/unixfs.go | 14 +++++++++----- core/coreunix/add.go | 8 ++++++++ 3 files changed, 30 insertions(+), 5 deletions(-) diff --git a/core/commands/add.go b/core/commands/add.go index cc6576914..a3cc8e1d1 100644 --- a/core/commands/add.go +++ b/core/commands/add.go @@ -61,6 +61,8 @@ const ( peerIdName = "peer-id" pinDurationCountOptionName = "pin-duration-count" uploadToBlockchainOptionName = "to-blockchain" + modeOptionName = "mode" + mtimeOptionName = "mtime" ) const adderOutChanSize = 8 @@ -214,6 +216,9 @@ only-hash, and progress/status related flags) will change the final hash. peerId, _ := req.Options[peerIdName].(string) pinDuration, _ := req.Options[pinDurationCountOptionName].(int) uploadToBlockchain, _ := req.Options[uploadToBlockchainOptionName].(bool) + // TODO mode mtime + mode, _ := req.Options[modeOptionName].(uint) + mtime, _ := req.Options[mtimeOptionName].(int64) hashFunCode, ok := mh.Names[strings.ToLower(hashFunStr)] if !ok { @@ -270,6 +275,14 @@ only-hash, and progress/status related flags) will change the final hash. opts = append(opts, options.Unixfs.PeerId(peerId)) } + // TODO mode mtime + // if mode != 0 { + // opts = append(opts, options.Unixfs.Mode(os.FileMode(mode))) + // } + // if mtime != 0 { + // opts = append(opts, options.Unixfs.Mtime(mtime, uint32(mtimeNsecs))) + // } + opts = append(opts, nil) // events option placeholder var added int diff --git a/core/coreapi/unixfs.go b/core/coreapi/unixfs.go index 5c201c4df..71ebb020b 100644 --- a/core/coreapi/unixfs.go +++ b/core/coreapi/unixfs.go @@ -53,7 +53,7 @@ func getOrCreateNilNode() (*core.IpfsNode, error) { return } node, err := core.NewNode(context.Background(), &core.BuildCfg{ - //TODO: need this to be true or all files + // TODO: need this to be true or all files // hashed will be stored in memory! NilRepo: true, }) @@ -82,10 +82,10 @@ func (api *UnixfsAPI) Add(ctx context.Context, filesNode files.Node, opts ...opt // check if repo will exceed storage limit if added // TODO: this doesn't handle the case if the hashed file is already in blocks (deduplicated) // TODO: conditional GC is disabled due to it is somehow not possible to pass the size to the daemon - //if err := corerepo.ConditionalGC(req.Context(), n, uint64(size)); err != nil { + // if err := corerepo.ConditionalGC(req.Context(), n, uint64(size)); err != nil { // res.SetError(err, cmds.ErrNormal) // return - //} + // } if settings.NoCopy && !(cfg.Experimental.FilestoreEnabled || cfg.Experimental.UrlstoreEnabled) { return nil, fmt.Errorf(`either the filestore or the urlstore must be enabled to use nocopy @@ -153,6 +153,10 @@ func (api *UnixfsAPI) Add(ctx context.Context, filesNode files.Node, opts ...opt fileAdder.NoCopy = settings.NoCopy fileAdder.CidBuilder = prefix + // TODO mode mtime + // fileAdder.FileMode = settings.FileMode + // fileAdder.FileMtime = settings.Mtime + switch settings.Layout { case options.BalancedLayout: // Default @@ -684,7 +688,7 @@ func (api *UnixfsAPI) lsFromLinksAsync(ctx context.Context, dir uio.Directory, s defer close(out) for l := range dir.EnumLinksAsync(ctx) { select { - case out <- api.processLink(ctx, l, settings): //TODO: perf: processing can be done in background and in parallel + case out <- api.processLink(ctx, l, settings): // TODO: perf: processing can be done in background and in parallel case <-ctx.Done(): return } @@ -699,7 +703,7 @@ func (api *UnixfsAPI) lsFromLinks(ctx context.Context, ndlinks []*ipld.Link, set for _, l := range ndlinks { lr := ft.LinkResult{Link: &ipld.Link{Name: l.Name, Size: l.Size, Cid: l.Cid}} - links <- api.processLink(ctx, lr, settings) //TODO: can be parallel if settings.Async + links <- api.processLink(ctx, lr, settings) // TODO: can be parallel if settings.Async } close(links) return links, nil diff --git a/core/coreunix/add.go b/core/coreunix/add.go index 8cfdb44c1..8be7c5437 100644 --- a/core/coreunix/add.go +++ b/core/coreunix/add.go @@ -6,8 +6,10 @@ import ( "errors" "fmt" "io" + "os" gopath "path" "strconv" + "time" chunker "github.com/bittorrent/go-btfs-chunker" files "github.com/bittorrent/go-btfs-files" @@ -94,6 +96,9 @@ type Adder struct { liveNodes uint64 TokenMetadata string PinDuration int64 + + FileMode os.FileMode + FileMtime time.Time } func (adder *Adder) GcLocker() bstore.GCLocker { @@ -176,6 +181,9 @@ func (adder *Adder) add(reader io.Reader, dirTreeBytes []byte) (ipld.Node, error CidBuilder: adder.CidBuilder, TokenMetadata: metaBytes, ChunkSize: chunkSize, + // TODO fileModeļ¼ŒfileMtime + // FileMode: adder.FileMode, + // FileModTime: adder.FileMtime, } db, err := params.New(chnk) From 6bc9e2d2dd950948285d2bd6bacafcdd425742db Mon Sep 17 00:00:00 2001 From: cody Date: Tue, 26 Nov 2024 16:20:58 +0800 Subject: [PATCH 02/10] feat: add mtime and mode for file command. --- core/commands/add.go | 48 +++++++++++++++++++++++++++++++++--------- core/commands/ls.go | 5 +++++ core/coreapi/unixfs.go | 5 ++--- core/coreunix/add.go | 6 +++--- 4 files changed, 48 insertions(+), 16 deletions(-) diff --git a/core/commands/add.go b/core/commands/add.go index a3cc8e1d1..f65136dae 100644 --- a/core/commands/add.go +++ b/core/commands/add.go @@ -8,7 +8,9 @@ import ( "math/big" "os" "path" + "strconv" "strings" + "time" "github.com/bittorrent/go-btfs/chain/abi" chainconfig "github.com/bittorrent/go-btfs/chain/config" @@ -32,11 +34,30 @@ import ( // ErrDepthLimitExceeded indicates that the max depth has been exceeded. var ErrDepthLimitExceeded = fmt.Errorf("depth limit exceeded") +type TimeParts struct { + t *time.Time +} + +func (t TimeParts) MarshalJSON() ([]byte, error) { + return t.t.MarshalJSON() +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +// The time is expected to be a quoted string in RFC 3339 format. +func (t *TimeParts) UnmarshalJSON(data []byte) (err error) { + // Fractional seconds are handled implicitly by Parse. + tt, err := time.Parse("\"2006-01-02T15:04:05Z\"", string(data)) + *t = TimeParts{&tt} + return +} + type AddEvent struct { Name string Hash string `json:",omitempty"` Bytes int64 `json:",omitempty"` Size string `json:",omitempty"` + Mode string `json:",omitempty"` + Mtime int64 `json:",omitempty"` } const ( @@ -170,6 +191,8 @@ only-hash, and progress/status related flags) will change the final hash. cmds.StringOption(peerIdName, "The peer id to encrypt the file."), cmds.IntOption(pinDurationCountOptionName, "d", "Duration for which the object is pinned in days.").WithDefault(0), cmds.BoolOption(uploadToBlockchainOptionName, "add file meta to blockchain").WithDefault(false), + cmds.UintOption(modeOptionName, "Custom POSIX file mode to store in created UnixFS entries. Disables raw-leaves. (experimental)"), + cmds.Int64Option(mtimeOptionName, "Custom POSIX modification time to store in created UnixFS entries (seconds before or after the Unix Epoch). Disables raw-leaves. (experimental)"), }, PreRun: func(req *cmds.Request, env cmds.Environment) error { quiet, _ := req.Options[quietOptionName].(bool) @@ -216,7 +239,6 @@ only-hash, and progress/status related flags) will change the final hash. peerId, _ := req.Options[peerIdName].(string) pinDuration, _ := req.Options[pinDurationCountOptionName].(int) uploadToBlockchain, _ := req.Options[uploadToBlockchainOptionName].(bool) - // TODO mode mtime mode, _ := req.Options[modeOptionName].(uint) mtime, _ := req.Options[mtimeOptionName].(int64) @@ -275,13 +297,12 @@ only-hash, and progress/status related flags) will change the final hash. opts = append(opts, options.Unixfs.PeerId(peerId)) } - // TODO mode mtime - // if mode != 0 { - // opts = append(opts, options.Unixfs.Mode(os.FileMode(mode))) - // } - // if mtime != 0 { - // opts = append(opts, options.Unixfs.Mtime(mtime, uint32(mtimeNsecs))) - // } + if mode != 0 { + opts = append(opts, options.Unixfs.Mode(os.FileMode(mode))) + } + if mtime != 0 { + opts = append(opts, options.Unixfs.Mtime(mtime)) + } opts = append(opts, nil) // events option placeholder @@ -317,12 +338,19 @@ only-hash, and progress/status related flags) will change the final hash. output.Name = path.Join(addit.Name(), output.Name) } - if err := res.Emit(&AddEvent{ + addEvent := AddEvent{ Name: output.Name, Hash: h, Bytes: output.Bytes, Size: output.Size, - }); err != nil { + Mtime: output.Mtime, + } + + if output.Mode != 0 { + addEvent.Mode = "0" + strconv.FormatUint(uint64(output.Mode), 8) + } + + if err := res.Emit(&addEvent); err != nil { return err } } diff --git a/core/commands/ls.go b/core/commands/ls.go index 7ad1c675d..29e6df1f4 100644 --- a/core/commands/ls.go +++ b/core/commands/ls.go @@ -6,6 +6,7 @@ import ( "os" "sort" "text/tabwriter" + "time" cmdenv "github.com/bittorrent/go-btfs/core/commands/cmdenv" @@ -23,6 +24,8 @@ type LsLink struct { Size uint64 Type unixfs_pb.Data_DataType Target string + Mode os.FileMode + Mtime time.Time } // LsObject is an element of LsOutput @@ -158,6 +161,8 @@ The JSON output contains type information. Size: link.Size, Type: ftype, Target: link.Target, + Mode: link.Mode, + Mtime: link.ModTime, } if err := processLink(paths[i], lsLink); err != nil { return err diff --git a/core/coreapi/unixfs.go b/core/coreapi/unixfs.go index 71ebb020b..4fb1cbe4c 100644 --- a/core/coreapi/unixfs.go +++ b/core/coreapi/unixfs.go @@ -153,9 +153,8 @@ func (api *UnixfsAPI) Add(ctx context.Context, filesNode files.Node, opts ...opt fileAdder.NoCopy = settings.NoCopy fileAdder.CidBuilder = prefix - // TODO mode mtime - // fileAdder.FileMode = settings.FileMode - // fileAdder.FileMtime = settings.Mtime + fileAdder.FileMode = settings.Mode + fileAdder.FileMtime = settings.Mtime switch settings.Layout { case options.BalancedLayout: diff --git a/core/coreunix/add.go b/core/coreunix/add.go index 8be7c5437..fec2075e8 100644 --- a/core/coreunix/add.go +++ b/core/coreunix/add.go @@ -181,9 +181,9 @@ func (adder *Adder) add(reader io.Reader, dirTreeBytes []byte) (ipld.Node, error CidBuilder: adder.CidBuilder, TokenMetadata: metaBytes, ChunkSize: chunkSize, - // TODO fileModeļ¼ŒfileMtime - // FileMode: adder.FileMode, - // FileModTime: adder.FileMtime, + + FileMode: adder.FileMode, + FileModTime: adder.FileMtime, } db, err := params.New(chnk) From dbeea51b00bdf0c92ce4c0605379029d9e9992dd Mon Sep 17 00:00:00 2001 From: cody Date: Thu, 28 Nov 2024 17:54:54 +0800 Subject: [PATCH 03/10] feat: files command add mode and mtime --- core/commands/add.go | 9 +++++++++ core/commands/files.go | 26 ++++++++++++++++++++++++++ core/commands/ls.go | 8 +++++++- core/commands/unixfs/ls.go | 4 ++++ core/coreapi/unixfs.go | 4 ++++ core/coreunix/add.go | 14 ++++++++++++-- core/coreunix/reed_solomon_add.go | 16 +++++++++++++--- 7 files changed, 75 insertions(+), 6 deletions(-) diff --git a/core/commands/add.go b/core/commands/add.go index f65136dae..5bb757c4a 100644 --- a/core/commands/add.go +++ b/core/commands/add.go @@ -82,6 +82,8 @@ const ( peerIdName = "peer-id" pinDurationCountOptionName = "pin-duration-count" uploadToBlockchainOptionName = "to-blockchain" + preserveModeOptionName = "preserve-mode" + preserveMtimeOptionName = "preserve-mtime" modeOptionName = "mode" mtimeOptionName = "mtime" ) @@ -191,6 +193,8 @@ only-hash, and progress/status related flags) will change the final hash. cmds.StringOption(peerIdName, "The peer id to encrypt the file."), cmds.IntOption(pinDurationCountOptionName, "d", "Duration for which the object is pinned in days.").WithDefault(0), cmds.BoolOption(uploadToBlockchainOptionName, "add file meta to blockchain").WithDefault(false), + cmds.BoolOption(preserveModeOptionName, "Apply existing POSIX permissions to created UnixFS entries. Disables raw-leaves. (experimental)"), + cmds.BoolOption(preserveMtimeOptionName, "Apply existing POSIX modification time to created UnixFS entries. Disables raw-leaves. (experimental)"), cmds.UintOption(modeOptionName, "Custom POSIX file mode to store in created UnixFS entries. Disables raw-leaves. (experimental)"), cmds.Int64Option(mtimeOptionName, "Custom POSIX modification time to store in created UnixFS entries (seconds before or after the Unix Epoch). Disables raw-leaves. (experimental)"), }, @@ -239,6 +243,8 @@ only-hash, and progress/status related flags) will change the final hash. peerId, _ := req.Options[peerIdName].(string) pinDuration, _ := req.Options[pinDurationCountOptionName].(int) uploadToBlockchain, _ := req.Options[uploadToBlockchainOptionName].(bool) + preserveMode, _ := req.Options[preserveModeOptionName].(bool) + preserveMtime, _ := req.Options[preserveMtimeOptionName].(bool) mode, _ := req.Options[modeOptionName].(uint) mtime, _ := req.Options[mtimeOptionName].(int64) @@ -277,6 +283,9 @@ only-hash, and progress/status related flags) will change the final hash. options.Unixfs.TokenMetadata(tokenMetadata), options.Unixfs.PinDuration(int64(pinDuration)), + + options.Unixfs.PreserveMode(preserveMode), + options.Unixfs.PreserveMtime(preserveMtime), } if cidVerSet { diff --git a/core/commands/files.go b/core/commands/files.go index 1b9768bf1..d20e86828 100644 --- a/core/commands/files.go +++ b/core/commands/files.go @@ -8,7 +8,9 @@ import ( "os" gopath "path" "sort" + "strconv" "strings" + "time" "github.com/bittorrent/go-btfs/core" "github.com/bittorrent/go-btfs/core/commands/cmdenv" @@ -101,6 +103,8 @@ type statOutput struct { WithLocality bool `json:",omitempty"` Local bool `json:",omitempty"` SizeLocal uint64 `json:",omitempty"` + Mode uint32 `json:",omitempty"` + Mtime int64 `json:",omitempty"` } const ( @@ -112,6 +116,7 @@ Type: ` filesFormatOptionName = "format" filesSizeOptionName = "size" filesWithLocalOptionName = "with-local" + filesStatUnspecified = "not set" ) var filesStatCmd = &cmds.Command{ @@ -196,12 +201,24 @@ var filesStatCmd = &cmds.Command{ }, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *statOutput) error { + mode, modeo := filesStatUnspecified, filesStatUnspecified + if out.Mode != 0 { + mode = strings.ToLower(os.FileMode(out.Mode).String()) + modeo = "0" + strconv.FormatInt(int64(out.Mode&0x1FF), 8) + } + mtime := filesStatUnspecified + if out.Mtime > 0 { + mtime = time.Unix(out.Mtime, 0).UTC().Format("2 Jan 2006, 15:04:05 MST") + } s, _ := statGetFormatOptions(req) s = strings.Replace(s, "", out.Hash, -1) s = strings.Replace(s, "", fmt.Sprintf("%d", out.Size), -1) s = strings.Replace(s, "", fmt.Sprintf("%d", out.CumulativeSize), -1) s = strings.Replace(s, "", fmt.Sprintf("%d", out.Blocks), -1) s = strings.Replace(s, "", out.Type, -1) + s = strings.Replace(s, "", mode, -1) + s = strings.Replace(s, "", modeo, -1) + s = strings.Replace(s, "", mtime, -1) fmt.Fprintln(w, s) @@ -267,12 +284,21 @@ func statNode(nd ipld.Node, enc cidenc.Encoder) (*statOutput, error) { return nil, fmt.Errorf("unrecognized node type: %s", d.Type()) } + var mode uint32 + if m := d.Mode(); m != 0 { + mode = uint32(m) + } else if d.Type() == ft.TSymlink { + mode = uint32(os.ModeSymlink | 0x1FF) + } + return &statOutput{ Hash: enc.Encode(c), Blocks: len(nd.Links()), Size: d.FileSize(), CumulativeSize: cumulsize, Type: ndtype, + Mode: mode, + Mtime: d.ModTime().Unix(), }, nil case *dag.RawNode: return &statOutput{ diff --git a/core/commands/ls.go b/core/commands/ls.go index 29e6df1f4..900477ac9 100644 --- a/core/commands/ls.go +++ b/core/commands/ls.go @@ -46,6 +46,7 @@ const ( lsResolveTypeOptionName = "resolve-type" lsSizeOptionName = "size" lsStreamOptionName = "stream" + lsMTimeOptionName = "mtime" ) var LsCmd = &cmds.Command{ @@ -69,6 +70,7 @@ The JSON output contains type information. cmds.BoolOption(lsResolveTypeOptionName, "Resolve linked objects to find out their types.").WithDefault(true), cmds.BoolOption(lsSizeOptionName, "Resolve linked objects to find out their file size.").WithDefault(true), cmds.BoolOption(lsStreamOptionName, "s", "Enable experimental streaming of directory entries as they are traversed."), + cmds.BoolOption(lsMTimeOptionName, "t", "").WithDefault(true), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { api, err := cmdenv.GetApi(env, req) @@ -207,6 +209,7 @@ func tabularOutput(req *cmds.Request, w io.Writer, out *LsOutput, lastObjectHash headers, _ := req.Options[lsHeadersOptionNameTime].(bool) stream, _ := req.Options[lsStreamOptionName].(bool) size, _ := req.Options[lsSizeOptionName].(bool) + mtime, _ := req.Options[lsMTimeOptionName].(bool) // in streaming mode we can't automatically align the tabs // so we take a best guess var minTabWidth int @@ -234,6 +237,9 @@ func tabularOutput(req *cmds.Request, w io.Writer, out *LsOutput, lastObjectHash if size { s = "Hash\tSize\tName" } + if mtime { + s = "Hash\tSize\tName\tTime" + } fmt.Fprintln(tw, s) } lastObjectHash = object.Hash @@ -256,7 +262,7 @@ func tabularOutput(req *cmds.Request, w io.Writer, out *LsOutput, lastObjectHash } } - fmt.Fprintf(tw, s, link.Hash, link.Size, link.Name) + fmt.Fprintf(tw, s, link.Hash, link.Size, link.Name, link.Mode.String(), link.Mtime.String()) } } tw.Flush() diff --git a/core/commands/unixfs/ls.go b/core/commands/unixfs/ls.go index f8df8a5b7..e9f49cc62 100644 --- a/core/commands/unixfs/ls.go +++ b/core/commands/unixfs/ls.go @@ -115,6 +115,7 @@ possible, please use 'btfs ls' instead. return merkledag.ErrNotProtobuf } + // TODO mtime and mode unixFSNode, err := unixfs.FSNodeFromBytes(ndpb.Data()) if err != nil { return err @@ -122,6 +123,9 @@ possible, please use 'btfs ls' instead. t := unixFSNode.Type() + fmt.Println(unixFSNode.Mode().String(), "---------") + fmt.Println(unixFSNode.ModTime().Unix(), "-----------") + output.Objects[hash] = &LsObject{ Hash: c.String(), Type: t.String(), diff --git a/core/coreapi/unixfs.go b/core/coreapi/unixfs.go index 4fb1cbe4c..845f08ab9 100644 --- a/core/coreapi/unixfs.go +++ b/core/coreapi/unixfs.go @@ -153,6 +153,8 @@ func (api *UnixfsAPI) Add(ctx context.Context, filesNode files.Node, opts ...opt fileAdder.NoCopy = settings.NoCopy fileAdder.CidBuilder = prefix + fileAdder.PreserveMode = settings.PreserveMode + fileAdder.PreserveMtime = settings.PreserveMtime fileAdder.FileMode = settings.Mode fileAdder.FileMtime = settings.Mtime @@ -674,6 +676,8 @@ func (api *UnixfsAPI) processLink(ctx context.Context, linkres ft.LinkResult, se lnk.Target = string(d.Data()) } lnk.Size = d.FileSize() + lnk.Mode = d.Mode() + lnk.ModTime = d.ModTime() } } diff --git a/core/coreunix/add.go b/core/coreunix/add.go index fec2075e8..cf53fb1fc 100644 --- a/core/coreunix/add.go +++ b/core/coreunix/add.go @@ -97,8 +97,10 @@ type Adder struct { TokenMetadata string PinDuration int64 - FileMode os.FileMode - FileMtime time.Time + PreserveMtime bool + PreserveMode bool + FileMode os.FileMode + FileMtime time.Time } func (adder *Adder) GcLocker() bstore.GCLocker { @@ -472,6 +474,14 @@ func (adder *Adder) addFileNode(ctx context.Context, path string, file files.Nod return err } + if adder.PreserveMtime { + adder.FileMtime = file.ModTime() + } + + if adder.PreserveMode { + adder.FileMode = file.Mode() + } + if adder.liveNodes >= liveCacheSize { // TODO: A smarter cache that uses some sort of lru cache with an eviction handler mr, err := adder.mfsRoot() diff --git a/core/coreunix/reed_solomon_add.go b/core/coreunix/reed_solomon_add.go index 0fca75ad9..6acb9d7fd 100644 --- a/core/coreunix/reed_solomon_add.go +++ b/core/coreunix/reed_solomon_add.go @@ -6,7 +6,9 @@ import ( "errors" "io" "io/ioutil" + "os" gopath "path" + "time" "container/list" "encoding/json" @@ -134,6 +136,14 @@ func (rsadder *ReedSolomonAdder) addFileNode(ctx context.Context, path string, f return nil, err } + if rsadder.FileMtime.Unix() != 0 { + rsadder.FileMtime = time.Now() + } + + if rsadder.FileMode != os.FileMode(0) { + rsadder.FileMode = file.Mode() + } + if rsadder.liveNodes >= liveCacheSize { // TODO: flush free memory rsadder.mfsRoot()'s FlushMemFree() to flush free memory log.Info("rsadder liveNodes >= liveCacheSize, needs flushing") @@ -162,7 +172,7 @@ func (rsadder *ReedSolomonAdder) addDir(ctx context.Context, path string, dir fi NodePath: path, NodeName: dstName}, // Kept the following line for possible use cases than `btfs get` - //Directory: dir, + // Directory: dir, } it := dir.Entries() @@ -195,7 +205,7 @@ func (rsadder *ReedSolomonAdder) addSymlink(path string, l *files.Symlink) (uio. NodeName: dstName}, Data: l.Target, // Kept the following line for the same reason as DirNode. - //Symlink: *l, + // Symlink: *l, }, nil } @@ -251,7 +261,7 @@ func (rsadder *ReedSolomonAdder) addFile(path string, file files.File) (uio.Node StartOffset: currentOffset, }, // Kept the following line for the same reason as DirNode. - //File: file, + // File: file, } return node, nil } From 347e9d5103d83fcac2714c988edb551d4fd6449a Mon Sep 17 00:00:00 2001 From: cody Date: Tue, 3 Dec 2024 14:27:10 +0800 Subject: [PATCH 04/10] feat: file stats ouput add mtime and mode. --- core/commands/files.go | 39 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/core/commands/files.go b/core/commands/files.go index d20e86828..0d4a1dc47 100644 --- a/core/commands/files.go +++ b/core/commands/files.go @@ -2,6 +2,7 @@ package commands import ( "context" + "encoding/json" "errors" "fmt" "io" @@ -107,12 +108,48 @@ type statOutput struct { Mtime int64 `json:",omitempty"` } +func (s *statOutput) MarshalJSON() ([]byte, error) { + type so statOutput + out := &struct { + *so + Mode string `json:",omitempty"` + }{so: (*so)(s)} + + if s.Mode != 0 { + out.Mode = fmt.Sprintf("%04o", s.Mode) + } + return json.Marshal(out) +} + +func (s *statOutput) UnmarshalJSON(data []byte) error { + var err error + type so statOutput + tmp := &struct { + *so + Mode string `json:",omitempty"` + }{so: (*so)(s)} + + if err := json.Unmarshal(data, &tmp); err != nil { + return err + } + + if tmp.Mode != "" { + mode, err := strconv.ParseUint(tmp.Mode, 8, 32) + if err == nil { + s.Mode = uint32(mode) + } + } + return err +} + const ( defaultStatFormat = ` Size: CumulativeSize: ChildBlocks: -Type: ` +Type: +Mode: () +Mtime: ` filesFormatOptionName = "format" filesSizeOptionName = "size" filesWithLocalOptionName = "with-local" From 411e56d2670334c6e749aba78f6cd3628a59352b Mon Sep 17 00:00:00 2001 From: cody Date: Tue, 3 Dec 2024 17:21:14 +0800 Subject: [PATCH 05/10] feat: ls add mtime and mode. --- core/commands/ls.go | 90 ++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 81 insertions(+), 9 deletions(-) diff --git a/core/commands/ls.go b/core/commands/ls.go index 900477ac9..c8758f9cf 100644 --- a/core/commands/ls.go +++ b/core/commands/ls.go @@ -1,10 +1,12 @@ package commands import ( + "encoding/json" "fmt" "io" "os" "sort" + "strconv" "text/tabwriter" "time" @@ -41,12 +43,59 @@ type LsOutput struct { Objects []LsObject } +func (s *LsLink) MarshalJSON() ([]byte, error) { + type so LsLink + out := &struct { + *so + Mode string `json:",omitempty"` + Mtime string `json:",omitempty"` + }{so: (*so)(s)} + + if s.Mode != 0 { + out.Mode = fmt.Sprintf("%04o", s.Mode) + } + if s.Mtime.Unix() > 0 { + out.Mtime = s.Mtime.UTC().Format("2 Jan 2006, 15:04:05 MST") + } + return json.Marshal(out) +} + +func (s *LsLink) UnmarshalJSON(data []byte) error { + var err error + type so LsLink + tmp := &struct { + *so + Mode string `json:",omitempty"` + Mtime string `json:",omitempty"` + }{so: (*so)(s)} + + if err := json.Unmarshal(data, &tmp); err != nil { + return err + } + + if tmp.Mode != "" { + mode, err := strconv.ParseUint(tmp.Mode, 8, 32) + if err == nil { + s.Mode = os.FileMode(mode) + } + } + + if tmp.Mtime != "" { + t, err := time.Parse("2 Jan 2006, 15:04:05 MST", tmp.Mtime) + if err == nil { + s.Mtime = t + } + } + return err +} + const ( lsHeadersOptionNameTime = "headers" lsResolveTypeOptionName = "resolve-type" lsSizeOptionName = "size" lsStreamOptionName = "stream" lsMTimeOptionName = "mtime" + lsModeOptionName = "mode" ) var LsCmd = &cmds.Command{ @@ -70,7 +119,8 @@ The JSON output contains type information. cmds.BoolOption(lsResolveTypeOptionName, "Resolve linked objects to find out their types.").WithDefault(true), cmds.BoolOption(lsSizeOptionName, "Resolve linked objects to find out their file size.").WithDefault(true), cmds.BoolOption(lsStreamOptionName, "s", "Enable experimental streaming of directory entries as they are traversed."), - cmds.BoolOption(lsMTimeOptionName, "t", "").WithDefault(true), + cmds.BoolOption(lsMTimeOptionName, "t", "Print modification time."), + cmds.BoolOption(lsModeOptionName, "m", "Print mode."), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { api, err := cmdenv.GetApi(env, req) @@ -210,6 +260,7 @@ func tabularOutput(req *cmds.Request, w io.Writer, out *LsOutput, lastObjectHash stream, _ := req.Options[lsStreamOptionName].(bool) size, _ := req.Options[lsSizeOptionName].(bool) mtime, _ := req.Options[lsMTimeOptionName].(bool) + mode, _ := req.Options[lsModeOptionName].(bool) // in streaming mode we can't automatically align the tabs // so we take a best guess var minTabWidth int @@ -237,9 +288,10 @@ func tabularOutput(req *cmds.Request, w io.Writer, out *LsOutput, lastObjectHash if size { s = "Hash\tSize\tName" } - if mtime { - s = "Hash\tSize\tName\tTime" - } + + s = buildHeader(mode, "Mode", s) + s = buildHeader(mtime, "Mtime", s) + fmt.Fprintln(tw, s) } lastObjectHash = object.Hash @@ -250,21 +302,41 @@ func tabularOutput(req *cmds.Request, w io.Writer, out *LsOutput, lastObjectHash switch link.Type { case unixfs.TDirectory, unixfs.THAMTShard, unixfs.TMetadata: if size { - s = "%[1]s\t-\t%[3]s/\n" + s = "%[1]s\t-\t%[3]s/" } else { - s = "%[1]s\t%[3]s/\n" + s = "%[1]s\t%[3]s/" } + s = buildString(mode, s, 4) + s = buildString(mtime, s, 5) + s = s + "\n" default: if size { - s = "%s\t%v\t%s\n" + s = "%[1]s\t%[2]v\t%[3]s" } else { - s = "%[1]s\t%[3]s\n" + s = "%[1]s\t%[3]s" } + s = buildString(mode, s, 4) + s = buildString(mtime, s, 5) + s = s + "\n" } - fmt.Fprintf(tw, s, link.Hash, link.Size, link.Name, link.Mode.String(), link.Mtime.String()) + fmt.Fprintf(tw, s, link.Hash, link.Size, link.Name, link.Mode, link.Mtime) } } tw.Flush() return lastObjectHash } + +func buildString(set bool, s string, index int) string { + if set { + return fmt.Sprintf("%s\t%%[%d]s", s, index) + } + return s +} + +func buildHeader(set bool, name, s string) string { + if set { + return s + "\t" + name + } + return s +} From f0c8ca2d1b36f9b21b75cb07bb4fcda6c55e50ab Mon Sep 17 00:00:00 2001 From: cody Date: Wed, 4 Dec 2024 16:26:35 +0800 Subject: [PATCH 06/10] feat: add file rbset can't with preserve-motime. --- core/commands/add.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/core/commands/add.go b/core/commands/add.go index 5bb757c4a..a25773c35 100644 --- a/core/commands/add.go +++ b/core/commands/add.go @@ -296,6 +296,19 @@ only-hash, and progress/status related flags) will change the final hash. opts = append(opts, options.Unixfs.RawLeaves(rawblks)) } + // Storing optional mode or mtime (UnixFS 1.5) requires root block + // to always be 'dag-pb' and not 'raw'. Below adjusts raw-leaves setting, if possible. + if preserveMode || preserveMtime || mode != 0 || mtime != 0 { + // Error if --raw-leaves flag was explicitly passed by the user. + // (let user make a decision to manually disable it and retry) + if rbset && rawblks { + return fmt.Errorf("%s can't be used with UnixFS metadata like mode or modification time", rawLeavesOptionName) + } + // No explicit preference from user, disable raw-leaves and continue + rbset = true + rawblks = false + } + if trickle { opts = append(opts, options.Unixfs.Layout(options.TrickleLayout)) } From cadc903f682e7ee98b43372ac495c4ed1d3e3f01 Mon Sep 17 00:00:00 2001 From: cody Date: Thu, 5 Dec 2024 16:01:23 +0800 Subject: [PATCH 07/10] feat: ls command out format mode and mtime. --- core/commands/ls.go | 59 ++++++++------------------------------------- 1 file changed, 10 insertions(+), 49 deletions(-) diff --git a/core/commands/ls.go b/core/commands/ls.go index c8758f9cf..6afba970a 100644 --- a/core/commands/ls.go +++ b/core/commands/ls.go @@ -1,12 +1,10 @@ package commands import ( - "encoding/json" "fmt" "io" "os" "sort" - "strconv" "text/tabwriter" "time" @@ -43,52 +41,6 @@ type LsOutput struct { Objects []LsObject } -func (s *LsLink) MarshalJSON() ([]byte, error) { - type so LsLink - out := &struct { - *so - Mode string `json:",omitempty"` - Mtime string `json:",omitempty"` - }{so: (*so)(s)} - - if s.Mode != 0 { - out.Mode = fmt.Sprintf("%04o", s.Mode) - } - if s.Mtime.Unix() > 0 { - out.Mtime = s.Mtime.UTC().Format("2 Jan 2006, 15:04:05 MST") - } - return json.Marshal(out) -} - -func (s *LsLink) UnmarshalJSON(data []byte) error { - var err error - type so LsLink - tmp := &struct { - *so - Mode string `json:",omitempty"` - Mtime string `json:",omitempty"` - }{so: (*so)(s)} - - if err := json.Unmarshal(data, &tmp); err != nil { - return err - } - - if tmp.Mode != "" { - mode, err := strconv.ParseUint(tmp.Mode, 8, 32) - if err == nil { - s.Mode = os.FileMode(mode) - } - } - - if tmp.Mtime != "" { - t, err := time.Parse("2 Jan 2006, 15:04:05 MST", tmp.Mtime) - if err == nil { - s.Mtime = t - } - } - return err -} - const ( lsHeadersOptionNameTime = "headers" lsResolveTypeOptionName = "resolve-type" @@ -320,7 +272,16 @@ func tabularOutput(req *cmds.Request, w io.Writer, out *LsOutput, lastObjectHash s = s + "\n" } - fmt.Fprintf(tw, s, link.Hash, link.Size, link.Name, link.Mode, link.Mtime) + modeS := "-" + mtimeS := "-" + + if link.Mode != 0 { + modeS = link.Mode.String() + } + if link.Mtime.Unix() != 0 { + mtimeS = link.Mtime.Format("2 Jan 2006, 15:04:05 MST") + } + fmt.Fprintf(tw, s, link.Hash, link.Size, link.Name, modeS, mtimeS) } } tw.Flush() From 158f2986afe4e9ab932a1c18f85364c0cedb25ee Mon Sep 17 00:00:00 2001 From: cody Date: Fri, 6 Dec 2024 11:02:45 +0800 Subject: [PATCH 08/10] feat: update go.mod for mtime and mode --- go.mod | 6 +++--- go.sum | 11 ++++++----- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index e57aa9916..fb141131b 100644 --- a/go.mod +++ b/go.mod @@ -11,13 +11,13 @@ require ( github.com/bittorrent/go-btfs-cmds v0.3.0 github.com/bittorrent/go-btfs-common v0.9.1-0.20240823025041-824b78e1c643 github.com/bittorrent/go-btfs-config v0.13.3-0.20240822075319-c9a0e978f0b2 - github.com/bittorrent/go-btfs-files v0.3.1 + github.com/bittorrent/go-btfs-files v0.3.2-0.20241206024949-1343cef68952 github.com/bittorrent/go-btns v0.2.0 github.com/bittorrent/go-common/v2 v2.4.0 github.com/bittorrent/go-eccrypto v0.1.0 github.com/bittorrent/go-mfs v0.4.0 - github.com/bittorrent/go-unixfs v0.7.0 - github.com/bittorrent/interface-go-btfs-core v0.8.2 + github.com/bittorrent/go-unixfs v0.7.1-0.20241206024825-93b8c4b57d53 + github.com/bittorrent/interface-go-btfs-core v0.8.3-0.20241206024747-09777076f753 github.com/bittorrent/protobuf v1.4.0 github.com/blang/semver v3.5.1+incompatible github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 diff --git a/go.sum b/go.sum index 67f45ec2d..113cb5adb 100644 --- a/go.sum +++ b/go.sum @@ -215,8 +215,8 @@ github.com/bittorrent/go-btfs-common v0.9.1-0.20240823025041-824b78e1c643/go.mod github.com/bittorrent/go-btfs-config v0.13.3-0.20240822075319-c9a0e978f0b2 h1:zbmWBI69iBg6uRriAJ496NQfeEWlAwOrhSgsNK1/lLg= github.com/bittorrent/go-btfs-config v0.13.3-0.20240822075319-c9a0e978f0b2/go.mod h1:ahh4rLSA+sl3FsHJ/ma6OOl6yu4ogCmeumzgBxKSIjg= github.com/bittorrent/go-btfs-files v0.3.0/go.mod h1:ylMf73m6oK94hL7VPblY1ZZpePsr6XbPV4BaNUwGZR0= -github.com/bittorrent/go-btfs-files v0.3.1 h1:esq3j+6FtZ+SlaxKjVtiYgvXk/SWUiTcv0Q1MeJoPnQ= -github.com/bittorrent/go-btfs-files v0.3.1/go.mod h1:ylMf73m6oK94hL7VPblY1ZZpePsr6XbPV4BaNUwGZR0= +github.com/bittorrent/go-btfs-files v0.3.2-0.20241206024949-1343cef68952 h1:UPb87q3LxRtDqKyoCxXAHKOkMFe4UGvzuMQcTTROnD0= +github.com/bittorrent/go-btfs-files v0.3.2-0.20241206024949-1343cef68952/go.mod h1:ylMf73m6oK94hL7VPblY1ZZpePsr6XbPV4BaNUwGZR0= github.com/bittorrent/go-btns v0.2.0 h1:OMpxUiRbtb/PRTK/z/flxcwOfTvNKMsTLOubYFhKy1s= github.com/bittorrent/go-btns v0.2.0/go.mod h1:+Cinr/1Jl7V/Pqgz+vbOdHXkLVFbMqjypmbAv8QiQPs= github.com/bittorrent/go-common/v2 v2.4.0 h1:u0jldKnQteTPQDNKj5GUBOUj2Tswn0+GfWN7yq2QAaY= @@ -227,10 +227,11 @@ github.com/bittorrent/go-mfs v0.4.0 h1:xb7Bxp65LQP8yhflx47ZMuXzIMSSo9ZrasVhroCvR github.com/bittorrent/go-mfs v0.4.0/go.mod h1:w7XQuaSCDsL0MhcMP02ViFJQHYg2tLf+/v0w/m7wMfM= github.com/bittorrent/go-path v0.4.1 h1:9qJe6V2/O3n8Z3tqgN3wgbYcXrcwAv1U3de5xiyYodg= github.com/bittorrent/go-path v0.4.1/go.mod h1:eNLsxJEEMxn/CDzUJ6wuNl+6No6tEUhOZcPKsZsYX0E= -github.com/bittorrent/go-unixfs v0.7.0 h1:2SPuQcAmubJUl+zuKoGWdculoZRn7D0zkDnTZ9pupqo= github.com/bittorrent/go-unixfs v0.7.0/go.mod h1:0UNGV0k5MFsMGOeNjOJFtURcXDFz8bjtyfhcom+vW7A= -github.com/bittorrent/interface-go-btfs-core v0.8.2 h1:iTStlXLoandcKyFruq4U0uVSR3CQU7ey9Lwf8Mu3jw0= -github.com/bittorrent/interface-go-btfs-core v0.8.2/go.mod h1:tQ3d3uI2gH+AO7ikbBwlulRgff0/dzobz9H3SL00yYo= +github.com/bittorrent/go-unixfs v0.7.1-0.20241206024825-93b8c4b57d53 h1:on4IJttnb729mMxjsKVNLGvUH8ZbWyvEMoDQsBkYsTk= +github.com/bittorrent/go-unixfs v0.7.1-0.20241206024825-93b8c4b57d53/go.mod h1:0UNGV0k5MFsMGOeNjOJFtURcXDFz8bjtyfhcom+vW7A= +github.com/bittorrent/interface-go-btfs-core v0.8.3-0.20241206024747-09777076f753 h1:ZI+v7O3LXkH+DWXJiifPVeA9ChYoFl0U5gTbvuI5DgE= +github.com/bittorrent/interface-go-btfs-core v0.8.3-0.20241206024747-09777076f753/go.mod h1:tQ3d3uI2gH+AO7ikbBwlulRgff0/dzobz9H3SL00yYo= github.com/bittorrent/protobuf v1.4.0 h1:3AW4SZUud3/8/orb8O/957CdspwxWjX/qprvF49aQ70= github.com/bittorrent/protobuf v1.4.0/go.mod h1:k2fZczatqZOyvWUezE02Xt5uFcVqdUd1tNeZwXjELCk= github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= From faa20797cb99f7650252dd753581382749d7f7f6 Mon Sep 17 00:00:00 2001 From: cody Date: Fri, 6 Dec 2024 11:09:52 +0800 Subject: [PATCH 09/10] chore: delete debug log. --- core/commands/unixfs/ls.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/core/commands/unixfs/ls.go b/core/commands/unixfs/ls.go index e9f49cc62..f8df8a5b7 100644 --- a/core/commands/unixfs/ls.go +++ b/core/commands/unixfs/ls.go @@ -115,7 +115,6 @@ possible, please use 'btfs ls' instead. return merkledag.ErrNotProtobuf } - // TODO mtime and mode unixFSNode, err := unixfs.FSNodeFromBytes(ndpb.Data()) if err != nil { return err @@ -123,9 +122,6 @@ possible, please use 'btfs ls' instead. t := unixFSNode.Type() - fmt.Println(unixFSNode.Mode().String(), "---------") - fmt.Println(unixFSNode.ModTime().Unix(), "-----------") - output.Objects[hash] = &LsObject{ Hash: c.String(), Type: t.String(), From 8a791dbe68141e8c3b5618cb38b1ce19eedcc014 Mon Sep 17 00:00:00 2001 From: cody Date: Fri, 6 Dec 2024 16:31:08 +0800 Subject: [PATCH 10/10] feat: update go-btfs-files dependency. --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index fb141131b..ccdc5364f 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/bittorrent/go-btfs-cmds v0.3.0 github.com/bittorrent/go-btfs-common v0.9.1-0.20240823025041-824b78e1c643 github.com/bittorrent/go-btfs-config v0.13.3-0.20240822075319-c9a0e978f0b2 - github.com/bittorrent/go-btfs-files v0.3.2-0.20241206024949-1343cef68952 + github.com/bittorrent/go-btfs-files v0.3.2-0.20241206082252-f4c1a3b2c90d github.com/bittorrent/go-btns v0.2.0 github.com/bittorrent/go-common/v2 v2.4.0 github.com/bittorrent/go-eccrypto v0.1.0 diff --git a/go.sum b/go.sum index 113cb5adb..54208a2c4 100644 --- a/go.sum +++ b/go.sum @@ -215,8 +215,8 @@ github.com/bittorrent/go-btfs-common v0.9.1-0.20240823025041-824b78e1c643/go.mod github.com/bittorrent/go-btfs-config v0.13.3-0.20240822075319-c9a0e978f0b2 h1:zbmWBI69iBg6uRriAJ496NQfeEWlAwOrhSgsNK1/lLg= github.com/bittorrent/go-btfs-config v0.13.3-0.20240822075319-c9a0e978f0b2/go.mod h1:ahh4rLSA+sl3FsHJ/ma6OOl6yu4ogCmeumzgBxKSIjg= github.com/bittorrent/go-btfs-files v0.3.0/go.mod h1:ylMf73m6oK94hL7VPblY1ZZpePsr6XbPV4BaNUwGZR0= -github.com/bittorrent/go-btfs-files v0.3.2-0.20241206024949-1343cef68952 h1:UPb87q3LxRtDqKyoCxXAHKOkMFe4UGvzuMQcTTROnD0= -github.com/bittorrent/go-btfs-files v0.3.2-0.20241206024949-1343cef68952/go.mod h1:ylMf73m6oK94hL7VPblY1ZZpePsr6XbPV4BaNUwGZR0= +github.com/bittorrent/go-btfs-files v0.3.2-0.20241206082252-f4c1a3b2c90d h1:4osxro2/lDZ/4wWat3n5kd4dLglnC8n9E2M+4cqIICc= +github.com/bittorrent/go-btfs-files v0.3.2-0.20241206082252-f4c1a3b2c90d/go.mod h1:ylMf73m6oK94hL7VPblY1ZZpePsr6XbPV4BaNUwGZR0= github.com/bittorrent/go-btns v0.2.0 h1:OMpxUiRbtb/PRTK/z/flxcwOfTvNKMsTLOubYFhKy1s= github.com/bittorrent/go-btns v0.2.0/go.mod h1:+Cinr/1Jl7V/Pqgz+vbOdHXkLVFbMqjypmbAv8QiQPs= github.com/bittorrent/go-common/v2 v2.4.0 h1:u0jldKnQteTPQDNKj5GUBOUj2Tswn0+GfWN7yq2QAaY=