Skip to content

Commit

Permalink
Merge pull request #479 from bittorrent/feat/unixfs-add-mtime
Browse files Browse the repository at this point in the history 
Feat/unixfs add mtime
  • Loading branch information
@mengcody
mengcody authored Dec 9, 2024
2 parents 1de8248 + 8a791db commit cb5afd2
Show file tree
Hide file tree
Showing 8 changed files with 230 additions and 24 deletions.
67 changes: 65 additions & 2 deletions core/commands/add.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,9 @@ import (
"math/big"
"os"
"path"
"strconv"
"strings"
"time"

"github.com/bittorrent/go-btfs/chain/abi"
chainconfig "github.com/bittorrent/go-btfs/chain/config"
Expand All @@ -32,11 +34,30 @@ import (
// ErrDepthLimitExceeded indicates that the max depth has been exceeded.
var ErrDepthLimitExceeded = fmt.Errorf("depth limit exceeded")

type TimeParts struct {
t *time.Time
}

func (t TimeParts) MarshalJSON() ([]byte, error) {
return t.t.MarshalJSON()
}

// UnmarshalJSON implements the json.Unmarshaler interface.
// The time is expected to be a quoted string in RFC 3339 format.
func (t *TimeParts) UnmarshalJSON(data []byte) (err error) {
// Fractional seconds are handled implicitly by Parse.
tt, err := time.Parse("\"2006-01-02T15:04:05Z\"", string(data))
*t = TimeParts{&tt}
return
}

type AddEvent struct {
Name string
Hash string `json:",omitempty"`
Bytes int64 `json:",omitempty"`
Size string `json:",omitempty"`
Mode string `json:",omitempty"`
Mtime int64 `json:",omitempty"`
}

const (
Expand All @@ -61,6 +82,10 @@ const (
peerIdName = "peer-id"
pinDurationCountOptionName = "pin-duration-count"
uploadToBlockchainOptionName = "to-blockchain"
preserveModeOptionName = "preserve-mode"
preserveMtimeOptionName = "preserve-mtime"
modeOptionName = "mode"
mtimeOptionName = "mtime"
)

const adderOutChanSize = 8
Expand Down Expand Up @@ -168,6 +193,10 @@ only-hash, and progress/status related flags) will change the final hash.
cmds.StringOption(peerIdName, "The peer id to encrypt the file."),
cmds.IntOption(pinDurationCountOptionName, "d", "Duration for which the object is pinned in days.").WithDefault(0),
cmds.BoolOption(uploadToBlockchainOptionName, "add file meta to blockchain").WithDefault(false),
cmds.BoolOption(preserveModeOptionName, "Apply existing POSIX permissions to created UnixFS entries. Disables raw-leaves. (experimental)"),
cmds.BoolOption(preserveMtimeOptionName, "Apply existing POSIX modification time to created UnixFS entries. Disables raw-leaves. (experimental)"),
cmds.UintOption(modeOptionName, "Custom POSIX file mode to store in created UnixFS entries. Disables raw-leaves. (experimental)"),
cmds.Int64Option(mtimeOptionName, "Custom POSIX modification time to store in created UnixFS entries (seconds before or after the Unix Epoch). Disables raw-leaves. (experimental)"),
},
PreRun: func(req *cmds.Request, env cmds.Environment) error {
quiet, _ := req.Options[quietOptionName].(bool)
Expand Down Expand Up @@ -214,6 +243,10 @@ only-hash, and progress/status related flags) will change the final hash.
peerId, _ := req.Options[peerIdName].(string)
pinDuration, _ := req.Options[pinDurationCountOptionName].(int)
uploadToBlockchain, _ := req.Options[uploadToBlockchainOptionName].(bool)
preserveMode, _ := req.Options[preserveModeOptionName].(bool)
preserveMtime, _ := req.Options[preserveMtimeOptionName].(bool)
mode, _ := req.Options[modeOptionName].(uint)
mtime, _ := req.Options[mtimeOptionName].(int64)

hashFunCode, ok := mh.Names[strings.ToLower(hashFunStr)]
if !ok {
Expand Down Expand Up @@ -250,6 +283,9 @@ only-hash, and progress/status related flags) will change the final hash.

options.Unixfs.TokenMetadata(tokenMetadata),
options.Unixfs.PinDuration(int64(pinDuration)),

options.Unixfs.PreserveMode(preserveMode),
options.Unixfs.PreserveMtime(preserveMtime),
}

if cidVerSet {
Expand All @@ -260,6 +296,19 @@ only-hash, and progress/status related flags) will change the final hash.
opts = append(opts, options.Unixfs.RawLeaves(rawblks))
}

// Storing optional mode or mtime (UnixFS 1.5) requires root block
// to always be 'dag-pb' and not 'raw'. Below adjusts raw-leaves setting, if possible.
if preserveMode || preserveMtime || mode != 0 || mtime != 0 {
// Error if --raw-leaves flag was explicitly passed by the user.
// (let user make a decision to manually disable it and retry)
if rbset && rawblks {
return fmt.Errorf("%s can't be used with UnixFS metadata like mode or modification time", rawLeavesOptionName)
}
// No explicit preference from user, disable raw-leaves and continue
rbset = true
rawblks = false
}

if trickle {
opts = append(opts, options.Unixfs.Layout(options.TrickleLayout))
}
Expand All @@ -270,6 +319,13 @@ only-hash, and progress/status related flags) will change the final hash.
opts = append(opts, options.Unixfs.PeerId(peerId))
}

if mode != 0 {
opts = append(opts, options.Unixfs.Mode(os.FileMode(mode)))
}
if mtime != 0 {
opts = append(opts, options.Unixfs.Mtime(mtime))
}

opts = append(opts, nil) // events option placeholder

var added int
Expand Down Expand Up @@ -304,12 +360,19 @@ only-hash, and progress/status related flags) will change the final hash.
output.Name = path.Join(addit.Name(), output.Name)
}

if err := res.Emit(&AddEvent{
addEvent := AddEvent{
Name: output.Name,
Hash: h,
Bytes: output.Bytes,
Size: output.Size,
}); err != nil {
Mtime: output.Mtime,
}

if output.Mode != 0 {
addEvent.Mode = "0" + strconv.FormatUint(uint64(output.Mode), 8)
}

if err := res.Emit(&addEvent); err != nil {
return err
}
}
Expand Down
65 changes: 64 additions & 1 deletion core/commands/files.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,16 @@ package commands

import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"os"
gopath "path"
"sort"
"strconv"
"strings"
"time"

"github.com/bittorrent/go-btfs/core"
"github.com/bittorrent/go-btfs/core/commands/cmdenv"
Expand Down Expand Up @@ -101,17 +104,56 @@ type statOutput struct {
WithLocality bool `json:",omitempty"`
Local bool `json:",omitempty"`
SizeLocal uint64 `json:",omitempty"`
Mode uint32 `json:",omitempty"`
Mtime int64 `json:",omitempty"`
}

func (s *statOutput) MarshalJSON() ([]byte, error) {
type so statOutput
out := &struct {
*so
Mode string `json:",omitempty"`
}{so: (*so)(s)}

if s.Mode != 0 {
out.Mode = fmt.Sprintf("%04o", s.Mode)
}
return json.Marshal(out)
}

func (s *statOutput) UnmarshalJSON(data []byte) error {
var err error
type so statOutput
tmp := &struct {
*so
Mode string `json:",omitempty"`
}{so: (*so)(s)}

if err := json.Unmarshal(data, &tmp); err != nil {
return err
}

if tmp.Mode != "" {
mode, err := strconv.ParseUint(tmp.Mode, 8, 32)
if err == nil {
s.Mode = uint32(mode)
}
}
return err
}

const (
defaultStatFormat = `<hash>
Size: <size>
CumulativeSize: <cumulsize>
ChildBlocks: <childs>
Type: <type>`
Type: <type>
Mode: <mode> (<mode-octal>)
Mtime: <mtime>`
filesFormatOptionName = "format"
filesSizeOptionName = "size"
filesWithLocalOptionName = "with-local"
filesStatUnspecified = "not set"
)

var filesStatCmd = &cmds.Command{
Expand Down Expand Up @@ -196,12 +238,24 @@ var filesStatCmd = &cmds.Command{
},
Encoders: cmds.EncoderMap{
cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *statOutput) error {
mode, modeo := filesStatUnspecified, filesStatUnspecified
if out.Mode != 0 {
mode = strings.ToLower(os.FileMode(out.Mode).String())
modeo = "0" + strconv.FormatInt(int64(out.Mode&0x1FF), 8)
}
mtime := filesStatUnspecified
if out.Mtime > 0 {
mtime = time.Unix(out.Mtime, 0).UTC().Format("2 Jan 2006, 15:04:05 MST")
}
s, _ := statGetFormatOptions(req)
s = strings.Replace(s, "<hash>", out.Hash, -1)
s = strings.Replace(s, "<size>", fmt.Sprintf("%d", out.Size), -1)
s = strings.Replace(s, "<cumulsize>", fmt.Sprintf("%d", out.CumulativeSize), -1)
s = strings.Replace(s, "<childs>", fmt.Sprintf("%d", out.Blocks), -1)
s = strings.Replace(s, "<type>", out.Type, -1)
s = strings.Replace(s, "<mode>", mode, -1)
s = strings.Replace(s, "<mode-octal>", modeo, -1)
s = strings.Replace(s, "<mtime>", mtime, -1)

fmt.Fprintln(w, s)

Expand Down Expand Up @@ -267,12 +321,21 @@ func statNode(nd ipld.Node, enc cidenc.Encoder) (*statOutput, error) {
return nil, fmt.Errorf("unrecognized node type: %s", d.Type())
}

var mode uint32
if m := d.Mode(); m != 0 {
mode = uint32(m)
} else if d.Type() == ft.TSymlink {
mode = uint32(os.ModeSymlink | 0x1FF)
}

return &statOutput{
Hash: enc.Encode(c),
Blocks: len(nd.Links()),
Size: d.FileSize(),
CumulativeSize: cumulsize,
Type: ndtype,
Mode: mode,
Mtime: d.ModTime().Unix(),
}, nil
case *dag.RawNode:
return &statOutput{
Expand Down
54 changes: 49 additions & 5 deletions core/commands/ls.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"os"
"sort"
"text/tabwriter"
"time"

cmdenv "github.com/bittorrent/go-btfs/core/commands/cmdenv"

Expand All @@ -23,6 +24,8 @@ type LsLink struct {
Size uint64
Type unixfs_pb.Data_DataType
Target string
Mode os.FileMode
Mtime time.Time
}

// LsObject is an element of LsOutput
Expand All @@ -43,6 +46,8 @@ const (
lsResolveTypeOptionName = "resolve-type"
lsSizeOptionName = "size"
lsStreamOptionName = "stream"
lsMTimeOptionName = "mtime"
lsModeOptionName = "mode"
)

var LsCmd = &cmds.Command{
Expand All @@ -66,6 +71,8 @@ The JSON output contains type information.
cmds.BoolOption(lsResolveTypeOptionName, "Resolve linked objects to find out their types.").WithDefault(true),
cmds.BoolOption(lsSizeOptionName, "Resolve linked objects to find out their file size.").WithDefault(true),
cmds.BoolOption(lsStreamOptionName, "s", "Enable experimental streaming of directory entries as they are traversed."),
cmds.BoolOption(lsMTimeOptionName, "t", "Print modification time."),
cmds.BoolOption(lsModeOptionName, "m", "Print mode."),
},
Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error {
api, err := cmdenv.GetApi(env, req)
Expand Down Expand Up @@ -158,6 +165,8 @@ The JSON output contains type information.
Size: link.Size,
Type: ftype,
Target: link.Target,
Mode: link.Mode,
Mtime: link.ModTime,
}
if err := processLink(paths[i], lsLink); err != nil {
return err
Expand Down Expand Up @@ -202,6 +211,8 @@ func tabularOutput(req *cmds.Request, w io.Writer, out *LsOutput, lastObjectHash
headers, _ := req.Options[lsHeadersOptionNameTime].(bool)
stream, _ := req.Options[lsStreamOptionName].(bool)
size, _ := req.Options[lsSizeOptionName].(bool)
mtime, _ := req.Options[lsMTimeOptionName].(bool)
mode, _ := req.Options[lsModeOptionName].(bool)
// in streaming mode we can't automatically align the tabs
// so we take a best guess
var minTabWidth int
Expand Down Expand Up @@ -229,6 +240,10 @@ func tabularOutput(req *cmds.Request, w io.Writer, out *LsOutput, lastObjectHash
if size {
s = "Hash\tSize\tName"
}

s = buildHeader(mode, "Mode", s)
s = buildHeader(mtime, "Mtime", s)

fmt.Fprintln(tw, s)
}
lastObjectHash = object.Hash
Expand All @@ -239,21 +254,50 @@ func tabularOutput(req *cmds.Request, w io.Writer, out *LsOutput, lastObjectHash
switch link.Type {
case unixfs.TDirectory, unixfs.THAMTShard, unixfs.TMetadata:
if size {
s = "%[1]s\t-\t%[3]s/\n"
s = "%[1]s\t-\t%[3]s/"
} else {
s = "%[1]s\t%[3]s/\n"
s = "%[1]s\t%[3]s/"
}
s = buildString(mode, s, 4)
s = buildString(mtime, s, 5)
s = s + "\n"
default:
if size {
s = "%s\t%v\t%s\n"
s = "%[1]s\t%[2]v\t%[3]s"
} else {
s = "%[1]s\t%[3]s\n"
s = "%[1]s\t%[3]s"
}
s = buildString(mode, s, 4)
s = buildString(mtime, s, 5)
s = s + "\n"
}

fmt.Fprintf(tw, s, link.Hash, link.Size, link.Name)
modeS := "-"
mtimeS := "-"

if link.Mode != 0 {
modeS = link.Mode.String()
}
if link.Mtime.Unix() != 0 {
mtimeS = link.Mtime.Format("2 Jan 2006, 15:04:05 MST")
}
fmt.Fprintf(tw, s, link.Hash, link.Size, link.Name, modeS, mtimeS)
}
}
tw.Flush()
return lastObjectHash
}

func buildString(set bool, s string, index int) string {
if set {
return fmt.Sprintf("%s\t%%[%d]s", s, index)
}
return s
}

func buildHeader(set bool, name, s string) string {
if set {
return s + "\t" + name
}
return s
}
Loading

0 comments on commit cb5afd2

Please sign in to comment.