Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support storing UnixFS 1.5 Mode and ModTime #7754

Closed
wants to merge 25 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
cf50c16
preliminary support for unixfs Mode and ModTime
kstuart Nov 5, 2020
d93f1ef
[ipfs get] pass the output path to FileArchive
kstuart Nov 10, 2020
0d883e9
[sharness] add tests for preserving/setting file mode and modificatio…
kstuart Nov 10, 2020
a85c903
Merge branch 'master' into feat/unixfs/ufs15
kstuart Dec 1, 2020
9d257fe
add file mode and mtime support for file stat
kstuart Apr 3, 2021
659e709
[ipfs add] emit mtime and mode in AddEvent
kstuart Apr 3, 2021
e1d9879
add optional mode and mtime to ipfs files stat output
kstuart Aug 28, 2021
ddf9073
Merge branch 'master' of https://github.com/ipfs/go-ipfs into feat/un…
kstuart Sep 2, 2021
16b57a6
humanize mode and mtime in ipfs files stat output
kstuart Sep 2, 2021
45713c2
don't set modification time when none provided
kstuart Sep 6, 2021
9172385
fix/enhance files stat template
kstuart Sep 6, 2021
a6f5f09
add ipfs files chmod and touch
kstuart Sep 6, 2021
d7e61f0
make files stat json output compatible with js-ipfs again
kstuart Sep 9, 2021
ae2d3d4
Merge remote-tracking branch 'upstream/master' into feat/unixfs/ufs15
kstuart Jan 1, 2022
46e0660
fix bad format of JSON statOutput.Mode
kstuart Jan 8, 2022
29860a1
remove leftover code
kstuart Jun 26, 2022
e40cf41
preserve mode/mtime on directories and symlinks
kstuart Jun 26, 2022
fd80be0
correct comment
kstuart Jun 26, 2022
ae46f6a
remove unneeded parameter
kstuart Jun 26, 2022
fb8d100
remove unneeded cast
kstuart Oct 25, 2022
574b656
sync with upstream master brach
kstuart Oct 25, 2022
c000653
finalize files stat format for mode and mtime
kstuart Oct 31, 2022
03a37d6
fix storing mode and mtime on symlinks and directories
kstuart Oct 31, 2022
ed2b842
complete sharness tests for mode and modification time
kstuart Oct 31, 2022
8b3f27d
add missing test and minor fixes
kstuart Oct 31, 2022
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
92 changes: 82 additions & 10 deletions core/commands/add.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,9 @@ import (
"io"
"os"
"path"
"strconv"
"strings"
"time"

"github.com/ipfs/kubo/core/commands/cmdenv"

Expand All @@ -23,11 +25,31 @@ import (
// ErrDepthLimitExceeded indicates that the max depth has been exceeded.
var ErrDepthLimitExceeded = fmt.Errorf("depth limit exceeded")

type TimeParts struct {
t *time.Time
}

func (t TimeParts) MarshalJSON() ([]byte, error) {
return t.t.MarshalJSON()
}

// UnmarshalJSON implements the json.Unmarshaler interface.
// The time is expected to be a quoted string in RFC 3339 format.
func (t *TimeParts) UnmarshalJSON(data []byte) (err error) {
// Fractional seconds are handled implicitly by Parse.
tt, err := time.Parse("\"2006-01-02T15:04:05Z\"", string(data))
*t = TimeParts{&tt}
return
}

type AddEvent struct {
Name string
Hash string `json:",omitempty"`
Bytes int64 `json:",omitempty"`
Size string `json:",omitempty"`
Name string
Hash string `json:",omitempty"`
Bytes int64 `json:",omitempty"`
Size string `json:",omitempty"`
Mode string `json:",omitempty"`
Mtime int64 `json:",omitempty"`
MtimeNsecs int `json:",omitempty"`
}

const (
Expand All @@ -48,6 +70,12 @@ const (
inlineOptionName = "inline"
inlineLimitOptionName = "inline-limit"
toFilesOptionName = "to-files"

preserveModeOptionName = "preserve-mode"
preserveMtimeOptionName = "preserve-mtime"
modeOptionName = "mode"
mtimeOptionName = "mtime"
mtimeNsecsOptionName = "mtime-nsecs"
Comment on lines +74 to +78
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

💭 Is it possible to preserve mtime without preserving mode?

To avoid cross-implementation issues, we should have "zero values" documented in https://github.com/ipfs/specs/pull/331/files

)

const adderOutChanSize = 8
Expand Down Expand Up @@ -164,6 +192,11 @@ See 'dag export' and 'dag import' for more information.
cmds.IntOption(inlineLimitOptionName, "Maximum block size to inline. (experimental)").WithDefault(32),
cmds.BoolOption(pinOptionName, "Pin locally to protect added files from garbage collection.").WithDefault(true),
cmds.StringOption(toFilesOptionName, "Add reference to Files API (MFS) at the provided path."),
cmds.BoolOption(preserveModeOptionName, "Apply permissions to created UnixFS entries"),
cmds.BoolOption(preserveMtimeOptionName, "Apply modification time to created UnixFS entries"),
cmds.UintOption(modeOptionName, "File mode to apply to created UnixFS entries"),
cmds.Int64Option(mtimeOptionName, "Modification time in seconds before or after the Unix Epoch to apply to created UnixFS entries"),
cmds.UintOption(mtimeNsecsOptionName, "Modification time fraction in nanoseconds"),
Comment on lines +195 to +199
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
cmds.BoolOption(preserveModeOptionName, "Apply permissions to created UnixFS entries"),
cmds.BoolOption(preserveMtimeOptionName, "Apply modification time to created UnixFS entries"),
cmds.UintOption(modeOptionName, "File mode to apply to created UnixFS entries"),
cmds.Int64Option(mtimeOptionName, "Modification time in seconds before or after the Unix Epoch to apply to created UnixFS entries"),
cmds.UintOption(mtimeNsecsOptionName, "Modification time fraction in nanoseconds"),
cmds.BoolOption(preserveModeOptionName, "Apply existing POSIX permissions to created UnixFS entries"),
cmds.BoolOption(preserveMtimeOptionName, "Apply existing POSIX modification time to created UnixFS entries"),
cmds.UintOption(modeOptionName, "Custom POSIX file mode to store in created UnixFS entries"),
cmds.Int64Option(mtimeOptionName, "Custom POSIX modification time to store in created UnixFS entries (seconds before or after the Unix Epoch)"),
cmds.UintOption(mtimeNsecsOptionName, "Custom POSIX modification time (optional time fraction in nanoseconds)"),

},
PreRun: func(req *cmds.Request, env cmds.Environment) error {
quiet, _ := req.Options[quietOptionName].(bool)
Expand Down Expand Up @@ -205,6 +238,11 @@ See 'dag export' and 'dag import' for more information.
inline, _ := req.Options[inlineOptionName].(bool)
inlineLimit, _ := req.Options[inlineLimitOptionName].(int)
toFilesStr, toFilesSet := req.Options[toFilesOptionName].(string)
preserveMode, _ := req.Options[preserveModeOptionName].(bool)
preserveMtime, _ := req.Options[preserveMtimeOptionName].(bool)
mode, _ := req.Options[modeOptionName].(uint)
mtime, _ := req.Options[mtimeOptionName].(int64)
mtimeNsecs, _ := req.Options[mtimeNsecsOptionName].(uint)

hashFunCode, ok := mh.Names[strings.ToLower(hashFunStr)]
if !ok {
Expand Down Expand Up @@ -241,6 +279,19 @@ See 'dag export' and 'dag import' for more information.

options.Unixfs.Progress(progress),
options.Unixfs.Silent(silent),

options.Unixfs.PreserveMode(preserveMode),
options.Unixfs.PreserveMtime(preserveMtime),
}

if mode != 0 {
opts = append(opts, options.Unixfs.Mode(os.FileMode(mode)))
}

if mtime != 0 {
opts = append(opts, options.Unixfs.Mtime(mtime, uint32(mtimeNsecs)))
} else if mtimeNsecs != 0 {
fmt.Printf("option %s ignored as no valid %s value provided\n", mtimeNsecsOptionName, mtimeOptionName)
}

if cidVerSet {
Expand Down Expand Up @@ -352,12 +403,33 @@ See 'dag export' and 'dag import' for more information.
output.Name = path.Join(addit.Name(), output.Name)
}

if err := res.Emit(&AddEvent{
Name: output.Name,
Hash: h,
Bytes: output.Bytes,
Size: output.Size,
}); err != nil {
output.Mode = addit.Node().Mode()
if ts := addit.Node().ModTime(); !ts.IsZero() {
output.Mtime = addit.Node().ModTime().Unix()
output.MtimeNsecs = addit.Node().ModTime().Nanosecond()
}

addEvent := AddEvent{
Name: output.Name,
Hash: h,
Bytes: output.Bytes,
Size: output.Size,
Mtime: output.Mtime,
MtimeNsecs: output.MtimeNsecs,
}

if output.Mode != 0 {
addEvent.Mode = "0" + strconv.FormatUint(uint64(output.Mode), 8)
}

if output.Mtime > 0 {
addEvent.Mtime = output.Mtime
if output.MtimeNsecs > 0 {
addEvent.MtimeNsecs = output.MtimeNsecs
}
}

if err := res.Emit(&addEvent); err != nil {
return err
}
}
Expand Down
2 changes: 2 additions & 0 deletions core/commands/commands_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -145,6 +145,8 @@ func TestCommands(t *testing.T) {
"/files/rm",
"/files/stat",
"/files/write",
"/files/chmod",
"/files/touch",
"/filestore",
"/filestore/dups",
"/filestore/ls",
Expand Down
Loading