Skip to content

Commit

Permalink
Create a way to preserve timestamps
Browse files Browse the repository at this point in the history 
The `--preserve-timestamp` flag will set metadata on the s3 objects and
then update the files on disk during a download.

If the `--preserve-timestamp` flag is passed during an upload the following metadata keys will be set:
  - x-amz-meta-file-ctime
  - x-amz-meta-file-mtime
  - x-amz-meta-file-atime

If the `--preserve-timesamp` flag is passed during a download, s5cmd
will attempt to modify the created files on disk.
  **There doesn't appear to be a way to change the created date on Linux
  • Loading branch information
@Ahuge
Ahuge committed Nov 7, 2022
1 parent 544af49 commit 7d7260d
Show file tree
Hide file tree
Showing 6 changed files with 150 additions and 31 deletions.
6 changes: 3 additions & 3 deletions command/cp.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -554,7 +554,7 @@ func (c Copy) doDownload(ctx context.Context, srcurl *url.URL, dsturl *url.URL)
if err != nil {
return err
}
err = storage.SetFileTime(dsturl.Absolute(), *obj.CreateTime, *obj.ModTime)
err = storage.SetFileTime(dsturl.Absolute(), *obj.AccessTime, *obj.ModTime, *obj.CreateTime)
if err != nil {
return err
}
Expand Down Expand Up @@ -609,11 +609,11 @@ func (c Copy) doUpload(ctx context.Context, srcurl *url.URL, dsturl *url.URL) er
SetExpires(c.expires)

if c.preserveTimestamp {
cTime, mTime, err := storage.GetFileTime(srcurl.Absolute())
aTime, mTime, cTime, err := storage.GetFileTime(srcurl.Absolute())
if err != nil {
return err
}
metadata.SetPreserveTimestamp(cTime.String(), mTime.String())
metadata.SetPreserveTimestamp(aTime, mTime, cTime)
}

if c.contentType != "" {
Expand Down
26 changes: 21 additions & 5 deletions storage/fs_darwin.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,22 +8,38 @@ import (
"time"
)

func GetFileTime(filename string) (time.Time, time.Time, error) {
func GetFileTime(filename string) (time.Time, time.Time, time.Time, error) {
fi, err := os.Stat(filename)
if err != nil {
return time.Time{}, time.Time{}, err
return time.Time{}, time.Time{}, time.Time{}, err
}

stat := fi.Sys().(*syscall.Stat_t)
cTime := time.Unix(int64(stat.Ctimespec.Sec), int64(stat.Ctimespec.Nsec))
aTime := time.Unix(int64(stat.Atimespec.Sec), int64(stat.Atimespec.Nsec))

mTime := fi.ModTime()

return cTime, mTime, nil
return aTime, mTime, cTime, nil
}

func SetFileTime(filename string, creationTime time.Time, modTime time.Time) error {
err := os.Chtimes(filename, modTime, modTime)
func SetFileTime(filename string, accessTime, modificationTime, creationTime time.Time) error {
var err error
if accessTime.IsZero() && modificationTime.IsZero() {
// Nothing recorded in s3. Return fast.
return nil
} else if accessTime.IsZero() {
accessTime, _, _, err = GetFileTime(filename)
if err != nil {
return err
}
} else if modificationTime.IsZero() {
_, modificationTime, _, err = GetFileTime(filename)
if err != nil {
return err
}
}
err = os.Chtimes(filename, accessTime, modificationTime)
if err != nil {
return err
}
Expand Down
28 changes: 23 additions & 5 deletions storage/fs_linux.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,22 +8,40 @@ import (
"time"
)

func GetFileTime(filename string) (time.Time, time.Time, error) {
func GetFileTime(filename string) (time.Time, time.Time, time.Time, error) {
fi, err := os.Stat(filename)
if err != nil {
return time.Time{}, time.Time{}, err
return time.Time{}, time.Time{}, time.Time{}, err
}

stat := fi.Sys().(*syscall.Stat_t)
cTime := time.Unix(int64(stat.Ctim.Sec), int64(stat.Ctim.Nsec))
aTime := time.Unix(int64(stat.Atim.Sec), int64(stat.Atim.Nsec))

mTime := fi.ModTime()

return cTime, mTime, nil
return aTime, mTime, cTime, nil
}

func SetFileTime(filename string, creationTime time.Time, modTime time.Time) error {
err := os.Chtimes(filename, modTime, modTime)
func SetFileTime(filename string, accessTime, modificationTime, creationTime time.Time) error {
if accessTime.IsZero() && modificationTime.IsZero() {
// Nothing recorded in s3. Return fast.
return nil
}
var err error
if accessTime.IsZero() {
accessTime, _, _, err = GetFileTime(filename)
if err != nil {
return err
}
}
if modificationTime.IsZero() {
_, modificationTime, _, err = GetFileTime(filename)
if err != nil {
return err
}
}
err = os.Chtimes(filename, accessTime, modificationTime)
if err != nil {
return err
}
Expand Down
40 changes: 33 additions & 7 deletions storage/fs_windows.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,29 +8,55 @@ import (
"time"
)

func GetFileTime(filename string) (time.Time, time.Time, error) {
func GetFileTime(filename string) (time.Time, time.Time, time.Time, error) {
fi, err := os.Stat(filename)
if err != nil {
return time.Time{}, time.Time{}, err
return time.Time{}, time.Time{}, time.Time{}, err
}

d := fi.Sys().(*syscall.Win32FileAttributeData)
cTime := time.Unix(0, d.CreationTime.Nanoseconds())
aTime := time.Unix(0, d.LastAccessTime.Nanoseconds())

mTime := fi.ModTime()

return cTime, mTime, nil
return aTime, mTime, cTime, nil
}

func SetFileTime(filename string, creationTime time.Time, modTime time.Time) error {
func SetFileTime(filename string, accessTime, modificationTime, creationTime time.Time) error {
var err error
if accessTime.IsZero() && modificationTime.IsZero() && creationTime.IsZero() {
// Nothing recorded in s3. Return fast.
return nil
} else if accessTime.IsZero() {
accessTime, _, _, err = GetFileTime(filename)
if err != nil {
return err
}
} else if modificationTime.IsZero() {
_, modificationTime, _, err = GetFileTime(filename)
if err != nil {
return err
}
} else if creationTime.IsZero() {
_, _, creationTime, err = GetFileTime(filename)
if err != nil {
return err
}
}

aft := syscall.NsecToFiletime(accessTime.UnixNano())
mft := syscall.NsecToFiletime(modificationTime.UnixNano())
cft := syscall.NsecToFiletime(creationTime.UnixNano())

fd, err := syscall.Open(filename, os.O_RDWR, 0775)
if err != nil {
return err
}
err = syscall.SetFileTime(fd, &cft, &aft, &mft)

defer syscall.Close(fd)

cft := syscall.NsecToFiletime(int64(creationTime.Nanosecond()))
mft := syscall.NsecToFiletime(int64(modTime.Nanosecond()))
err = syscall.SetFileTime(fd, &cft, &mft, &mft)
if err != nil {
return err
}
Expand Down
64 changes: 58 additions & 6 deletions storage/s3.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -131,10 +131,42 @@ func (s *S3) Stat(ctx context.Context, url *url.URL) (*Object, error) {
mod := aws.TimeValue(output.LastModified)

obj := &Object{
URL: url,
Etag: strings.Trim(etag, `"`),
ModTime: &mod,
Size: aws.Int64Value(output.ContentLength),
URL: url,
Etag: strings.Trim(etag, `"`),
ModTime: &mod,
Size: aws.Int64Value(output.ContentLength),
CreateTime: &time.Time{},
AccessTime: &time.Time{},
}

cTimeS := aws.StringValue(output.Metadata["file-ctime"])
if cTimeS != "" {
ctime, err := strconv.ParseInt(cTimeS, 10, 64)
if err != nil {
return nil, err
}
creationTime := time.Unix(0, ctime)
obj.CreateTime = &creationTime
}

mTimeS := aws.StringValue(output.Metadata["file-mtime"])
if mTimeS != "" {
mtime, err := strconv.ParseInt(mTimeS, 10, 64)
if err != nil {
return nil, err
}
modificationTime := time.Unix(0, mtime)
obj.ModTime = &modificationTime
}

aTimeS := aws.StringValue(output.Metadata["file-atime"])
if aTimeS != "" {
atime, err := strconv.ParseInt(aTimeS, 10, 64)
if err != nil {
return nil, err
}
accessTime := time.Unix(0, atime)
obj.AccessTime = &accessTime
}

ctime := output.Metadata["x-amz-meta-file-ctime"]
Expand Down Expand Up @@ -411,12 +443,17 @@ func (s *S3) Copy(ctx context.Context, from, to *url.URL, metadata Metadata) err

ctime := metadata.cTime()
if ctime != "" {
input.Metadata["x-amz-meta-file-ctime"] = aws.String(ctime)
input.Metadata["file-ctime"] = aws.String(ctime)
}

mtime := metadata.mTime()
if ctime != "" {
input.Metadata["x-amz-meta-file-mtime"] = aws.String(mtime)
input.Metadata["file-mtime"] = aws.String(mtime)
}

atime := metadata.aTime()
if ctime != "" {
input.Metadata["file-atime"] = aws.String(atime)
}
_, err := s.api.CopyObject(input)
return err
Expand Down Expand Up @@ -583,6 +620,21 @@ func (s *S3) Put(
input.Expires = aws.Time(t)
}

ctime := metadata.cTime()
if ctime != "" {
input.Metadata["file-ctime"] = aws.String(ctime)
}

mtime := metadata.mTime()
if ctime != "" {
input.Metadata["file-mtime"] = aws.String(mtime)
}

atime := metadata.aTime()
if ctime != "" {
input.Metadata["file-atime"] = aws.String(atime)
}

sseEncryption := metadata.SSE()
if sseEncryption != "" {
input.ServerSideEncryption = aws.String(sseEncryption)
Expand Down
17 changes: 12 additions & 5 deletions storage/storage.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"encoding/json"
"fmt"
"os"
"strconv"
"time"

"github.com/peak/s5cmd/log"
Expand Down Expand Up @@ -105,6 +106,7 @@ func (o *Options) SetRegion(region string) {
type Object struct {
URL *url.URL `json:"key,omitempty"`
Etag string `json:"etag,omitempty"`
AccessTime *time.Time `json:"accessed,omitempty"`
ModTime *time.Time `json:"last_modified,omitempty"`
CreateTime *time.Time `json:"created,omitempty"`
Type ObjectType `json:"type,omitempty"`
Expand Down Expand Up @@ -238,16 +240,21 @@ func (m Metadata) SetExpires(expires string) Metadata {
}

func (m Metadata) cTime() string {
return m["x-amz-meta-file-ctime"]
return m["file-ctime"]
}

func (m Metadata) mTime() string {
return m["x-amz-meta-file-mtime"]
return m["file-mtime"]
}

func (m Metadata) SetPreserveTimestamp(cTime, mTime string) Metadata {
m["x-amz-meta-file-ctime"] = cTime
m["x-amz-meta-file-mtime"] = mTime
func (m Metadata) aTime() string {
return m["file-atime"]
}

func (m Metadata) SetPreserveTimestamp(aTime, mTime, cTime time.Time) Metadata {
m["file-ctime"] = strconv.Itoa(int(cTime.UnixNano()))
m["file-mtime"] = strconv.Itoa(int(mTime.UnixNano()))
m["file-atime"] = strconv.Itoa(int(aTime.UnixNano()))
return m
}

Expand Down

0 comments on commit 7d7260d

Please sign in to comment.