-
Notifications
You must be signed in to change notification settings - Fork 999
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Gateway: add object metadata #5536
Changes from 4 commits
ba8ef65
8cd99ab
a04fd12
c267988
63e82c2
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -18,6 +18,7 @@ package gateway | |
|
||
import ( | ||
"context" | ||
"encoding/json" | ||
"errors" | ||
"fmt" | ||
"io" | ||
|
@@ -56,10 +57,11 @@ var mctx meta.Context | |
var logger = utils.GetLogger("juicefs") | ||
|
||
type Config struct { | ||
MultiBucket bool | ||
KeepEtag bool | ||
Umask uint16 | ||
ObjTag bool | ||
MultiBucket bool | ||
KeepEtag bool | ||
Umask uint16 | ||
ObjTag bool | ||
ObjMeta bool | ||
} | ||
|
||
func NewJFSGateway(jfs *fs.FileSystem, conf *vfs.Config, gConf *Config) (minio.ObjectLayer, error) { | ||
|
@@ -533,6 +535,12 @@ func (n *jfsObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBu | |
} | ||
dst := n.path(dstBucket, dstObject) | ||
src := n.path(srcBucket, srcObject) | ||
|
||
err = n.setObjMeta(dst, srcInfo.UserDefined) | ||
if err != nil { | ||
logger.Errorf("set object metadata error, path: %s error %s", dst, err) | ||
} | ||
|
||
if minio.IsStringEqual(src, dst) { | ||
return n.GetObjectInfo(ctx, srcBucket, srcObject, minio.ObjectOptions{}) | ||
} | ||
|
@@ -687,6 +695,18 @@ func (n *jfsObjects) GetObjectInfo(ctx context.Context, bucket, object string, o | |
return minio.ObjectInfo{}, errno | ||
} | ||
} | ||
objMeta, err := n.getObjMeta(n.path(bucket, object)) | ||
if err != nil { | ||
return minio.ObjectInfo{}, err | ||
} | ||
if opts.UserDefined == nil { | ||
opts.UserDefined = make(map[string]string) | ||
} | ||
for k, v := range objMeta { | ||
opts.UserDefined[k] = v | ||
} | ||
contentType := utils.GuessMimeType(object) | ||
contentType = n.getObjContentType(objMeta, contentType) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The code here is strange, the function There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Inlined. |
||
return minio.ObjectInfo{ | ||
Bucket: bucket, | ||
Name: object, | ||
|
@@ -695,7 +715,7 @@ func (n *jfsObjects) GetObjectInfo(ctx context.Context, bucket, object string, o | |
IsDir: fi.IsDir(), | ||
AccTime: fi.ModTime(), | ||
ETag: string(etag), | ||
ContentType: utils.GuessMimeType(object), | ||
ContentType: contentType, | ||
UserTags: string(tagStr), | ||
UserDefined: minio.CleanMetadata(opts.UserDefined), | ||
}, nil | ||
|
@@ -821,6 +841,10 @@ func (n *jfsObjects) PutObject(ctx context.Context, bucket string, object string | |
} | ||
} | ||
} | ||
err = n.setObjMeta(tmpName, opts.UserDefined) | ||
if err != nil { | ||
logger.Errorf("set object metadata error, path: %s error %s", p, err) | ||
} | ||
}); err != nil { | ||
return | ||
} | ||
|
@@ -829,6 +853,7 @@ func (n *jfsObjects) PutObject(ctx context.Context, bucket string, object string | |
if eno != 0 { | ||
return objInfo, jfsToObjectErr(ctx, eno, bucket, object) | ||
} | ||
|
||
return minio.ObjectInfo{ | ||
Bucket: bucket, | ||
Name: object, | ||
|
@@ -861,6 +886,10 @@ func (n *jfsObjects) NewMultipartUpload(ctx context.Context, bucket string, obje | |
} | ||
} | ||
} | ||
err = n.setObjMeta(p, opts.UserDefined) | ||
if err != nil { | ||
logger.Errorf("set object metadata error, path: %s error %s", p, err) | ||
} | ||
} | ||
return | ||
} | ||
|
@@ -871,6 +900,73 @@ const s3Etag = "s3-etag" | |
// less than 64k ref: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html#tag-restrictions | ||
const s3Tags = "s3-tags" | ||
|
||
// S3 object metadata | ||
const s3Meta = "s3-meta" | ||
const amzMeta = "x-amz-meta-" | ||
const metaContentType = "content-type" | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. There is no need to define a constant There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Removed. |
||
|
||
var s3UserControlledSystemMeta = []string{ | ||
"cache-control", | ||
"content-disposition", | ||
"content-type", | ||
} | ||
|
||
func (n *jfsObjects) getObjMeta(p string) (objMeta map[string]string, err error) { | ||
if n.gConf.ObjMeta { | ||
var errno syscall.Errno | ||
var metadataStr []byte | ||
if metadataStr, errno = n.fs.GetXattr(mctx, p, s3Meta); errno != 0 && errno != meta.ENOATTR { | ||
return objMeta, errno | ||
} | ||
if len(metadataStr) > 0 { | ||
err = json.Unmarshal(metadataStr, &objMeta) | ||
return objMeta, err | ||
} | ||
} else { | ||
objMeta = make(map[string]string) | ||
} | ||
return objMeta, nil | ||
} | ||
|
||
func (n *jfsObjects) getObjContentType(objMeta map[string]string, fileContentType string) (contentType string) { | ||
var exist bool | ||
contentType, exist = objMeta[metaContentType] | ||
if !exist || len(contentType) == 0 { | ||
return fileContentType | ||
} else { | ||
return contentType | ||
} | ||
} | ||
|
||
func (n *jfsObjects) setObjMeta(p string, metadata map[string]string) error { | ||
if n.gConf.ObjMeta && metadata != nil { | ||
meta := make(map[string]string) | ||
for k, v := range metadata { | ||
k = strings.ToLower(k) | ||
if strings.HasPrefix(k, amzMeta) { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. s3 metadata does not have a fixed prefix rule.
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Sorry, but I don't understand. The documentation you provide says:
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. We should add a whitelist to support the storage of these system-defined metadata. |
||
meta[k] = v | ||
} else { | ||
for _, systemMetaKey := range s3UserControlledSystemMeta { | ||
if k == systemMetaKey { | ||
meta[k] = v | ||
break | ||
} | ||
} | ||
} | ||
} | ||
if len(meta) > 0 { | ||
s3MetadataValue, err := json.Marshal(meta) | ||
if err != nil { | ||
return err | ||
} | ||
if eno := n.fs.SetXattr(mctx, p, s3Meta, s3MetadataValue, 0); eno != 0 { | ||
logger.Errorf("set object metadata error, path: %s,value: %s error: %s", p, string(s3Meta), eno) | ||
} | ||
} | ||
} | ||
return nil | ||
} | ||
|
||
func (n *jfsObjects) ListMultipartUploads(ctx context.Context, bucket string, prefix string, keyMarker string, uploadIDMarker string, delimiter string, maxUploads int) (lmi minio.ListMultipartsInfo, err error) { | ||
if err = n.checkBucket(ctx, bucket); err != nil { | ||
return | ||
|
@@ -1100,6 +1196,15 @@ func (n *jfsObjects) CompleteMultipartUpload(ctx context.Context, bucket, object | |
} | ||
} | ||
|
||
var objMeta map[string]string | ||
if n.gConf.ObjMeta { | ||
if objMeta, err = n.getObjMeta(n.upath(bucket, uploadID)); err != nil { | ||
logger.Errorf("get object meta error, path: %s, error: %s", n.upath(bucket, uploadID), err) | ||
} else if err = n.setObjMeta(tmp, objMeta); err != nil { | ||
logger.Errorf("set object meta error, path: %s, error: %s", tmp, err) | ||
} | ||
} | ||
|
||
name := n.path(bucket, object) | ||
eno = n.fs.Rename(mctx, tmp, name, 0) | ||
if eno == syscall.ENOENT { | ||
|
@@ -1128,14 +1233,15 @@ func (n *jfsObjects) CompleteMultipartUpload(ctx context.Context, bucket, object | |
// remove parts | ||
_ = n.fs.Rmr(mctx, n.upath(bucket, uploadID)) | ||
return minio.ObjectInfo{ | ||
Bucket: bucket, | ||
Name: object, | ||
ETag: s3MD5, | ||
ModTime: fi.ModTime(), | ||
Size: fi.Size(), | ||
IsDir: fi.IsDir(), | ||
AccTime: fi.ModTime(), | ||
UserTags: string(tagStr), | ||
Bucket: bucket, | ||
Name: object, | ||
ETag: s3MD5, | ||
ModTime: fi.ModTime(), | ||
Size: fi.Size(), | ||
IsDir: fi.IsDir(), | ||
AccTime: fi.ModTime(), | ||
UserTags: string(tagStr), | ||
UserDefined: minio.CleanMetadata(opts.UserDefined), | ||
}, nil | ||
} | ||
|
||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
It should be handled like an object-tag, first set to the tmp object.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Done.