Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Gateway: add object metadata #5536

Merged
merged 5 commits into from
Jan 22, 2025
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 9 additions & 4 deletions cmd/gateway.go
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,10 @@ func cmdGateway() *cli.Command {
Name: "object-tag",
Usage: "enable object tagging api",
},
&cli.BoolFlag{
Name: "object-meta",
Usage: "enable object metadata api",
},
&cli.StringFlag{
Name: "domain",
Usage: "domain for virtual-host-style requests",
Expand Down Expand Up @@ -149,10 +153,11 @@ func gateway(c *cli.Context) error {
jfs,
conf,
&jfsgateway.Config{
MultiBucket: c.Bool("multi-buckets"),
KeepEtag: c.Bool("keep-etag"),
Umask: uint16(umask),
ObjTag: c.Bool("object-tag"),
MultiBucket: c.Bool("multi-buckets"),
KeepEtag: c.Bool("keep-etag"),
Umask: uint16(umask),
ObjTag: c.Bool("object-tag"),
ObjMeta: c.Bool("object-meta"),
},
)
if err != nil {
Expand Down
4 changes: 4 additions & 0 deletions docs/en/guide/gateway.md
Original file line number Diff line number Diff line change
Expand Up @@ -147,6 +147,10 @@ By default, JuiceFS S3 Gateway does not save or return object ETag information.

Object tags are not supported by default, but you can use `--object-tag` to enable them.

### Enable object metadata

Object metadata is not supported by default, but you can use `--object-meta` to enable them.

### Enable virtual host-style requests

By default, JuiceFS S3 Gateway supports path-style requests in the format of `http://mydomain.com/bucket/object`. The `MINIO_DOMAIN` environment variable is used to enable virtual host-style requests. If the request's `Host` header information matches `(.+).mydomain.com`, the matched pattern `$1` is used as the bucket, and the path is used as the object.
Expand Down
132 changes: 119 additions & 13 deletions pkg/gateway/gateway.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ package gateway

import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
Expand Down Expand Up @@ -56,10 +57,11 @@ var mctx meta.Context
var logger = utils.GetLogger("juicefs")

type Config struct {
MultiBucket bool
KeepEtag bool
Umask uint16
ObjTag bool
MultiBucket bool
KeepEtag bool
Umask uint16
ObjTag bool
ObjMeta bool
}

func NewJFSGateway(jfs *fs.FileSystem, conf *vfs.Config, gConf *Config) (minio.ObjectLayer, error) {
Expand Down Expand Up @@ -533,6 +535,12 @@ func (n *jfsObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBu
}
dst := n.path(dstBucket, dstObject)
src := n.path(srcBucket, srcObject)

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It should be handled like an object-tag, first set to the tmp object.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done.

err = n.setObjMeta(dst, srcInfo.UserDefined)
if err != nil {
logger.Errorf("set object metadata error, path: %s error %s", dst, err)
}

if minio.IsStringEqual(src, dst) {
return n.GetObjectInfo(ctx, srcBucket, srcObject, minio.ObjectOptions{})
}
Expand Down Expand Up @@ -687,6 +695,18 @@ func (n *jfsObjects) GetObjectInfo(ctx context.Context, bucket, object string, o
return minio.ObjectInfo{}, errno
}
}
objMeta, err := n.getObjMeta(n.path(bucket, object))
if err != nil {
return minio.ObjectInfo{}, err
}
if opts.UserDefined == nil {
opts.UserDefined = make(map[string]string)
}
for k, v := range objMeta {
opts.UserDefined[k] = v
}
contentType := utils.GuessMimeType(object)
contentType = n.getObjContentType(objMeta, contentType)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The code here is strange, the function getObjContentType is only used here, it doesn't need to be a separate function.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Inlined.

return minio.ObjectInfo{
Bucket: bucket,
Name: object,
Expand All @@ -695,7 +715,7 @@ func (n *jfsObjects) GetObjectInfo(ctx context.Context, bucket, object string, o
IsDir: fi.IsDir(),
AccTime: fi.ModTime(),
ETag: string(etag),
ContentType: utils.GuessMimeType(object),
ContentType: contentType,
UserTags: string(tagStr),
UserDefined: minio.CleanMetadata(opts.UserDefined),
}, nil
Expand Down Expand Up @@ -821,6 +841,10 @@ func (n *jfsObjects) PutObject(ctx context.Context, bucket string, object string
}
}
}
err = n.setObjMeta(tmpName, opts.UserDefined)
if err != nil {
logger.Errorf("set object metadata error, path: %s error %s", p, err)
}
}); err != nil {
return
}
Expand All @@ -829,6 +853,7 @@ func (n *jfsObjects) PutObject(ctx context.Context, bucket string, object string
if eno != 0 {
return objInfo, jfsToObjectErr(ctx, eno, bucket, object)
}

return minio.ObjectInfo{
Bucket: bucket,
Name: object,
Expand Down Expand Up @@ -861,6 +886,10 @@ func (n *jfsObjects) NewMultipartUpload(ctx context.Context, bucket string, obje
}
}
}
err = n.setObjMeta(p, opts.UserDefined)
if err != nil {
logger.Errorf("set object metadata error, path: %s error %s", p, err)
}
}
return
}
Expand All @@ -871,6 +900,73 @@ const s3Etag = "s3-etag"
// less than 64k ref: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html#tag-restrictions
const s3Tags = "s3-tags"

// S3 object metadata
const s3Meta = "s3-meta"
const amzMeta = "x-amz-meta-"
const metaContentType = "content-type"
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There is no need to define a constant

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Removed.


var s3UserControlledSystemMeta = []string{
"cache-control",
"content-disposition",
"content-type",
}

func (n *jfsObjects) getObjMeta(p string) (objMeta map[string]string, err error) {
if n.gConf.ObjMeta {
var errno syscall.Errno
var metadataStr []byte
if metadataStr, errno = n.fs.GetXattr(mctx, p, s3Meta); errno != 0 && errno != meta.ENOATTR {
return objMeta, errno
}
if len(metadataStr) > 0 {
err = json.Unmarshal(metadataStr, &objMeta)
return objMeta, err
}
} else {
objMeta = make(map[string]string)
}
return objMeta, nil
}

func (n *jfsObjects) getObjContentType(objMeta map[string]string, fileContentType string) (contentType string) {
var exist bool
contentType, exist = objMeta[metaContentType]
if !exist || len(contentType) == 0 {
return fileContentType
} else {
return contentType
}
}

func (n *jfsObjects) setObjMeta(p string, metadata map[string]string) error {
if n.gConf.ObjMeta && metadata != nil {
meta := make(map[string]string)
for k, v := range metadata {
k = strings.ToLower(k)
if strings.HasPrefix(k, amzMeta) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

s3 metadata does not have a fixed prefix rule.

There are two kinds of object metadata in Amazon S3: system-defined metadata and user-defined metadata. System-defined metadata includes metadata such as the object's creation date, size, and storage class. User-defined metadata is metadata that you can choose to set at the time that you upload an object. This user-defined metadata is a set of name-value pairs. For more information, see System-defined object metadata and User-defined object metadata.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sorry, but I don't understand. The documentation you provide says:

the optional user-defined metadata names must begin with x-amz-meta- to distinguish them from other HTTP headers

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Content-Type Can it be saved now?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We should add a whitelist to support the storage of these system-defined metadata.
Cache-Control,Content-Disposition,Content-Type These 3 should be supported.

meta[k] = v
} else {
for _, systemMetaKey := range s3UserControlledSystemMeta {
if k == systemMetaKey {
meta[k] = v
break
}
}
}
}
if len(meta) > 0 {
s3MetadataValue, err := json.Marshal(meta)
if err != nil {
return err
}
if eno := n.fs.SetXattr(mctx, p, s3Meta, s3MetadataValue, 0); eno != 0 {
logger.Errorf("set object metadata error, path: %s,value: %s error: %s", p, string(s3Meta), eno)
}
}
}
return nil
}

func (n *jfsObjects) ListMultipartUploads(ctx context.Context, bucket string, prefix string, keyMarker string, uploadIDMarker string, delimiter string, maxUploads int) (lmi minio.ListMultipartsInfo, err error) {
if err = n.checkBucket(ctx, bucket); err != nil {
return
Expand Down Expand Up @@ -1100,6 +1196,15 @@ func (n *jfsObjects) CompleteMultipartUpload(ctx context.Context, bucket, object
}
}

var objMeta map[string]string
if n.gConf.ObjMeta {
if objMeta, err = n.getObjMeta(n.upath(bucket, uploadID)); err != nil {
logger.Errorf("get object meta error, path: %s, error: %s", n.upath(bucket, uploadID), err)
} else if err = n.setObjMeta(tmp, objMeta); err != nil {
logger.Errorf("set object meta error, path: %s, error: %s", tmp, err)
}
}

name := n.path(bucket, object)
eno = n.fs.Rename(mctx, tmp, name, 0)
if eno == syscall.ENOENT {
Expand Down Expand Up @@ -1128,14 +1233,15 @@ func (n *jfsObjects) CompleteMultipartUpload(ctx context.Context, bucket, object
// remove parts
_ = n.fs.Rmr(mctx, n.upath(bucket, uploadID))
return minio.ObjectInfo{
Bucket: bucket,
Name: object,
ETag: s3MD5,
ModTime: fi.ModTime(),
Size: fi.Size(),
IsDir: fi.IsDir(),
AccTime: fi.ModTime(),
UserTags: string(tagStr),
Bucket: bucket,
Name: object,
ETag: s3MD5,
ModTime: fi.ModTime(),
Size: fi.Size(),
IsDir: fi.IsDir(),
AccTime: fi.ModTime(),
UserTags: string(tagStr),
UserDefined: minio.CleanMetadata(opts.UserDefined),
}, nil
}

Expand Down
Loading