diff --git a/cmd/gateway.go b/cmd/gateway.go index 37b34e012d11..3b8ddcc32c1d 100644 --- a/cmd/gateway.go +++ b/cmd/gateway.go @@ -80,6 +80,10 @@ func cmdGateway() *cli.Command { Name: "object-tag", Usage: "enable object tagging api", }, + &cli.BoolFlag{ + Name: "object-meta", + Usage: "enable object metadata api", + }, &cli.StringFlag{ Name: "domain", Usage: "domain for virtual-host-style requests", @@ -149,10 +153,11 @@ func gateway(c *cli.Context) error { jfs, conf, &jfsgateway.Config{ - MultiBucket: c.Bool("multi-buckets"), - KeepEtag: c.Bool("keep-etag"), - Umask: uint16(umask), - ObjTag: c.Bool("object-tag"), + MultiBucket: c.Bool("multi-buckets"), + KeepEtag: c.Bool("keep-etag"), + Umask: uint16(umask), + ObjTag: c.Bool("object-tag"), + ObjMeta: c.Bool("object-meta"), }, ) if err != nil { diff --git a/docs/en/guide/gateway.md b/docs/en/guide/gateway.md index bb38bc29ae48..260b67342182 100644 --- a/docs/en/guide/gateway.md +++ b/docs/en/guide/gateway.md @@ -147,6 +147,10 @@ By default, JuiceFS S3 Gateway does not save or return object ETag information. Object tags are not supported by default, but you can use `--object-tag` to enable them. +### Enable object metadata + +Object metadata is not supported by default, but you can use `--object-meta` to enable them. + ### Enable virtual host-style requests By default, JuiceFS S3 Gateway supports path-style requests in the format of `http://mydomain.com/bucket/object`. The `MINIO_DOMAIN` environment variable is used to enable virtual host-style requests. If the request's `Host` header information matches `(.+).mydomain.com`, the matched pattern `$1` is used as the bucket, and the path is used as the object. diff --git a/pkg/gateway/gateway.go b/pkg/gateway/gateway.go index f5363ba957d8..7dbb9594e02b 100644 --- a/pkg/gateway/gateway.go +++ b/pkg/gateway/gateway.go @@ -18,6 +18,7 @@ package gateway import ( "context" + "encoding/json" "errors" "fmt" "io" @@ -60,6 +61,7 @@ type Config struct { KeepEtag bool Umask uint16 ObjTag bool + ObjMeta bool } func NewJFSGateway(jfs *fs.FileSystem, conf *vfs.Config, gConf *Config) (minio.ObjectLayer, error) { @@ -533,7 +535,13 @@ func (n *jfsObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBu } dst := n.path(dstBucket, dstObject) src := n.path(srcBucket, srcObject) + if minio.IsStringEqual(src, dst) { + // if we copy the same object for set metadata + err = n.setObjMeta(dst, srcInfo.UserDefined) + if err != nil { + logger.Errorf("set object metadata error, path: %s error %s", dst, err) + } return n.GetObjectInfo(ctx, srcBucket, srcObject, minio.ObjectOptions{}) } tmp := n.tpath(dstBucket, "tmp", minio.MustGetUUID()) @@ -577,6 +585,10 @@ func (n *jfsObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBu } } } + err = n.setObjMeta(tmp, srcInfo.UserDefined) + if err != nil { + logger.Errorf("set object metadata error, path: %s error %s", dst, err) + } eno = n.fs.Rename(mctx, tmp, dst, 0) if eno == syscall.ENOENT { @@ -687,6 +699,20 @@ func (n *jfsObjects) GetObjectInfo(ctx context.Context, bucket, object string, o return minio.ObjectInfo{}, errno } } + objMeta, err := n.getObjMeta(n.path(bucket, object)) + if err != nil { + return minio.ObjectInfo{}, err + } + if opts.UserDefined == nil { + opts.UserDefined = make(map[string]string) + } + for k, v := range objMeta { + opts.UserDefined[k] = v + } + contentType := utils.GuessMimeType(object) + if c, exist := objMeta["content-type"]; exist && len(c) > 0 { + contentType = c + } return minio.ObjectInfo{ Bucket: bucket, Name: object, @@ -695,7 +721,7 @@ func (n *jfsObjects) GetObjectInfo(ctx context.Context, bucket, object string, o IsDir: fi.IsDir(), AccTime: fi.ModTime(), ETag: string(etag), - ContentType: utils.GuessMimeType(object), + ContentType: contentType, UserTags: string(tagStr), UserDefined: minio.CleanMetadata(opts.UserDefined), }, nil @@ -821,6 +847,10 @@ func (n *jfsObjects) PutObject(ctx context.Context, bucket string, object string } } } + err = n.setObjMeta(tmpName, opts.UserDefined) + if err != nil { + logger.Errorf("set object metadata error, path: %s error %s", p, err) + } }); err != nil { return } @@ -829,6 +859,7 @@ func (n *jfsObjects) PutObject(ctx context.Context, bucket string, object string if eno != 0 { return objInfo, jfsToObjectErr(ctx, eno, bucket, object) } + return minio.ObjectInfo{ Bucket: bucket, Name: object, @@ -861,6 +892,10 @@ func (n *jfsObjects) NewMultipartUpload(ctx context.Context, bucket string, obje } } } + err = n.setObjMeta(p, opts.UserDefined) + if err != nil { + logger.Errorf("set object metadata error, path: %s error %s", p, err) + } } return } @@ -871,6 +906,62 @@ const s3Etag = "s3-etag" // less than 64k ref: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html#tag-restrictions const s3Tags = "s3-tags" +// S3 object metadata +const s3Meta = "s3-meta" +const amzMeta = "x-amz-meta-" + +var s3UserControlledSystemMeta = []string{ + "cache-control", + "content-disposition", + "content-type", +} + +func (n *jfsObjects) getObjMeta(p string) (objMeta map[string]string, err error) { + if n.gConf.ObjMeta { + var errno syscall.Errno + var metadataStr []byte + if metadataStr, errno = n.fs.GetXattr(mctx, p, s3Meta); errno != 0 && errno != meta.ENOATTR { + return objMeta, errno + } + if len(metadataStr) > 0 { + err = json.Unmarshal(metadataStr, &objMeta) + return objMeta, err + } + } else { + objMeta = make(map[string]string) + } + return objMeta, nil +} + +func (n *jfsObjects) setObjMeta(p string, metadata map[string]string) error { + if n.gConf.ObjMeta && metadata != nil { + meta := make(map[string]string) + for k, v := range metadata { + k = strings.ToLower(k) + if strings.HasPrefix(k, amzMeta) { + meta[k] = v + } else { + for _, systemMetaKey := range s3UserControlledSystemMeta { + if k == systemMetaKey { + meta[k] = v + break + } + } + } + } + if len(meta) > 0 { + s3MetadataValue, err := json.Marshal(meta) + if err != nil { + return err + } + if eno := n.fs.SetXattr(mctx, p, s3Meta, s3MetadataValue, 0); eno != 0 { + logger.Errorf("set object metadata error, path: %s,value: %s error: %s", p, string(s3Meta), eno) + } + } + } + return nil +} + func (n *jfsObjects) ListMultipartUploads(ctx context.Context, bucket string, prefix string, keyMarker string, uploadIDMarker string, delimiter string, maxUploads int) (lmi minio.ListMultipartsInfo, err error) { if err = n.checkBucket(ctx, bucket); err != nil { return @@ -1100,6 +1191,15 @@ func (n *jfsObjects) CompleteMultipartUpload(ctx context.Context, bucket, object } } + var objMeta map[string]string + if n.gConf.ObjMeta { + if objMeta, err = n.getObjMeta(n.upath(bucket, uploadID)); err != nil { + logger.Errorf("get object meta error, path: %s, error: %s", n.upath(bucket, uploadID), err) + } else if err = n.setObjMeta(tmp, objMeta); err != nil { + logger.Errorf("set object meta error, path: %s, error: %s", tmp, err) + } + } + name := n.path(bucket, object) eno = n.fs.Rename(mctx, tmp, name, 0) if eno == syscall.ENOENT { @@ -1128,14 +1228,15 @@ func (n *jfsObjects) CompleteMultipartUpload(ctx context.Context, bucket, object // remove parts _ = n.fs.Rmr(mctx, n.upath(bucket, uploadID)) return minio.ObjectInfo{ - Bucket: bucket, - Name: object, - ETag: s3MD5, - ModTime: fi.ModTime(), - Size: fi.Size(), - IsDir: fi.IsDir(), - AccTime: fi.ModTime(), - UserTags: string(tagStr), + Bucket: bucket, + Name: object, + ETag: s3MD5, + ModTime: fi.ModTime(), + Size: fi.Size(), + IsDir: fi.IsDir(), + AccTime: fi.ModTime(), + UserTags: string(tagStr), + UserDefined: minio.CleanMetadata(opts.UserDefined), }, nil }