Gateway: add object metadata

cmd/gateway.go

@@ -80,6 +80,10 @@ func cmdGateway() *cli.Command {
 			Name:  "object-tag",
 			Usage: "enable object tagging api",
 		},
+		&cli.BoolFlag{
+			Name:  "object-meta",
+			Usage: "enable object metadata api",
+		},
 		&cli.StringFlag{
 			Name:  "domain",
 			Usage: "domain for virtual-host-style requests",
@@ -149,10 +153,11 @@ func gateway(c *cli.Context) error {
 		jfs,
 		conf,
 		&jfsgateway.Config{
-			MultiBucket: c.Bool("multi-buckets"),
-			KeepEtag:    c.Bool("keep-etag"),
-			Umask:       uint16(umask),
-			ObjTag:      c.Bool("object-tag"),
+			MultiBucket:   c.Bool("multi-buckets"),
+			KeepEtag:      c.Bool("keep-etag"),
+			Umask:         uint16(umask),
+			ObjTag:        c.Bool("object-tag"),
+			ObjMeta:       c.Bool("object-meta"),
 		},
 	)
 	if err != nil {

docs/en/guide/gateway.md

@@ -147,6 +147,10 @@ By default, JuiceFS S3 Gateway does not save or return object ETag information.
 
 Object tags are not supported by default, but you can use `--object-tag` to enable them.
 
+### Enable object metadata
+
+Object metadata is not supported by default, but you can use `--object-meta` to enable them.
+
 ### Enable virtual host-style requests
 
 By default, JuiceFS S3 Gateway supports path-style requests in the format of `http://mydomain.com/bucket/object`. The `MINIO_DOMAIN` environment variable is used to enable virtual host-style requests. If the request's `Host` header information matches `(.+).mydomain.com`, the matched pattern `$1` is used as the bucket, and the path is used as the object.

pkg/gateway/gateway.go

@@ -18,6 +18,7 @@ package gateway
 
 import (
 	"context"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
@@ -56,10 +57,11 @@ var mctx meta.Context
 var logger = utils.GetLogger("juicefs")
 
 type Config struct {
-	MultiBucket bool
-	KeepEtag    bool
-	Umask       uint16
-	ObjTag      bool
+	MultiBucket   bool
+	KeepEtag      bool
+	Umask         uint16
+	ObjTag        bool
+	ObjMeta       bool
 }
 
 func NewJFSGateway(jfs *fs.FileSystem, conf *vfs.Config, gConf *Config) (minio.ObjectLayer, error) {
@@ -533,6 +535,12 @@ func (n *jfsObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBu
 	}
 	dst := n.path(dstBucket, dstObject)
 	src := n.path(srcBucket, srcObject)
+
+	err = n.setObjMeta(dst, srcInfo.UserDefined)
+	if err != nil {
+		logger.Errorf("set object metadata error, path: %s error %s", dst, err)
+	}
+
 	if minio.IsStringEqual(src, dst) {
 		return n.GetObjectInfo(ctx, srcBucket, srcObject, minio.ObjectOptions{})
 	}
@@ -687,6 +695,18 @@ func (n *jfsObjects) GetObjectInfo(ctx context.Context, bucket, object string, o
 			return minio.ObjectInfo{}, errno
 		}
 	}
+	objMeta, err := n.getObjMeta(n.path(bucket, object))
+	if err != nil {
+		return minio.ObjectInfo{}, err
+	}
+	if opts.UserDefined == nil {
+		opts.UserDefined = make(map[string]string)
+	}
+	for k, v := range objMeta {
+		opts.UserDefined[k] = v
+	}
+	contentType := utils.GuessMimeType(object)
+	contentType = n.getObjContentType(objMeta, contentType)
 	return minio.ObjectInfo{
 		Bucket:      bucket,
 		Name:        object,
@@ -695,7 +715,7 @@ func (n *jfsObjects) GetObjectInfo(ctx context.Context, bucket, object string, o
 		IsDir:       fi.IsDir(),
 		AccTime:     fi.ModTime(),
 		ETag:        string(etag),
-		ContentType: utils.GuessMimeType(object),
+		ContentType: contentType,
 		UserTags:    string(tagStr),
 		UserDefined: minio.CleanMetadata(opts.UserDefined),
 	}, nil
@@ -821,6 +841,10 @@ func (n *jfsObjects) PutObject(ctx context.Context, bucket string, object string
 					}
 				}
 			}
+			err = n.setObjMeta(tmpName, opts.UserDefined)
+			if err != nil {
+				logger.Errorf("set object metadata error, path: %s error %s", p, err)
+			}
 		}); err != nil {
 			return
 		}
@@ -829,6 +853,7 @@ func (n *jfsObjects) PutObject(ctx context.Context, bucket string, object string
 	if eno != 0 {
 		return objInfo, jfsToObjectErr(ctx, eno, bucket, object)
 	}
+
 	return minio.ObjectInfo{
 		Bucket:      bucket,
 		Name:        object,
@@ -861,6 +886,10 @@ func (n *jfsObjects) NewMultipartUpload(ctx context.Context, bucket string, obje
 				}
 			}
 		}
+		err = n.setObjMeta(p, opts.UserDefined)
+		if err != nil {
+			logger.Errorf("set object metadata error, path: %s  error %s", p, err)
+		}
 	}
 	return
 }
@@ -871,6 +900,73 @@ const s3Etag = "s3-etag"
 // less than 64k ref: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html#tag-restrictions
 const s3Tags = "s3-tags"
 
+// S3 object metadata
+const s3Meta = "s3-meta"
+const amzMeta = "x-amz-meta-"
+const metaContentType = "content-type"
+
+var s3UserControlledSystemMeta = []string{
+	"cache-control",
+	"content-disposition",
+	"content-type",
+}
+
+func (n *jfsObjects) getObjMeta(p string) (objMeta map[string]string, err error) {
+	if n.gConf.ObjMeta {
+		var errno syscall.Errno
+		var metadataStr []byte
+		if metadataStr, errno = n.fs.GetXattr(mctx, p, s3Meta); errno != 0 && errno != meta.ENOATTR {
+			return objMeta, errno
+		}
+		if len(metadataStr) > 0 {
+			err = json.Unmarshal(metadataStr, &objMeta)
+			return objMeta, err
+		}
+	} else {
+		objMeta = make(map[string]string)
+	}
+	return objMeta, nil
+}
+
+func (n *jfsObjects) getObjContentType(objMeta map[string]string, fileContentType string) (contentType string) {
+	var exist bool
+	contentType, exist = objMeta[metaContentType]
+	if !exist || len(contentType) == 0 {
+		return fileContentType
+	} else {
+		return contentType
+	}
+}
+
+func (n *jfsObjects) setObjMeta(p string, metadata map[string]string) error {
+	if n.gConf.ObjMeta && metadata != nil {
+		meta := make(map[string]string)
+		for k, v := range metadata {
+			k = strings.ToLower(k)
+			if strings.HasPrefix(k, amzMeta) {
+				meta[k] = v
+			} else {
+				for _, systemMetaKey := range s3UserControlledSystemMeta {
+					if k == systemMetaKey {
+						meta[k] = v
+						break
+					}
+				}
+			}
+		}
+		if len(meta) > 0 {
+			s3MetadataValue, err := json.Marshal(meta)
+			if err != nil {
+				return err
+			}
+			if eno := n.fs.SetXattr(mctx, p, s3Meta, s3MetadataValue, 0); eno != 0 {
+				logger.Errorf("set object metadata error, path: %s,value: %s error: %s", p, string(s3Meta), eno)
+			}
+		}
+	}
+	return nil
+}
+
 func (n *jfsObjects) ListMultipartUploads(ctx context.Context, bucket string, prefix string, keyMarker string, uploadIDMarker string, delimiter string, maxUploads int) (lmi minio.ListMultipartsInfo, err error) {
 	if err = n.checkBucket(ctx, bucket); err != nil {
 		return
@@ -1100,6 +1196,15 @@ func (n *jfsObjects) CompleteMultipartUpload(ctx context.Context, bucket, object
 		}
 	}
 
+	var objMeta map[string]string
+	if n.gConf.ObjMeta {
+		if objMeta, err = n.getObjMeta(n.upath(bucket, uploadID)); err != nil {
+			logger.Errorf("get object meta error, path: %s, error: %s", n.upath(bucket, uploadID), err)
+		} else if err = n.setObjMeta(tmp, objMeta); err != nil {
+			logger.Errorf("set object meta error, path: %s, error: %s", tmp, err)
+		}
+	}
+
 	name := n.path(bucket, object)
 	eno = n.fs.Rename(mctx, tmp, name, 0)
 	if eno == syscall.ENOENT {
@@ -1128,14 +1233,15 @@ func (n *jfsObjects) CompleteMultipartUpload(ctx context.Context, bucket, object
 	// remove parts
 	_ = n.fs.Rmr(mctx, n.upath(bucket, uploadID))
 	return minio.ObjectInfo{
-		Bucket:   bucket,
-		Name:     object,
-		ETag:     s3MD5,
-		ModTime:  fi.ModTime(),
-		Size:     fi.Size(),
-		IsDir:    fi.IsDir(),
-		AccTime:  fi.ModTime(),
-		UserTags: string(tagStr),
+		Bucket:      bucket,
+		Name:        object,
+		ETag:        s3MD5,
+		ModTime:     fi.ModTime(),
+		Size:        fi.Size(),
+		IsDir:       fi.IsDir(),
+		AccTime:     fi.ModTime(),
+		UserTags:    string(tagStr),
+		UserDefined: minio.CleanMetadata(opts.UserDefined),
 	}, nil
 }