upload with more Put policy

cmd/upload.go

@@ -1,7 +1,6 @@
 package cmd
 
 import (
-	"github.com/qiniu/go-sdk/v7/storage"
 	"github.com/spf13/cobra"
 
 	"github.com/qiniu/qshell/v2/docs"
@@ -48,9 +47,7 @@ var upload2CmdBuilder = func(cfg *iqshell.Config) *cobra.Command {
 		LogRotate = 7
 	)
 	info := operations.BatchUpload2Info{
-		UploadConfig: operations.UploadConfig{
-			Policy: &storage.PutPolicy{},
-		},
+		UploadConfig: operations.UploadConfig{},
 	}
 	cmd := &cobra.Command{
 		Use:   "qupload2",
@@ -102,19 +99,34 @@ var upload2CmdBuilder = func(cfg *iqshell.Config) *cobra.Command {
 	cmd.Flags().IntVarP(&info.FileType, "storage", "", 0, "set storage type of file, same to --file-type")
 	_ = cmd.Flags().MarkDeprecated("storage", "use --file-type instead") // 废弃 storage
 
-	cmd.Flags().StringVarP(&info.Policy.CallbackURL, "callback-urls", "l", "", "upload callback urls, separated by comma")
-	cmd.Flags().StringVarP(&info.Policy.CallbackHost, "callback-host", "T", "", "upload callback host")
 	//cmd.Flags().StringVar(&cfg.CmdCfg.Up.BindUpIp, "bind-up-ip", "", "upload host ip to bind")
 	//cmd.Flags().StringVar(&cfg.CmdCfg.Up.BindRsIp, "bind-rs-ip", "", "rs host ip to bind")
 	//cmd.Flags().StringVar(&cfg.CmdCfg.Up.BindNicIp, "bind-nic-ip", "", "local network interface card to bind")
 
+	cmd.Flags().StringVarP(&info.EndUser, "end-user", "", "", "Owner identification")
+	cmd.Flags().StringVarP(&info.CallbackURL, "callback-urls", "l", "", "upload callback urls, separated by comma")
+	cmd.Flags().StringVarP(&info.CallbackHost, "callback-host", "T", "", "upload callback host")
+	cmd.Flags().StringVarP(&info.CallbackBody, "callback-body", "", "", "upload callback body")
+	cmd.Flags().StringVarP(&info.CallbackBodyType, "callback-body-type", "", "", "upload callback body type")
+	cmd.Flags().StringVarP(&info.PersistentOps, "persistent-ops", "", "", "List of pre-transfer persistence processing instructions that are triggered after successful resource upload. This parameter is not supported when fileType=2 or 3 (upload archive storage or deep archive storage files). Supports magic variables and custom variables. Each directive is an API specification string, and multiple directives are separated by ;.")
+	cmd.Flags().StringVarP(&info.PersistentNotifyURL, "persistent-notify-url", "", "", "URL to receive notification of persistence processing results. It must be a valid URL that can make POST requests normally on the public Internet and respond successfully. The content obtained by this URL is consistent with the processing result of the persistence processing status query. To send a POST request whose body format is application/json, you need to read the body of the request in the form of a read stream to obtain it.")
+	cmd.Flags().StringVarP(&info.PersistentPipeline, "persistent-pipeline", "", "", "Transcoding queue name. After the resource is successfully uploaded, an independent queue is designated for transcoding when transcoding is triggered. If it is empty, it means that the public queue is used, and the processing speed is slower. It is recommended to use a dedicated queue.")
+	cmd.Flags().IntVarP(&info.DetectMime, "detect-mime", "", 0, `Turn on the MimeType detection function and perform detection according to the following rules; if the correct value cannot be detected, application/octet-stream will be used by default.
+If set to a value of 1, the file MimeType information passed by the uploader will be ignored, and the MimeType value will be detected in the following order:
+	1. Detection content;
+	2. Check the file extension;
+	3. Check the Key extension.
+The default value is set to 0.If the uploader specifies MimeType (except application/octet-stream), this value will be used directly. Otherwise, the MimeType value will be detected in the following order:
+	1. Check the file extension;
+	2. Check the Key extension;
+	3. Detect content.
+Set to a value of -1 and use this value regardless of what value is specified on the uploader.`)
+	cmd.Flags().Uint64VarP(&info.TrafficLimit, "traffic-limit", "", 0, "Upload request single link speed limit to control client bandwidth usage. The speed limit value range is 819200 ~ 838860800, and the unit is bit/s.")
 	return cmd
 }
 
 var syncCmdBuilder = func(cfg *iqshell.Config) *cobra.Command {
-	info := operations.SyncInfo{
-		Policy: &storage.PutPolicy{},
-	}
+	info := operations.SyncInfo{}
 	cmd := &cobra.Command{
 		Use:   "sync <SrcResUrl> <Buckets> [-k <Key>]",
 		Short: "Sync big file to qiniu bucket",
@@ -141,13 +153,30 @@ var syncCmdBuilder = func(cfg *iqshell.Config) *cobra.Command {
 
 	cmd.Flags().BoolVarP(&info.Overwrite, "overwrite", "", false, "overwrite the file of same key in bucket")
 
+	cmd.Flags().StringVarP(&info.Policy.EndUser, "end-user", "", "", "Owner identification")
+	cmd.Flags().StringVarP(&info.Policy.CallbackURL, "callback-urls", "l", "", "upload callback urls, separated by comma")
+	cmd.Flags().StringVarP(&info.Policy.CallbackHost, "callback-host", "T", "", "upload callback host")
+	cmd.Flags().StringVarP(&info.Policy.CallbackBody, "callback-body", "", "", "upload callback body")
+	cmd.Flags().StringVarP(&info.Policy.CallbackBodyType, "callback-body-type", "", "", "upload callback body type")
+	cmd.Flags().StringVarP(&info.Policy.PersistentOps, "persistent-ops", "", "", "List of pre-transfer persistence processing instructions that are triggered after successful resource upload. This parameter is not supported when fileType=2 or 3 (upload archive storage or deep archive storage files). Supports magic variables and custom variables. Each directive is an API specification string, and multiple directives are separated by ;.")
+	cmd.Flags().StringVarP(&info.Policy.PersistentNotifyURL, "persistent-notify-url", "", "", "URL to receive notification of persistence processing results. It must be a valid URL that can make POST requests normally on the public Internet and respond successfully. The content obtained by this URL is consistent with the processing result of the persistence processing status query. To send a POST request whose body format is application/json, you need to read the body of the request in the form of a read stream to obtain it.")
+	cmd.Flags().StringVarP(&info.Policy.PersistentPipeline, "persistent-pipeline", "", "", "Transcoding queue name. After the resource is successfully uploaded, an independent queue is designated for transcoding when transcoding is triggered. If it is empty, it means that the public queue is used, and the processing speed is slower. It is recommended to use a dedicated queue.")
+	cmd.Flags().IntVarP(&info.Policy.DetectMime, "detect-mime", "", 0, `Turn on the MimeType detection function and perform detection according to the following rules; if the correct value cannot be detected, application/octet-stream will be used by default.
+If set to a value of 1, the file MimeType information passed by the uploader will be ignored, and the MimeType value will be detected in the following order:
+	1. Detection content;
+	2. Check the file extension;
+	3. Check the Key extension.
+The default value is set to 0.If the uploader specifies MimeType (except application/octet-stream), this value will be used directly. Otherwise, the MimeType value will be detected in the following order:
+	1. Check the file extension;
+	2. Check the Key extension;
+	3. Detect content.
+Set to a value of -1 and use this value regardless of what value is specified on the uploader.`)
+	cmd.Flags().Uint64VarP(&info.Policy.TrafficLimit, "traffic-limit", "", 0, "Upload request single link speed limit to control client bandwidth usage. The speed limit value range is 819200 ~ 838860800, and the unit is bit/s.")
 	return cmd
 }
 
 var formUploadCmdBuilder = func(cfg *iqshell.Config) *cobra.Command {
-	info := operations.UploadInfo{
-		Policy: &storage.PutPolicy{},
-	}
+	info := operations.UploadInfo{}
 	cmd := &cobra.Command{
 		Use:   "fput <Bucket> <Key> <LocalFile>",
 		Short: "Form upload a local file",
@@ -174,15 +203,31 @@ var formUploadCmdBuilder = func(cfg *iqshell.Config) *cobra.Command {
 	_ = cmd.Flags().MarkDeprecated("storage", "use --file-type instead") // 废弃 storage
 
 	cmd.Flags().StringVarP(&info.UpHost, "up-host", "u", "", "uphost")
+
+	cmd.Flags().StringVarP(&info.Policy.EndUser, "end-user", "", "", "Owner identification")
 	cmd.Flags().StringVarP(&info.Policy.CallbackURL, "callback-urls", "l", "", "upload callback urls, separated by comma")
 	cmd.Flags().StringVarP(&info.Policy.CallbackHost, "callback-host", "T", "", "upload callback host")
+	cmd.Flags().StringVarP(&info.Policy.CallbackBody, "callback-body", "", "", "upload callback body")
+	cmd.Flags().StringVarP(&info.Policy.CallbackBodyType, "callback-body-type", "", "", "upload callback body type")
+	cmd.Flags().StringVarP(&info.Policy.PersistentOps, "persistent-ops", "", "", "List of pre-transfer persistence processing instructions that are triggered after successful resource upload. This parameter is not supported when fileType=2 or 3 (upload archive storage or deep archive storage files). Supports magic variables and custom variables. Each directive is an API specification string, and multiple directives are separated by ;.")
+	cmd.Flags().StringVarP(&info.Policy.PersistentNotifyURL, "persistent-notify-url", "", "", "URL to receive notification of persistence processing results. It must be a valid URL that can make POST requests normally on the public Internet and respond successfully. The content obtained by this URL is consistent with the processing result of the persistence processing status query. To send a POST request whose body format is application/json, you need to read the body of the request in the form of a read stream to obtain it.")
+	cmd.Flags().StringVarP(&info.Policy.PersistentPipeline, "persistent-pipeline", "", "", "Transcoding queue name. After the resource is successfully uploaded, an independent queue is designated for transcoding when transcoding is triggered. If it is empty, it means that the public queue is used, and the processing speed is slower. It is recommended to use a dedicated queue.")
+	cmd.Flags().IntVarP(&info.Policy.DetectMime, "detect-mime", "", 0, `Turn on the MimeType detection function and perform detection according to the following rules; if the correct value cannot be detected, application/octet-stream will be used by default.
+If set to a value of 1, the file MimeType information passed by the uploader will be ignored, and the MimeType value will be detected in the following order:
+	1. Detection content;
+	2. Check the file extension;
+	3. Check the Key extension.
+The default value is set to 0.If the uploader specifies MimeType (except application/octet-stream), this value will be used directly. Otherwise, the MimeType value will be detected in the following order:
+	1. Check the file extension;
+	2. Check the Key extension;
+	3. Detect content.
+Set to a value of -1 and use this value regardless of what value is specified on the uploader.`)
+	cmd.Flags().Uint64VarP(&info.Policy.TrafficLimit, "traffic-limit", "", 0, "Upload request single link speed limit to control client bandwidth usage. The speed limit value range is 819200 ~ 838860800, and the unit is bit/s.")
 	return cmd
 }
 
 var resumeUploadCmdBuilder = func(cfg *iqshell.Config) *cobra.Command {
-	info := operations.UploadInfo{
-		Policy: &storage.PutPolicy{},
-	}
+	info := operations.UploadInfo{}
 	cmd := &cobra.Command{
 		Use:   "rput <Bucket> <Key> <LocalFile>",
 		Short: "Resumable upload a local file",
@@ -217,8 +262,26 @@ var resumeUploadCmdBuilder = func(cfg *iqshell.Config) *cobra.Command {
 
 	cmd.Flags().IntVarP(&info.ResumeWorkerCount, "worker", "c", 3, "worker count")
 	cmd.Flags().StringVarP(&info.UpHost, "up-host", "u", "", "uphost")
+
+	cmd.Flags().StringVarP(&info.Policy.EndUser, "end-user", "", "", "Owner identification")
 	cmd.Flags().StringVarP(&info.Policy.CallbackURL, "callback-urls", "l", "", "upload callback urls, separated by comma")
 	cmd.Flags().StringVarP(&info.Policy.CallbackHost, "callback-host", "T", "", "upload callback host")
+	cmd.Flags().StringVarP(&info.Policy.CallbackBody, "callback-body", "", "", "upload callback body")
+	cmd.Flags().StringVarP(&info.Policy.CallbackBodyType, "callback-body-type", "", "", "upload callback body type")
+	cmd.Flags().StringVarP(&info.Policy.PersistentOps, "persistent-ops", "", "", "List of pre-transfer persistence processing instructions that are triggered after successful resource upload. This parameter is not supported when fileType=2 or 3 (upload archive storage or deep archive storage files). Supports magic variables and custom variables. Each directive is an API specification string, and multiple directives are separated by ;.")
+	cmd.Flags().StringVarP(&info.Policy.PersistentNotifyURL, "persistent-notify-url", "", "", "URL to receive notification of persistence processing results. It must be a valid URL that can make POST requests normally on the public Internet and respond successfully. The content obtained by this URL is consistent with the processing result of the persistence processing status query. To send a POST request whose body format is application/json, you need to read the body of the request in the form of a read stream to obtain it.")
+	cmd.Flags().StringVarP(&info.Policy.PersistentPipeline, "persistent-pipeline", "", "", "Transcoding queue name. After the resource is successfully uploaded, an independent queue is designated for transcoding when transcoding is triggered. If it is empty, it means that the public queue is used, and the processing speed is slower. It is recommended to use a dedicated queue.")
+	cmd.Flags().IntVarP(&info.Policy.DetectMime, "detect-mime", "", 0, `Turn on the MimeType detection function and perform detection according to the following rules; if the correct value cannot be detected, application/octet-stream will be used by default.
+If set to a value of 1, the file MimeType information passed by the uploader will be ignored, and the MimeType value will be detected in the following order:
+	1. Detection content;
+	2. Check the file extension;
+	3. Check the Key extension.
+The default value is set to 0. If the uploader specifies MimeType (except application/octet-stream), this value will be used directly. Otherwise, the MimeType value will be detected in the following order:
+	1. Check the file extension;
+	2. Check the Key extension;
+	3. Detect content.
+Set to a value of -1 and use this value regardless of what value is specified on the uploader.`)
+	cmd.Flags().Uint64VarP(&info.Policy.TrafficLimit, "traffic-limit", "", 0, "Upload request single link speed limit to control client bandwidth usage. The speed limit value range is 819200 ~ 838860800, and the unit is bit/s.")
 	return cmd
 }
 

cmd_test/upload_form_test.go

@@ -3,9 +3,10 @@
 package cmd
 
 import (
-	"github.com/qiniu/qshell/v2/cmd_test/test"
 	"strings"
 	"testing"
+
+	"github.com/qiniu/qshell/v2/cmd_test/test"
 )
 
 func TestFormUpload(t *testing.T) {
@@ -19,7 +20,8 @@ func TestFormUpload(t *testing.T) {
 	result, errs := test.RunCmdWithError("fput", test.Bucket, "qshell_fput_1M", path,
 		"--mimetype", "image/jpg",
 		"--storage", "0",
-		"--file-type", "1")
+		"--file-type", "1",
+		"--end-user", "10")
 	if len(errs) > 0 {
 		t.Fail()
 	}

cmd_test/upload_test.go

@@ -4,11 +4,12 @@ package cmd
 
 import (
 	"encoding/json"
-	"github.com/qiniu/qshell/v2/cmd_test/test"
-	"github.com/qiniu/qshell/v2/iqshell/storage/object/upload/operations"
 	"path/filepath"
 	"strings"
 	"testing"
+
+	"github.com/qiniu/qshell/v2/cmd_test/test"
+	"github.com/qiniu/qshell/v2/iqshell/storage/object/upload/operations"
 )
 
 func TestQUpload(t *testing.T) {
@@ -71,7 +72,6 @@ func TestQUpload(t *testing.T) {
 			DisableForm:            false,
 			WorkerCount:            4,
 			RecordRoot:             "",
-			Policy:                 nil,
 		},
 		LogFile:    logPath,
 		RecordRoot: recordPath,

docs/fput.md

@@ -35,6 +35,26 @@ $ qshell fput --doc
 - -u/--up-host: 指定上传域名。 【可选】
 - -l/--callback-urls：上传回调地址， 可以指定多个地址，以逗号分隔。 【可选】
 - -T/--callback-host：上传回调的 HOST, 必须和 CallbackUrls 一起指定。 【可选】
+-    --callback-body：上传成功后，七牛云向业务服务器发送 Content-Type: application/x-www-form-urlencoded 的 POST 请求。业务服务器可以通过直接读取请求的 query 来获得该字段，支持魔法变量和自定义变量。callbackBody 要求是合法的 url query string。例如key=$(key)&hash=$(etag)&w=$(imageInfo.width)&h=$(imageInfo.height)。如果callbackBodyType指定为application/json，则callbackBody应为json格式，例如:{“key”:"$(key)",“hash”:"$(etag)",“w”:"$(imageInfo.width)",“h”:"$(imageInfo.height)"}。【可选】
+-    --callback-body-type：上传成功后，七牛云向业务服务器发送回调通知 callbackBody 的 Content-Type。默认为 application/x-www-form-urlencoded，也可设置为 application/json。【可选】
+-    --end-user：上传成功后，七牛云向业务服务器发送回调通知 callbackBody 的 Content-Type。默认为 application/x-www-form-urlencoded，也可设置为 application/json。【可选】
+-    --persistent-ops：资源上传成功后触发执行的预转持久化处理指令列表。fileType=2或3（上传归档存储或深度归档存储文件）时，不支持使用该参数。支持魔法变量和自定义变量。每个指令是一个 API 规格字符串，多个指令用;分隔。【可选】
+-    --persistent-notify-url：接收持久化处理结果通知的 URL。必须是公网上可以正常进行 POST 请求并能成功响应的有效 URL。该 URL 获取的内容和持久化处理状态查询的处理结果一致。发送 body 格式是 Content-Type 为 application/json 的 POST 请求，需要按照读取流的形式读取请求的 body 才能获取。【可选】
+-    --persistent-pipeline：转码队列名。资源上传成功后，触发转码时指定独立的队列进行转码。为空则表示使用公用队列，处理速度比较慢。建议使用专用队列。【可选】
+-    --detect-mime：开启 MimeType 侦测功能，并按照下述规则进行侦测；如不能侦测出正确的值，会默认使用 application/octet-stream 。【可选】
+```
+    1. 设为 1 值，则忽略上传端传递的文件 MimeType 信息，并按如下顺序侦测 MimeType 值：
+        1) 侦测内容；
+        2) 检查文件扩展名；
+        3) 检查 Key 扩展名。
+    2. 默认设为 0 值，如上传端指定了 MimeType（application/octet-stream 除外）则直接使用该值，否则按如下顺序侦测 MimeType 值：
+        1) 检查文件扩展名；
+        2) 检查 Key 扩展名；
+        3) 侦测内容。
+    3. 设为 -1 值，无论上传端指定了何值直接使用该值。
+```  
+-    --traffic-limit：上传请求单链接速度限制，控制客户端带宽占用。限速值取值范围为 819200 ~ 838860800，单位为 bit/s。【可选】
+
 
 # 示例
 1 上传本地文件 `/Users/jemy/Documents/qiniu.jpg` 到空间 `if-pbl` 里面。