diff --git a/CHANGELOG.md b/CHANGELOG.md index 803a14cd..c90f5706 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,12 @@ +# 2.16.0 +## 新增 +1. create-share / share-ls / share-cp 命令支持创建对文件夹的共享,列举和下载 +2. 上传命令支持额外的 --accelerate 选项 +3. fop 命令支持 --workflow-template-id 和 --type 选项 + +## 更新 +1. listbucket2 现在即使使用了 --limit 参数得到的 marker 也是精确的 + # 2.15.0 ## 新增 1. buckets 命令列举全量 bucket @@ -56,7 +65,7 @@ # 2.9.0 1. 新增 batchforbidden 命令,支持批量修改文件启用/禁用状态,具体参考命令使用说明 2. qdownload 命令新增支持自定义文件本地存储路径, 具体参考命令使用说明 -4. stat 命令补充部分文件元数据信息,比如过期删除时间等 +4. stat 命令补充部分文件元数据信息,比如过期删除时间等 5. bucket 命令补充部分空间信息,比如可查看空间配置的 Refer 等 6. 优化 qupload 关于上传流程中检测文件是否存在机制 7. 优化 qdownload 关于下载流程中检测文件是否存在机制 diff --git a/Makefile b/Makefile index 918a2e5b..b1c0284f 100644 --- a/Makefile +++ b/Makefile @@ -32,52 +32,52 @@ linux: $(LINUX86).zip $(LINUX64).zip $(LINUXARM).zip $(LINUXARM64).zip $(LINUXMI windows: $(WIN86).zip $(WIN64).zip $(WINARM).zip $(WINARM64).zip qshell-$(VERSION)-darwin-%.zip: qshell-$(VERSION)-darwin-% - @zip $@ $< + zip $@ $< qshell-$(VERSION)-linux-%.zip: qshell-$(VERSION)-linux-% - @zip $@ $< + zip $@ $< qshell-$(VERSION)-windows-%.zip: qshell-$(VERSION)-windows-%.exe - @zip $@ $< + zip $@ $< $(DARWIN): - @GOOS=darwin GOARCH=amd64 $(GO) build -ldflags $(LDFLAGS) -o $(DARWIN) . + GOOS=darwin GOARCH=amd64 $(GO) build -ldflags $(LDFLAGS) -o $(DARWIN) ./main/ $(DARWINARM64): - @GOOS=darwin GOARCH=arm64 $(GO) build -ldflags $(LDFLAGS) -o $(DARWINARM64) . + GOOS=darwin GOARCH=arm64 $(GO) build -ldflags $(LDFLAGS) -o $(DARWINARM64) ./main/ $(LINUX86): - @CGO_ENABLED=0 GOOS=linux GOARCH=386 $(GO) build -ldflags $(LDFLAGS) -o $(LINUX86) . + CGO_ENABLED=0 GOOS=linux GOARCH=386 $(GO) build -ldflags $(LDFLAGS) -o $(LINUX86) ./main/ $(LINUX64): - @CGO_ENABLED=0 GOOS=linux GOARCH=amd64 $(GO) build -ldflags $(LDFLAGS) -o $(LINUX64) . + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 $(GO) build -ldflags $(LDFLAGS) -o $(LINUX64) ./main/ $(LINUXARM): - @CGO_ENABLED=0 GOOS=linux GOARCH=arm $(GO) build -ldflags $(LDFLAGS) -o $(LINUXARM) . + CGO_ENABLED=0 GOOS=linux GOARCH=arm $(GO) build -ldflags $(LDFLAGS) -o $(LINUXARM) ./main/ $(LINUXARM64): - @CGO_ENABLED=0 GOOS=linux GOARCH=arm64 $(GO) build -ldflags $(LDFLAGS) -o $(LINUXARM64) . + CGO_ENABLED=0 GOOS=linux GOARCH=arm64 $(GO) build -ldflags $(LDFLAGS) -o $(LINUXARM64) ./main/ $(LINUXMIPS): - @CGO_ENABLED=0 GOOS=linux GOARCH=mips $(GO) build -ldflags $(LDFLAGS) -o $(LINUXMIPS) . + CGO_ENABLED=0 GOOS=linux GOARCH=mips $(GO) build -ldflags $(LDFLAGS) -o $(LINUXMIPS) ./main/ $(LINUXMIPSLE): - @CGO_ENABLED=0 GOOS=linux GOARCH=mipsle $(GO) build -ldflags $(LDFLAGS) -o $(LINUXMIPSLE) . + CGO_ENABLED=0 GOOS=linux GOARCH=mipsle $(GO) build -ldflags $(LDFLAGS) -o $(LINUXMIPSLE) ./main/ $(LINUXMIPS64): - @CGO_ENABLED=0 GOOS=linux GOARCH=mips64 $(GO) build -ldflags $(LDFLAGS) -o $(LINUXMIPS64) . + CGO_ENABLED=0 GOOS=linux GOARCH=mips64 $(GO) build -ldflags $(LDFLAGS) -o $(LINUXMIPS64) ./main/ $(LINUXMIPS64LE): - @CGO_ENABLED=0 GOOS=linux GOARCH=mips64le $(GO) build -ldflags $(LDFLAGS) -o $(LINUXMIPS64LE) . + CGO_ENABLED=0 GOOS=linux GOARCH=mips64le $(GO) build -ldflags $(LDFLAGS) -o $(LINUXMIPS64LE) ./main/ $(LINUXLOONG64): - @CGO_ENABLED=0 GOOS=linux GOARCH=loong64 $(GO) build -ldflags $(LDFLAGS) -o $(LINUXLOONG64) . + CGO_ENABLED=0 GOOS=linux GOARCH=loong64 $(GO) build -ldflags $(LDFLAGS) -o $(LINUXLOONG64) ./main/ $(LINUXRISCV64): - @CGO_ENABLED=0 GOOS=linux GOARCH=riscv64 $(GO) build -ldflags $(LDFLAGS) -o $(LINUXRISCV64) . + CGO_ENABLED=0 GOOS=linux GOARCH=riscv64 $(GO) build -ldflags $(LDFLAGS) -o $(LINUXRISCV64) ./main/ $(WIN86).exe: - @CGO_ENABLED=0 GOOS=windows GOARCH=386 $(GO) build -ldflags $(LDFLAGS) -o $(WIN86).exe . + CGO_ENABLED=0 GOOS=windows GOARCH=386 $(GO) build -ldflags $(LDFLAGS) -o $(WIN86).exe ./main/ $(WIN64).exe: - @CGO_ENABLED=0 GOOS=windows GOARCH=amd64 $(GO) build -ldflags $(LDFLAGS) -o $(WIN64).exe . + CGO_ENABLED=0 GOOS=windows GOARCH=amd64 $(GO) build -ldflags $(LDFLAGS) -o $(WIN64).exe ./main/ $(WINARM).exe: - @CGO_ENABLED=0 GOOS=windows GOARCH=arm $(GO) build -ldflags $(LDFLAGS) -o $(WINARM).exe . + CGO_ENABLED=0 GOOS=windows GOARCH=arm $(GO) build -ldflags $(LDFLAGS) -o $(WINARM).exe ./main/ $(WINARM64).exe: - @CGO_ENABLED=0 GOOS=windows GOARCH=arm64 $(GO) build -ldflags $(LDFLAGS) -o $(WINARM64).exe . + CGO_ENABLED=0 GOOS=windows GOARCH=arm64 $(GO) build -ldflags $(LDFLAGS) -o $(WINARM64).exe ./main/ .PHONY: cleanzip cleanbin clean upload cleanzip: - @rm -f qshell-$(VERSION)-*.zip + rm -f qshell-$(VERSION)-*.zip cleanbin: - @rm -f qshell-$(VERSION)-* + rm -f qshell-$(VERSION)-* clean: cleanzip cleanbin diff --git a/cmd/fop.go b/cmd/fop.go index bb4e08e2..2578bae9 100644 --- a/cmd/fop.go +++ b/cmd/fop.go @@ -26,8 +26,8 @@ var preFopStatusCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { return cmd } -var preFopCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { - var info = operations.PreFopInfo{} +var pfopCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { + var info = operations.PfopInfo{} var cmd = &cobra.Command{ Use: "pfop ", Short: "Issue a request to process file in bucket", @@ -42,14 +42,17 @@ var preFopCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { if len(args) > 2 { info.Fops = args[2] } - operations.PreFop(cfg, info) + operations.Pfop(cfg, info) }, } cmd.Flags().StringVarP(&info.Pipeline, "pipeline", "p", "", "task pipeline") cmd.Flags().StringVarP(&info.NotifyURL, "notify-url", "u", "", "notfiy url") + cmd.Flags().StringVarP(&info.WorkflowTemplateID, "workflow-template-id", "", "", "Workflow template ID") - cmd.Flags().BoolVarP(&info.NotifyForce, "force", "y", false, "force execute") - cmd.Flags().BoolVarP(&info.NotifyForce, "force-old", "f", false, "force execute, deprecated") + cmd.Flags().BoolVarP(&info.Force, "force", "y", false, "force execute") + cmd.Flags().BoolVarP(&info.Force, "force-old", "f", false, "force execute, deprecated") + + cmd.Flags().Int64VarP(&info.Type, "type", "", 0, "task type") _ = cmd.Flags().MarkDeprecated("force-old", "use --force instead") return cmd @@ -61,7 +64,7 @@ func init() { func fopCmdLoader(superCmd *cobra.Command, cfg *iqshell.Config) { superCmd.AddCommand( - preFopCmdBuilder(cfg), + pfopCmdBuilder(cfg), preFopStatusCmdBuilder(cfg), ) } diff --git a/cmd/share.go b/cmd/share.go new file mode 100644 index 00000000..e1a6d17d --- /dev/null +++ b/cmd/share.go @@ -0,0 +1,90 @@ +package cmd + +import ( + "strings" + + "github.com/qiniu/qshell/v2/docs" + "github.com/qiniu/qshell/v2/iqshell" + "github.com/qiniu/qshell/v2/iqshell/storage/object/operations" + "github.com/spf13/cobra" +) + +var createShareCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { + var info = operations.CreateShareInfo{} + var cmd = &cobra.Command{ + Use: "create-share [kodo://]/", + Short: "Share the specified directory", + Run: func(cmd *cobra.Command, args []string) { + cfg.CmdCfg.CmdId = docs.CreateShareType + if len(args) > 0 { + url := args[0] + url = strings.TrimPrefix(url, "kodo://") + parts := strings.SplitN(url, "/", 2) + if len(parts) > 0 { + info.Bucket = parts[0] + } + if len(parts) > 1 { + info.Prefix = parts[1] + } + info.Permission = "READONLY" + } + operations.CreateShare(cfg, info) + }, + } + cmd.Flags().StringVarP(&info.ExtractCode, "extract-code", "", "", "Specify extract code for the share, must consist of 6 alphabets and numbers") + cmd.Flags().Int64VarP(&info.DurationSeconds, "validity-period", "", 900, "Specify validity period in seconds, must be longer than 1 mintutes and shorter than 2 hours") + cmd.Flags().StringVarP(&info.OutputPath, "output", "", "", "Specify path to save output") + return cmd +} + +var listShareCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { + var info = operations.ListShareInfo{} + var cmd = &cobra.Command{ + Use: "share-ls ", + Short: "List shared files", + Run: func(cmd *cobra.Command, args []string) { + cfg.CmdCfg.CmdId = docs.ShareLsType + if len(args) > 0 { + info.LinkURL = args[0] + } + operations.ListShare(cfg, info) + }, + } + cmd.Flags().StringVarP(&info.ExtractCode, "extract-code", "", "", "Specify extract code for the share, must consist of 6 alphabets and numbers") + cmd.Flags().StringVarP(&info.Prefix, "prefix", "p", "", "list the shared directory with this prefix if set") + cmd.Flags().Int64VarP(&info.Limit, "limit", "n", 0, "max count of items to output") + cmd.Flags().StringVarP(&info.Marker, "marker", "m", "", "list marker") + return cmd +} + +var copyShareCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { + var info = operations.CopyShareInfo{} + var cmd = &cobra.Command{ + Use: "share-cp --to=", + Short: "Copy shared files", + Run: func(cmd *cobra.Command, args []string) { + cfg.CmdCfg.CmdId = docs.ShareCpType + if len(args) > 0 { + info.LinkURL = args[0] + } + operations.CopyShare(cfg, info) + }, + } + cmd.Flags().StringVarP(&info.ExtractCode, "extract-code", "", "", "Specify extract code for the share, must consist of 6 alphabets and numbers") + cmd.Flags().StringVarP(&info.FromPath, "from", "", "", "copy from path") + cmd.Flags().StringVarP(&info.ToPath, "to", "", "", "copy to path") + cmd.Flags().BoolVarP(&info.Recursive, "recursive", "r", false, "copy directories recursively") + return cmd +} + +func init() { + registerLoader(shareCmdLoader) +} + +func shareCmdLoader(superCmd *cobra.Command, cfg *iqshell.Config) { + superCmd.AddCommand( + createShareCmdBuilder(cfg), + listShareCmdBuilder(cfg), + copyShareCmdBuilder(cfg), + ) +} diff --git a/cmd/upload.go b/cmd/upload.go index 11eed348..38515d63 100644 --- a/cmd/upload.go +++ b/cmd/upload.go @@ -90,6 +90,7 @@ var upload2CmdBuilder = func(cfg *iqshell.Config) *cobra.Command { cmd.Flags().StringVar(&info.SkipFixedStrings, "skip-fixed-strings", "", "skip files with the fixed string in the name") cmd.Flags().StringVar(&info.SkipSuffixes, "skip-suffixes", "", "skip files with these suffixes") cmd.Flags().StringVar(&info.UpHost, "up-host", "", "upload host") + cmd.Flags().BoolVarP(&info.Accelerate, "accelerate", "", false, "enable uploading acceleration") cmd.Flags().StringVar(&info.RecordRoot, "record-root", "", "record root dir, and will save record info to the dir(db and log), default /.qshell") cmd.Flags().StringVar(&LogFile, "log-file", "", "log file") cmd.Flags().StringVar(&LogLevel, "log-level", "debug", "log level") @@ -146,6 +147,7 @@ var syncCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { cmd.Flags().BoolVarP(&info.UseResumeV2, "resumable-api-v2", "", false, "use resumable upload v2 APIs to upload") cmd.Flags().Int64VarP(&info.ChunkSize, "resumable-api-v2-part-size", "", data.BLOCK_SIZE, "the part size when use resumable upload v2 APIs to upload, default 4M") cmd.Flags().StringVarP(&info.UpHost, "up-host", "u", "", "upload host") + cmd.Flags().BoolVarP(&info.Accelerate, "accelerate", "", false, "enable uploading acceleration") cmd.Flags().IntVarP(&info.FileType, "file-type", "", 0, "set storage type of file, 0:STANDARD storage, 1:IA storage, 2:ARCHIVE storage, 3:DEEP_ARCHIVE storage, 4:ARCHIVE_IR storage") cmd.Flags().IntVarP(&info.FileType, "storage", "s", 0, "set storage type of file, same to --file-type") @@ -203,6 +205,7 @@ var formUploadCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { _ = cmd.Flags().MarkDeprecated("storage", "use --file-type instead") // 废弃 storage cmd.Flags().StringVarP(&info.UpHost, "up-host", "u", "", "uphost") + cmd.Flags().BoolVarP(&info.Accelerate, "accelerate", "", false, "enable uploading acceleration") cmd.Flags().StringVarP(&info.Policy.EndUser, "end-user", "", "", "Owner identification") cmd.Flags().StringVarP(&info.Policy.CallbackURL, "callback-urls", "l", "", "upload callback urls, separated by comma") @@ -262,6 +265,7 @@ var resumeUploadCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { cmd.Flags().IntVarP(&info.ResumeWorkerCount, "worker", "c", 3, "worker count") cmd.Flags().StringVarP(&info.UpHost, "up-host", "u", "", "uphost") + cmd.Flags().BoolVarP(&info.Accelerate, "accelerate", "", false, "enable uploading acceleration") cmd.Flags().StringVarP(&info.Policy.EndUser, "end-user", "", "", "Owner identification") cmd.Flags().StringVarP(&info.Policy.CallbackURL, "callback-urls", "l", "", "upload callback urls, separated by comma") diff --git a/docs/create-share.go b/docs/create-share.go new file mode 100644 index 00000000..904139af --- /dev/null +++ b/docs/create-share.go @@ -0,0 +1,12 @@ +package docs + +import _ "embed" + +//go:embed create-share.md +var createShareDocument string + +const CreateShareType = "create-share" + +func init() { + addCmdDocumentInfo(CreateShareType, createShareDocument) +} diff --git a/docs/create-share.md b/docs/create-share.md new file mode 100644 index 00000000..71aeaa11 --- /dev/null +++ b/docs/create-share.md @@ -0,0 +1,42 @@ +# 简介 +`create-share` 命令为需要分享的目录创建授权链接。 + +# 格式 +``` +qshell create-share [kodo://]/ +``` + +# 帮助文档 +可以在命令行输入如下命令获取帮助文档: +``` +// 简单描述 +$ qshell create-share -h + +// 详细文档(此文档) +$ qshell create-share --doc +``` + +# 鉴权 +需要使用 `qshell account` 或者 `qshell user add` 命令设置鉴权信息 `AccessKey`, `SecretKey` 和 `Name`。 + +# 参数 +- Bucket: 空间名称【必选】 +- Prefix: 前缀【必选】 + +# 选项 +- --extract-code: 提取码,只能包含六位大小写字母或者数字,如果不填写,将会自动生成。【可选】 +- --validity-period: 有效时间,如果不填写,默认为 15 分钟。【可选】 +- --output: 保存路径,以 JSON 格式保存输出内容,如果不填写,则直接以文本形式输出。【可选】 + +# 示例 +``` +$ qshell create-share kodo://bucketname/prefix +Link: +http://portal.qiniu.com/kodo-shares/verify?id=AGQEKDRxBBjbGmsKduQS9oFx59rz&token=qhtbC5YmDCO-WiPriuoCG_t4hZ1LboSOtRYSJXo_%3A9uJY8FiNrKjNrt4MpBx547jlgwr8aes15z5i8VY6l5SU6ga2IKWDBSGTv1jo-rOocklE7QqApzG6okJktZ36umLoqv9x1kuo5fNmgasLXowyTuHIM3kXsaV_DoXmvQsGr5ol6j4RtrmLcKdtXhpkGH8MfSjEgRV91Bx_Q_mSwpJ1028p8yZCSad_QOu_kSPxzeLZmWlUpAtO2oEXdbMTBxhTCH_3awCgqkgoogi0FQGP4zHxeFr0n3vj69DpmWqe6DiYbYLivCuU0kOF5Khv4I6-w6vjjdY +Extract Code: +wp7gqc +Expire: +2024-10-09 10:44:41 +0000 + +$ qshell create-share --output=share.json kodo://bucketname/prefix +``` diff --git a/docs/fput.md b/docs/fput.md index b43507c4..a19a47a3 100644 --- a/docs/fput.md +++ b/docs/fput.md @@ -14,7 +14,7 @@ qshell fput [--overwrite] [--callback-urls ] [--callback-host ] [--limit ] [--prefix | 可以在命令行输入如下命令获取帮助文档: ``` // 简单描述 -$ qshell listbucket2 -h +$ qshell listbucket2 -h // 详细文档(此文档) $ qshell listbucket2 --doc @@ -31,10 +31,10 @@ $ qshell listbucket2 --doc # 参数 - Bucket:空间名,可以为私有空间或者公开空间名称。【必选】 - + # 选项 - --prefix: 七牛空间中文件名的前缀,该参数为可选参数,如果不指定则获取空间中所有的文件列表。 【可选】 -- --limit:最多列条目的数量;当设置了此选项,输出的 marker 会有误差,不能使用; 默认输出所有条目。【可选】 +- --limit:最多列条目的数量;默认输出所有条目。【可选】 - -o/--outfile:获取的文件列表保存在本地的文件名,如果不指定该参数,则会把结果输出到终端,一般可用于获取小规模文件列表测试使用。 【可选】 - --output-file-max-lines:每个输出文件的最大行数,大于此行数会自动创建新的文件(新文件的文件名规律示例,源文件:/x/x/a.txt,新建文件为:/x/x/a-${index}.txt,index 为创建文件的序列号,从 0 开始),0:不限制单个输出文件的行数,默认:0。 【可选】 - --output-file-max-size:每个输出文件的最大尺寸,大于此值会自动创建新的文件(新文件的文件名规律示例,源文件:/x/x/a.txt,新建文件为:/x/x/a-${index}.txt,index 为创建文件的序列号,从 0 开始),0:不限制单个输出文件的尺寸,单位:B,默认:0。 【可选】 @@ -60,7 +60,7 @@ $ qshell listbucket2 --doc ``` qshell listbucket2 -o ``` - + 2 如果本地文件 `ListBucketResultFile` 已经存在,有上一次列举的内容,如果希望把新的列表添加到该文件中,需要使用选项 -a 开启 -o 选项的 append 模式 ``` qshell listbucket2 -a -o @@ -68,7 +68,7 @@ qshell listbucket2 -o 3 获取空间所有文件,输出到屏幕上(标准输出) ``` - qshell listbucket2 + qshell listbucket2 ``` 4 获取空间中指定前缀的文件列表 @@ -80,7 +80,7 @@ qshell listbucket2 [--prefix ] -o ``` qshell listbucket2 [--prefix ] ``` - + 6 获取 `2018-10-30` 到 `2018-11-02` 上传的文件 ``` qshell listbucket2 --start 2018-10-30 --end 2018-11-03 @@ -91,7 +91,7 @@ qshell listbucket2 [--prefix ] -o ``` qshell listbucket2 --suffixes mp4,html ``` - + 8 通常列举的文件的大小都是以字节显示,如果想以人工可读的方式 B, KB, MB 等显示,可以使用 -r 或者 --readable 选项 ``` qshell listbucket2 -r diff --git a/docs/pfop.md b/docs/pfop.md index 41eb600b..7c789170 100644 --- a/docs/pfop.md +++ b/docs/pfop.md @@ -6,13 +6,14 @@ # 格式 ``` qshell pfop [--pipeline ] -``` +qshell pfop [--pipeline ] --workflow-template-id +``` # 帮助文档 可以在命令行输入如下命令获取帮助文档: ``` // 简单描述 -$ qshell pfop -h +$ qshell pfop -h // 详细文档(此文档) $ qshell pfop --doc @@ -21,13 +22,15 @@ $ qshell pfop --doc # 参数 - Bucket:空间名,可以为公开空间或者私有空间【必选】 - Key:空间中文件的名称【必选】 -- Fops:数据处理命令列表,以;分隔,可以指定多个数据处理命令。 +- Fops:数据处理命令列表,以;分隔,可以指定多个数据处理命令。如果没有指定 `--workflow-template-id` 选项,则必选。 如: `avthumb/mp4|saveas/cWJ1Y2tldDpxa2V5;avthumb/flv|saveas/cWJ1Y2tldDpxa2V5Mg==`,是将上传的视频文件同时转码成mp4格式和flv格式后另存。 - + # 选项 - -p/--pipeline:处理队列名称, 如果没有制定该选项,默认使用公有队列【可选】 - -u/--notify-url:处理结果通知接收 URL,七牛将会向你设置的 URL 发起 Content-Type: application/json 的 POST 请求。【可选】 - -y/--force:强制执行数据处理。当服务端发现 fops 指定的数据处理结果已经存在,那就认为已经处理成功,避免重复处理浪费资源。 增加此选项(--force),则可强制执行数据处理并覆盖原结果。【可选】 +- --workflow-template-id:工作流模版 ID【可选】 +- --type:任务类型【可选】 # 示例 1 把 qiniutest 空间下的文件 `test.avi` 转码成 `mp4` 文件,转码后的结果保存到 `qiniutest` 空间中 diff --git a/docs/qupload.md b/docs/qupload.md index 6990ca10..f2b08cc2 100644 --- a/docs/qupload.md +++ b/docs/qupload.md @@ -11,7 +11,7 @@ qshell qupload [-c ] [--success-list ] [--failure- 可以在命令行输入如下命令获取帮助文档: ``` // 简单描述 -$ qshell qupload -h +$ qshell qupload -h // 详细文档(此文档) $ qshell qupload --doc @@ -25,6 +25,7 @@ $ qshell qupload --doc # 选项 - -c/--worker:配置下载的并发协程数量(ThreadCount),默认为 1,即文件一个一个上传,对于大量小文件来说,可以通过提高该参数值来提升同步速度。关于 `ThreadCount` 的值,并不是越大越好,所以工具里面限制了范围 `[1, 2000]`(如果不在范围内则重置为 5),在实际情况下最好根据所拥有的上传带宽和文件的平均大小来计算下这个并发数,最简单的算法就是带宽除以平均文件大小即可得到并发数。 假设上传带宽有 10Mbps,文件平均大小 500KB,那么利用 10*1024/8/500 = 2.56,那么并发数差不多就是 3~6 左右。 +- --accelerate:启用上传加速 - -s/--success-list:指定一个文件名字,导入上传成功的文件列表到该文件。 - -e/--failure-list:指定一个文件名字, 导入上传失败的文件列表到该文件。 - -w/--overwrite-list:指定一个文件名字, 导入存储空间中被覆盖的文件列表到该文件。 @@ -56,7 +57,8 @@ $ qshell qupload --doc "log_stdout" : true, "file_type" : 0, "resumable_api_v2" : false, - "resumable_api_v2_part_size" : 4194304 + "resumable_api_v2_part_size" : 4194304, + "uploading_acceleration" : true } ``` 参数说明: @@ -83,10 +85,11 @@ $ qshell qupload --doc - delete_on_success:上传成功的文件,同时删除本地文件,以达到节约磁盘的目的,比如日志归档的场景,默认为 `false`,如果需要开启功能,设置为 `true` 即可。【可选】 - resumable_api_v2:使用分片 V2 进行上传,默认为 `false` 使用分片 V1 。【可选】 - resumable_api_v2_part_size:使用分片 V2 进行上传时定制分片大小,默认 4194304(4M) 。【可选】 +- uploading_acceleration:启用上传加速。【可选】 - put_threshold:上传阈值,上传文件大小超过此值会使用分片上传,不超过使用表单上传;单位:B,默认为 8388608(8M) 。【可选】 - sequential_read_file: 文件读为顺序读,不涉及跳读;开启后,上传中的分片数据会被加载至内存。此选项可能会增加挂载网络文件系统的文件上传速度。默认是:false。 【可选】 - record_root:上传记录信息保存路径,包括日志文件和上传进度文件;默认为 `qshell` 上传目录;【可选】 - - 通过 `-L` 指定工作目录时,`record_root` 则为此工作目录/qdownload/$jobId, + - 通过 `-L` 指定工作目录时,`record_root` 则为此工作目录/qdownload/$jobId, - 未通过 `-L` 指定工作目录时为 `用户目录/.qshell/users/$CurrentUserName/qdownload/$jobId` - 注意 `jobId` 是根据上传任务动态生成;具体方式为 MD5("$SrcDir:$Bucket:$FileList"); `CurrentUserName` 当前用户的名称 - worker_count:分片上传中单个文件并发上传的分片数;默认为 3。【可选】 @@ -109,7 +112,7 @@ $ qshell qupload --doc 2) 检查 Key 扩展名; 3) 侦测内容。 3. 设为 -1 值,无论上传端指定了何值直接使用该值。 -``` +``` - traffic_limit:上传请求单链接速度限制,控制客户端带宽占用。限速值取值范围为 819200 ~ 838860800,单位为 bit/s。【可选】 @@ -184,9 +187,9 @@ See upload log at path /Users/jemy/.qshell/qupload/290438bcd0bcc7121bb22a56b1c95 - 有,查看本地文件时间戳是否变化,检测服务文件修改时间是否变化, - 二者均未发生变化则认为文件未发生变化,不再上传 - 本地/服务文件任一文件时间戳发生变化,则触发上传 - + 上传 -- 是否配置检查文件是否存在(`check_exists`) +- 是否配置检查文件是否存在(`check_exists`) - 未配置,直接上传 - 配置,查看文件是否配置检测 Hash(`check_hash`)。 - 配置检测 Hash,则检查本地文件 Hash 是否和服务 Hash 一致 @@ -242,10 +245,10 @@ demo2.gif ### 默认的上传入口 很多情况下,该工具的使用者的网络和七牛的网络都不是在一个内网,或者是机房或者是普通的办公网络和家庭网络。这种情况下,为了保证上传的速度和效率,必须走加速上传通道。目前本工具中使用的空间所在机房和对应的上传加速域名如下: -- 华东: http://upload.qiniu.com -- 华北: http://upload-z1.qiniu.com -- 华南: http://upload-z2.qiniu.com -- 北美: http://upload-na0.qiniu.com +- 华东: http://upload.qiniu.com +- 华北: http://upload-z1.qiniu.com +- 华南: http://upload-z2.qiniu.com +- 北美: http://upload-na0.qiniu.com - 东南亚: http://upload-as0.qiniup.com 本工具会根据配置文件中指定的空间参数,自动获取所应该使用的默认加速域名,所以不需要担心是否需要额外设置上传域名。 @@ -255,10 +258,10 @@ demo2.gif - 华东 - 上传加速域名:http(s)://upload.qiniup.com - 源站上传域名:http(s)://up.qiniup.com -- 华北 +- 华北 - 上传加速域名:http(s)://upload-z1.qiniup.com - 源站上传域名:http(s)://up-z1.qiniup.com -- 华南 +- 华南 - 上传加速域名:http(s)://upload-z2.qiniup.com - 源站上传域名:http(s)://up-z2.qiniup.com - 北美 @@ -325,7 +328,7 @@ demo2.gif 命令格式: ``` -$ qshell qupload --success-list success.txt upload.conf +$ qshell qupload --success-list success.txt upload.conf ``` 特别提示:由于这些选项所指定的文件在每次运行命令时,如果文件已存在,则已有内容会被清空然后写入新的内容,所以注意每次命令运行指定不同的文件。 diff --git a/docs/qupload2.md b/docs/qupload2.md index 8a953914..48890084 100644 --- a/docs/qupload2.md +++ b/docs/qupload2.md @@ -10,7 +10,7 @@ qshell qupload2 --src-dir=/home/jemy/temp --bucket=test `qupload2` 的 `--check-hash` 选项含义可参考 `qupload` 的 `check_hash` 配置; ``` -jemy•~» qshell qupload2 -h +jemy•~» qshell qupload2 -h ``` ``` @@ -20,6 +20,7 @@ Usage: qshell qupload2 [flags] Flags: + --accelerate enable uploading acceleration --bucket string bucket --callback-body string upload callback body -T, --callback-host string upload callback host diff --git a/docs/rput.md b/docs/rput.md index edfbf28b..9d05b66e 100644 --- a/docs/rput.md +++ b/docs/rput.md @@ -16,7 +16,7 @@ qshell rput [--overwrite] [--v2] [--mimetype ] [--callback-urls --to= +``` + +# 帮助文档 +可以在命令行输入如下命令获取帮助文档: +``` +// 简单描述 +$ qshell share-cp -h + +// 详细文档(此文档) +$ qshell share-cp --doc +``` + +# 鉴权 +需要使用 `qshell account` 或者 `qshell user add` 命令设置鉴权信息 `AccessKey`, `SecretKey` 和 `Name`。 + +# 参数 +- Link: 分享目录链接,或是通过 `create-share` 命令的 `--output` 选项输出的文件路径【必选】 + +# 选项 +- --extract-code: 提取码,只能包含六位大小写字母或者数字,如果不填写,且 `Link` 不是通过 `create-share` 命令的 `--output` 选项输出的文件路径,将会用交互模式提示输入。【可选】 +- --from: 下载共享目录内的子目录或是子文件,如果不填写,将列举被分享的目录。【可选】 +- --to: 下载目标路径。【必填】 + +# 示例 +``` +$ qshell share-cp 'http://portal.qiniu.com/kodo-shares/verify?id=AGQEKDRxBBjbGmsKduQS9oFx59rz&token=qhtbC5YmDCO-WiPriuoCG_t4hZ1LboSOtRYSJXo_%3A9uJY8FiNrKjNrt4MpBx547jlgwr8aes15z5i8VY6l5SU6ga2IKWDBSGTv1jo-rOocklE7QqApzG6okJktZ36umLoqv9x1kuo5fNmgasLXowyTuHIM3kXsaV_DoXmvQsGr5ol6j4RtrmLcKdtXhpkGH8MfSjEgRV91Bx_Q_mSwpJ1028p8yZCSad_QOu_kSPxzeLZmWlUpAtO2oEXdbMTBxhTCH_3awCgqkgoogi0FQGP4zHxeFr0n3vj69DpmWqe6DiYbYLivCuU0kOF5Khv4I6-w6vjjdY' --to=shared/ +Input Extract Code: +wp7gqc + +$ qshell share-cp share.json --from=main.go --to=shared/ +``` diff --git a/docs/share-ls.go b/docs/share-ls.go new file mode 100644 index 00000000..a502f0a6 --- /dev/null +++ b/docs/share-ls.go @@ -0,0 +1,12 @@ +package docs + +import _ "embed" + +//go:embed share-ls.md +var shareLsDocument string + +const ShareLsType = "share-ls" + +func init() { + addCmdDocumentInfo(ShareLsType, shareLsDocument) +} diff --git a/docs/share-ls.md b/docs/share-ls.md new file mode 100644 index 00000000..d1f4d2d8 --- /dev/null +++ b/docs/share-ls.md @@ -0,0 +1,79 @@ +# 简介 +`share-ls` 命令用来列举分享的目录和文件,对于每个文件,将会输出它的大小,文件类型和修改时间。 + +# 格式 +``` +qshell share-ls +``` + +# 帮助文档 +可以在命令行输入如下命令获取帮助文档: +``` +// 简单描述 +$ qshell share-ls -h + +// 详细文档(此文档) +$ qshell share-ls --doc +``` + +# 鉴权 +需要使用 `qshell account` 或者 `qshell user add` 命令设置鉴权信息 `AccessKey`, `SecretKey` 和 `Name`。 + +# 参数 +- Link: 分享目录链接,或是通过 `create-share` 命令的 `--output` 选项输出的文件路径【必选】 + +# 选项 +- --extract-code: 提取码,只能包含六位大小写字母或者数字,如果不填写,且 `Link` 不是通过 `create-share` 命令的 `--output` 选项输出的文件路径,将会用交互模式提示输入。【可选】 +- --prefix: 列举前缀,如果不填写,将列举被分享目录下的所有文件。【可选】 +- --limit: 限制列举条目的数量,当设置此选项,会在结束时输出 `Marker` 以供下次执行该命令时作为 `--marker` 选项的值,默认输出所有条目。【可选】 +- --marker: 标记列举过程中的位置。【可选】 + +# 示例 +``` +$ qshell share-ls 'http://portal.qiniu.com/kodo-shares/verify?id=AGQEKDRxBBjbGmsKduQS9oFx59rz&token=qhtbC5YmDCO-WiPriuoCG_t4hZ1LboSOtRYSJXo_%3A9uJY8FiNrKjNrt4MpBx547jlgwr8aes15z5i8VY6l5SU6ga2IKWDBSGTv1jo-rOocklE7QqApzG6okJktZ36umLoqv9x1kuo5fNmgasLXowyTuHIM3kXsaV_DoXmvQsGr5ol6j4RtrmLcKdtXhpkGH8MfSjEgRV91Bx_Q_mSwpJ1028p8yZCSad_QOu_kSPxzeLZmWlUpAtO2oEXdbMTBxhTCH_3awCgqkgoogi0FQGP4zHxeFr0n3vj69DpmWqe6DiYbYLivCuU0kOF5Khv4I6-w6vjjdY' +Input Extract Code: +wp7gqc +go.mod 912 STANDARD 2024-09-04 02:21:13 +0000 UTC +go.sum 9092 STANDARD 2024-09-04 02:21:13 +0000 UTC +main 9093344 STANDARD 2024-09-04 02:21:14 +0000 UTC +pfop-example/ +pfop-example/main.go 1094 STANDARD 2024-09-04 02:21:13 +0000 UTC +upload-download/ +upload-download/main.go 4110 STANDARD 2024-09-04 02:21:13 +0000 UTC +upload-example/ +upload-example/main.go 1486 STANDARD 2024-09-04 02:21:13 +0000 UTC +upload-lots-of-files/ +upload-lots-of-files/main.go 2843 STANDARD 2024-09-04 02:21:13 +0000 UTC +Total size: 8.69MB +Folder number: 7 +File number: 4 + +$ qshell share-ls share.json +go.mod 912 STANDARD 2024-09-04 02:21:13 +0000 UTC +go.sum 9092 STANDARD 2024-09-04 02:21:13 +0000 UTC +main 9093344 STANDARD 2024-09-04 02:21:14 +0000 UTC +pfop-example/ +pfop-example/main.go 1094 STANDARD 2024-09-04 02:21:13 +0000 UTC +upload-download/ +upload-download/main.go 4110 STANDARD 2024-09-04 02:21:13 +0000 UTC +upload-example/ +upload-example/main.go 1486 STANDARD 2024-09-04 02:21:13 +0000 UTC +upload-lots-of-files/ +upload-lots-of-files/main.go 2843 STANDARD 2024-09-04 02:21:13 +0000 UTC +Total size: 8.69MB +Folder number: 7 +File number: 4 + +$ qshell share-ls share.json --limit=7 +go.mod 912 STANDARD 2024-09-04 02:21:13 +0000 UTC +go.sum 9092 STANDARD 2024-09-04 02:21:13 +0000 UTC +main 9093344 STANDARD 2024-09-04 02:21:14 +0000 UTC +pfop-example/ +pfop-example/main.go 1094 STANDARD 2024-09-04 02:21:13 +0000 UTC +upload-download/ +upload-download/main.go 4110 STANDARD 2024-09-04 02:21:13 +0000 UTC +Marker: eyJjIjowLCJrIjoicHJlZml4L3VwbG9hZC1leGFtcGxlLyJ9 +Total size: 8.68MB +Folder number: 5 +File number: 2 +``` diff --git a/docs/sync.md b/docs/sync.md index fcf82cb0..8603ba22 100644 --- a/docs/sync.md +++ b/docs/sync.md @@ -16,7 +16,7 @@ qshell sync [-k ] 可以在命令行输入如下命令获取帮助文档: ``` // 简单描述 -$ qshell sync -h +$ qshell sync -h // 详细文档(此文档) $ qshell sync --doc @@ -30,6 +30,7 @@ $ qshell sync --doc - Bucket:空间名,可以为公开空间或者私有空间。 【必选】 # 选项 +- --accelerate:启用上传加速。【可选】 - -k/--key:该资源保存在空间中的 key,不配置时使用资源 Url 中文件名作为存储的 key。 【可选】 - -u/--uphost:上传入口的 IP 地址,一般在大文件的情况下,可以指定上传入口的 IP 来减少 DNS 环节,提升同步速度。 【可选】 - --file-type:文件存储类型,默认为 `0` (标准存储),`1` 为低频存储,`2` 为归档存储,`3` 为深度归档存储,`4` 为归档直读存储【可选】 @@ -44,7 +45,7 @@ $ qshell sync --doc - --persistent-ops:资源上传成功后触发执行的预转持久化处理指令列表。fileType=2或3(上传归档存储或深度归档存储文件)时,不支持使用该参数。支持魔法变量和自定义变量。每个指令是一个 API 规格字符串,多个指令用;分隔。【可选】 - --persistent-notify-url:接收持久化处理结果通知的 URL。必须是公网上可以正常进行 POST 请求并能成功响应的有效 URL。该 URL 获取的内容和持久化处理状态查询的处理结果一致。发送 body 格式是 Content-Type 为 application/json 的 POST 请求,需要按照读取流的形式读取请求的 body 才能获取。【可选】 - --persistent-pipeline:转码队列名。资源上传成功后,触发转码时指定独立的队列进行转码。为空则表示使用公用队列,处理速度比较慢。建议使用专用队列。【可选】 -- --detect-mime:开启 MimeType 侦测功能,并按照下述规则进行侦测;如不能侦测出正确的值,会默认使用 application/octet-stream 。【可选】 +- --detect-mime:开启 MimeType 侦测功能,并按照下述规则进行侦测;如不能侦测出正确的值,会默认使用 application/octet-stream 。【可选】 ``` 1. 设为 1 值,则忽略上传端传递的文件 MimeType 信息,并按如下顺序侦测 MimeType 值: 1) 侦测内容; @@ -55,7 +56,7 @@ $ qshell sync --doc 2) 检查 Key 扩展名; 3) 侦测内容。 3. 设为 -1 值,无论上传端指定了何值直接使用该值。 -``` +``` - --traffic-limit:上传请求单链接速度限制,控制客户端带宽占用。限速值取值范围为 819200 ~ 838860800,单位为 bit/s。【可选】 diff --git a/go.mod b/go.mod index b2c57c56..b818379b 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ require ( github.com/astaxie/beego v1.12.3 github.com/aws/aws-sdk-go v1.37.31 github.com/mitchellh/go-homedir v1.1.0 - github.com/qiniu/go-sdk/v7 v7.18.1 + github.com/qiniu/go-sdk/v7 v7.24.0 github.com/schollz/progressbar/v3 v3.8.6 github.com/spf13/cast v1.3.1 github.com/spf13/cobra v1.1.3 @@ -16,14 +16,25 @@ require ( ) require ( + github.com/BurntSushi/toml v1.3.2 // indirect github.com/Masterminds/goutils v1.1.1 // indirect github.com/Masterminds/semver/v3 v3.1.1 // indirect + github.com/alex-ant/gomath v0.0.0-20160516115720-89013a210a82 // indirect + github.com/elastic/go-sysinfo v1.0.2 // indirect + github.com/elastic/go-windows v1.0.0 // indirect + github.com/gammazero/toposort v0.1.1 // indirect + github.com/gofrs/flock v0.8.1 // indirect github.com/google/uuid v1.1.1 // indirect github.com/huandu/xstrings v1.3.1 // indirect github.com/imdario/mergo v0.3.11 // indirect + github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 // indirect github.com/mitchellh/copystructure v1.0.0 // indirect github.com/mitchellh/reflectwalk v1.0.0 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/prometheus/procfs v0.1.3 // indirect github.com/shopspring/decimal v1.2.0 // indirect + howett.net/plist v0.0.0-20181124034731-591f970eefbb // indirect + modernc.org/fileutil v1.3.0 // indirect ) require ( diff --git a/go.sum b/go.sum index c04e2e75..bfde0543 100644 --- a/go.sum +++ b/go.sum @@ -12,6 +12,8 @@ cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2k cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= +github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Knetic/govaluate v3.0.0+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= @@ -25,6 +27,8 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alex-ant/gomath v0.0.0-20160516115720-89013a210a82 h1:7dONQ3WNZ1zy960TmkxJPuwoolZwL7xKtpcM04MBnt4= +github.com/alex-ant/gomath v0.0.0-20160516115720-89013a210a82/go.mod h1:nLnM0KdK1CmygvjpDUO6m1TjSsiQtL61juhNsvV/JVI= github.com/alicebob/gopher-json v0.0.0-20180125190556-5a6b3ba71ee6/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc= github.com/alicebob/miniredis v2.5.0+incompatible/go.mod h1:8HZjEj4yU0dwhYHky+DxYx+6BMjkBbe5ONFIF1MXffk= github.com/aliyun/aliyun-oss-go-sdk v2.1.6+incompatible h1:Ft+KeWIJxFP76LqgJbvtOA1qBIoC8vGkTV3QeCOeJC4= @@ -62,6 +66,7 @@ github.com/couchbase/goutils v0.0.0-20180530154633-e865a1461c8a/go.mod h1:BQwMFl github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cupcake/rdb v0.0.0-20161107195141-43ba34106c76/go.mod h1:vYwsqCOLxGiisLwp9rITslkFNpZD5rz43tf41QFkTWY= +github.com/dave/jennifer v1.6.1/go.mod h1:nXbxhEmQfOZhWml3D1cDK5M1FLnMSozpbFN/m3RmGZc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -69,11 +74,17 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZm github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/edsrzf/mmap-go v0.0.0-20170320065105-0bce6a688712/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/elastic/go-elasticsearch/v6 v6.8.5/go.mod h1:UwaDJsD3rWLM5rKNFzv9hgox93HoX8utj1kxD9aFUcI= +github.com/elastic/go-sysinfo v1.0.2 h1:Wq1bOgnSz7Obl7DbMjbn0tzx1bE5G8Cfy3MVFa6C1Cc= +github.com/elastic/go-sysinfo v1.0.2/go.mod h1:O/D5m1VpYLwGjCYzEt63g3Z1uO3jXfwyzzjiW90t8cY= +github.com/elastic/go-windows v1.0.0 h1:qLURgZFkkrYyTTkvYpsZIgf83AUsdIHfvlJaqaZ7aSY= +github.com/elastic/go-windows v1.0.0/go.mod h1:TsU0Nrp7/y3+VwE82FoZF8gC/XFg/Elz6CcloAxnPgU= github.com/elazarl/go-bindata-assetfs v1.0.0/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/gammazero/toposort v0.1.1 h1:OivGxsWxF3U3+U80VoLJ+f50HcPU1MIqE1JlKzoJ2Eg= +github.com/gammazero/toposort v0.1.1/go.mod h1:H2cozTnNpMw0hg2VHAYsAxmkHXBYroNangj2NTBQDvw= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/glendc/gopher-json v0.0.0-20170414221815-dc4743023d0c/go.mod h1:Gja1A+xZ9BoviGJNA2E9vFkPjjsl+CoJxSXiQM1UXtw= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= @@ -86,10 +97,12 @@ github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTM github.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa7l++s0DH+LDiLs= github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA= -github.com/go-playground/validator/v10 v10.8.0/go.mod h1:9JhgTzTaE31GZDpH/HSvHiRJrJ3iKAgqqH0Bl/Ocjdk= +github.com/go-playground/validator/v10 v10.7.0/go.mod h1:xm76BBt941f7yWdGnI2DVPFFg1UK3YY04qifoXU3lOk= github.com/go-redis/redis v6.14.2+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= +github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -116,6 +129,7 @@ github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= @@ -128,6 +142,8 @@ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= +github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= @@ -158,14 +174,18 @@ github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/huandu/xstrings v1.3.1 h1:4jgBlKK6tLKFvO8u5pmYjG91cqytmDCDvGh7ECVFfFs= github.com/huandu/xstrings v1.3.1/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= +github.com/iancoleman/strcase v0.3.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= github.com/imdario/mergo v0.3.11 h1:3tnifQM4i+fbajXKBHXWEH+KvNHqojZ778UH75j3bGA= github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 h1:rp+c0RAYOWj8l6qbCUTSiRLG/iKnW3K3/QfPPuSsBt4= +github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901/go.mod h1:Z86h9688Y0wesXCyonoVr47MasHilkuLMqGhRZ4Hpak= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -188,6 +208,7 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/ledisdb/ledisdb v0.0.0-20200510135210-d35789ec47e6/go.mod h1:n931TsDuKuq+uX4v1fulaMbA/7ZLLhjc85h7chZGBCQ= +github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= @@ -241,6 +262,7 @@ github.com/peterh/liner v1.0.1-0.20171122030339-3681c2a91233/go.mod h1:xIteQHvHu github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -258,14 +280,17 @@ github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y8 github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190425082905-87a4384529e0/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.1.3 h1:F0+tqvhOksq22sc6iCHF5WGlWjdwj92p0udFh1VFBS8= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/qiniu/dyn v1.3.0/go.mod h1:E8oERcm8TtwJiZvkQPbcAh0RL8jO1G0VXJMW3FAWdkk= -github.com/qiniu/go-sdk/v7 v7.18.1 h1:DHWqWwVy1m4zMNDS4OzEo2tKPCyJCt0RbsqEFk9dzZ4= -github.com/qiniu/go-sdk/v7 v7.18.1/go.mod h1:nqoYCNo53ZlGA521RvRethvxUDvXKt4gtYXOwye868w= +github.com/qiniu/go-sdk/v7 v7.24.0 h1:1Vq8Xb1GrwqpeLp1HAnKv6xi0d9GyyiAapJb5/X2na8= +github.com/qiniu/go-sdk/v7 v7.24.0/go.mod h1:uZE85Pi0ftIHT/UNLShosdzwsovqpdas0LwAGO7cPao= github.com/qiniu/x v1.10.5/go.mod h1:03Ni9tj+N2h2aKnAz+6N0Xfl8FwMEDRC2PAlxekASDs= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= @@ -351,10 +376,8 @@ golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200414173820-0848c9571904/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220131195533-30dcbda58838/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.5.0 h1:U/0M97KRkSFvyD/3FSmdP5W5swImpNgle/EHFhOsQPE= golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -396,7 +419,6 @@ golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.5.0 h1:GyT4nK/YDHSqa1c4753ouYCDajOYKTja9Xb/OHtgvSw= golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -422,6 +444,7 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190425145619-16072639606e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -438,12 +461,10 @@ golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.0 h1:Zr2JFtRQNX3BCZ8YtxRE9hNJYC8J6I1MVbMg6owUp18= golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.4.0 h1:O7UWfv5+A2qiuulQk30kVinPoMtoIPeVaKLEgLpVkvg= golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -452,7 +473,6 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.6.0 h1:3XmdazWV+ubf7QgHSTWeykHOci5oeekaGJBLkrkaw4k= golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -541,4 +561,10 @@ honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +howett.net/plist v0.0.0-20181124034731-591f970eefbb h1:jhnBjNi9UFpfpl8YZhA9CrOqpnJdvzuiHsl/dnxl11M= +howett.net/plist v0.0.0-20181124034731-591f970eefbb/go.mod h1:vMygbs4qMhSZSc4lCUl2OEE+rDiIIJAIdR4m7MiMcm0= +modernc.org/fileutil v1.0.0/go.mod h1:JHsWpkrk/CnVV1H/eGlFf85BEpfkrp56ro8nojIq9Q8= +modernc.org/fileutil v1.3.0 h1:gQ5SIzK3H9kdfai/5x41oQiKValumqNTDXMvKo62HvE= +modernc.org/fileutil v1.3.0/go.mod h1:XatxS8fZi3pS8/hKG2GH/ArUogfxjpEKs3Ku3aK4JyQ= +modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= diff --git a/iqshell/common/config/config.go b/iqshell/common/config/config.go index 8bb45aa1..572e4358 100644 --- a/iqshell/common/config/config.go +++ b/iqshell/common/config/config.go @@ -2,6 +2,7 @@ package config import ( "encoding/json" + "github.com/qiniu/go-sdk/v7/auth" "github.com/qiniu/go-sdk/v7/storage" "github.com/qiniu/qshell/v2/iqshell/common/data" @@ -27,6 +28,17 @@ func (c *Config) HasCredentials() bool { return c.Credentials != nil && len(c.Credentials.AccessKey) > 0 && c.Credentials.SecretKey != nil } +func (c *Config) GetPortalHost() string { + var portalHost string + if hosts := c.Hosts; hosts != nil { + portalHost = hosts.Portal + } + if portalHost == "" { + portalHost = "portal.qiniu.com" + } + return portalHost +} + func (c *Config) GetRegion() *storage.Region { if len(c.Hosts.Api) == 0 && len(c.Hosts.Rs) == 0 && len(c.Hosts.Rsf) == 0 && len(c.Hosts.Io) == 0 && len(c.Hosts.Up) == 0 { diff --git a/iqshell/common/config/hosts.go b/iqshell/common/config/hosts.go index d130dfa2..6ee05a11 100644 --- a/iqshell/common/config/hosts.go +++ b/iqshell/common/config/hosts.go @@ -3,12 +3,13 @@ package config import "github.com/qiniu/qshell/v2/iqshell/common/utils" type Hosts struct { - UC []string `json:"uc,omitempty"` - Api []string `json:"api,omitempty"` - Rs []string `json:"rs,omitempty"` - Rsf []string `json:"rsf,omitempty"` - Io []string `json:"io,omitempty"` - Up []string `json:"up,omitempty"` + UC []string `json:"uc,omitempty"` + Api []string `json:"api,omitempty"` + Rs []string `json:"rs,omitempty"` + Rsf []string `json:"rsf,omitempty"` + Io []string `json:"io,omitempty"` + Up []string `json:"up,omitempty"` + Portal string `json:"portal,omitempty"` } func (h *Hosts) GetOneUc() string { @@ -35,6 +36,10 @@ func (h *Hosts) GetOneUp() string { return getOneHostFromStringArray(h.Up) } +func (h *Hosts) GetOnePortal() string { + return h.Portal +} + func getOneHostFromStringArray(hosts []string) string { hosts = getRealHosts(hosts) if len(hosts) > 0 { @@ -84,4 +89,8 @@ func (h *Hosts) merge(from *Hosts) { if len(h.Up) == 0 { h.Up = getRealHosts(from.Up) } + + if len(h.Portal) == 0 { + h.Portal = from.Portal + } } diff --git a/iqshell/common/data/error.go b/iqshell/common/data/error.go index d12ed748..c7f0fd1b 100644 --- a/iqshell/common/data/error.go +++ b/iqshell/common/data/error.go @@ -27,6 +27,8 @@ func ConvertError(err error) *CodeError { rErr := NewEmptyError().AppendError(err) if errors.Is(err, context.Canceled) { rErr.Code = ErrorCodeCancel + } else if hErr, ok := err.(interface{ HttpCode() int }); ok { + rErr.Code = hErr.HttpCode() } return rErr } diff --git a/iqshell/common/workspace/workspace.go b/iqshell/common/workspace/workspace.go index eb26b582..15a31760 100644 --- a/iqshell/common/workspace/workspace.go +++ b/iqshell/common/workspace/workspace.go @@ -2,13 +2,16 @@ package workspace import ( "context" + "sync" + "github.com/qiniu/go-sdk/v7/auth/qbox" "github.com/qiniu/go-sdk/v7/storage" + "github.com/qiniu/go-sdk/v7/storagev2/http_client" + "github.com/qiniu/go-sdk/v7/storagev2/region" "github.com/qiniu/qshell/v2/iqshell/common/account" "github.com/qiniu/qshell/v2/iqshell/common/config" "github.com/qiniu/qshell/v2/iqshell/common/data" "github.com/qiniu/qshell/v2/iqshell/common/log" - "sync" ) const ( @@ -64,6 +67,20 @@ func GetStorageConfig() *storage.Config { } } +func GetHttpClientOptions() *http_client.Options { + var options http_client.Options + options.UseInsecureProtocol = !cfg.IsUseHttps() + if region := cfg.GetRegion(); region != nil { + options.Regions = region + } + ucHost := cfg.Hosts.GetOneUc() + if len(ucHost) > 0 { + log.DebugF("ucHost: %s", ucHost) + options.SetBucketHosts(region.Endpoints{Preferred: []string{ucHost}}) + } + return &options +} + func GetAccount() (account.Account, *data.CodeError) { if currentAccount == nil { return account.Account{}, data.NewEmptyError().AppendDesc("can't get current user") diff --git a/iqshell/storage/bucket/list.go b/iqshell/storage/bucket/list.go index cf25f894..20386492 100644 --- a/iqshell/storage/bucket/list.go +++ b/iqshell/storage/bucket/list.go @@ -157,6 +157,13 @@ func List(info ListApiInfo, for !complete && (info.MaxRetry < 0 || retryCount <= info.MaxRetry) { lErr = nil var hasMore = false + limit := info.V1Limit + if info.OutputLimit > 0 { + limit = info.OutputLimit - outputCount + if limit > info.V1Limit { + limit = info.V1Limit + } + } if !workspace.IsCmdInterrupt() { hasMore, lErr = list.ListBucket(workspace.GetContext(), list.ApiInfo{ @@ -166,7 +173,7 @@ func List(info ListApiInfo, Prefix: info.Prefix, Delimiter: info.Delimiter, Marker: info.Marker, - V1Limit: info.V1Limit, + V1Limit: limit, }, func(marker string, dir string, listItem list.Item) (stop bool) { if marker != info.Marker { info.Marker = marker @@ -215,6 +222,7 @@ func List(info ListApiInfo, errorHandler(info.Marker, lErr) if workspace.IsCmdInterrupt() || // 取消 + lErr.Code >= 300 && lErr.Code < 500 || // Bad Request strings.Contains(lErr.Error(), "no such bucket") || // 空间不存在,直接结束 strings.Contains(lErr.Error(), "incorrect zone") || // 空间不正确 strings.Contains(lErr.Error(), "query region error") || // 查询空间错误 diff --git a/iqshell/storage/bucket/storage_v2.go b/iqshell/storage/bucket/storage_v2.go new file mode 100644 index 00000000..0ef03c3d --- /dev/null +++ b/iqshell/storage/bucket/storage_v2.go @@ -0,0 +1,37 @@ +package bucket + +import ( + "github.com/qiniu/go-sdk/v7/auth/qbox" + "github.com/qiniu/go-sdk/v7/storagev2/apis" + "github.com/qiniu/go-sdk/v7/storagev2/http_client" + "github.com/qiniu/qshell/v2/iqshell/common/data" + "github.com/qiniu/qshell/v2/iqshell/common/workspace" +) + +func GetStorageV2() (storageClient *apis.Storage, err *data.CodeError) { + acc, gErr := workspace.GetAccount() + if gErr != nil { + err = data.NewEmptyError().AppendDescF("GetStorageV2: get current account error:%v", gErr) + return + } + + mac := qbox.NewMac(acc.AccessKey, acc.SecretKey) + options := workspace.GetHttpClientOptions() + options.Credentials = mac + storageClient = apis.NewStorage(options) + return +} + +func GetHttpClient() (httpClient *http_client.Client, err *data.CodeError) { + acc, gErr := workspace.GetAccount() + if gErr != nil { + err = data.NewEmptyError().AppendDescF("GetStorageV2: get current account error:%v", gErr) + return + } + + mac := qbox.NewMac(acc.AccessKey, acc.SecretKey) + options := workspace.GetHttpClientOptions() + options.Credentials = mac + httpClient = http_client.NewClient(options) + return +} diff --git a/iqshell/storage/object/fop.go b/iqshell/storage/object/fop.go index ce50046e..4f091bc6 100644 --- a/iqshell/storage/object/fop.go +++ b/iqshell/storage/object/fop.go @@ -1,6 +1,8 @@ package object import ( + "context" + "github.com/qiniu/go-sdk/v7/storage" "github.com/qiniu/qshell/v2/iqshell/common/account" "github.com/qiniu/qshell/v2/iqshell/common/alert" @@ -26,16 +28,18 @@ func PreFopStatus(info PreFopStatusApiInfo) (storage.PrefopRet, *data.CodeError) return ret, data.ConvertError(e) } -type PreFopApiInfo struct { - Bucket string `json:"bucket"` - Key string `json:"key"` - Fops string `json:"fops"` - Pipeline string `json:"pipeline"` - NotifyURL string `json:"notify_url"` - NotifyForce bool `json:"notify_force"` +type PfopApiInfo struct { + Bucket string + Key string + Fops string + Pipeline string + NotifyURL string + Force bool + Type int64 + WorkflowTemplateID string } -func PreFop(info PreFopApiInfo) (string, *data.CodeError) { +func Pfop(info PfopApiInfo) (string, *data.CodeError) { if len(info.Bucket) == 0 { return "", alert.CannotEmptyError("bucket", "") } @@ -44,7 +48,7 @@ func PreFop(info PreFopApiInfo) (string, *data.CodeError) { return "", alert.CannotEmptyError("key", "") } - if len(info.Fops) == 0 { + if len(info.Fops) == 0 && len(info.WorkflowTemplateID) == 0 { return "", alert.CannotEmptyError("fops", "") } @@ -52,8 +56,25 @@ func PreFop(info PreFopApiInfo) (string, *data.CodeError) { if err != nil { return "", err } - persistentId, e := opManager.Pfop(info.Bucket, info.Key, info.Fops, info.Pipeline, info.NotifyURL, info.NotifyForce) - return persistentId, data.ConvertError(e) + force := int64(0) + if info.Force { + force = 1 + } + pfopRequest := storage.PfopRequest{ + BucketName: info.Bucket, + ObjectName: info.Key, + Fops: info.Fops, + NotifyUrl: info.NotifyURL, + Force: force, + Type: info.Type, + Pipeline: info.Pipeline, + WorkflowTemplateID: info.WorkflowTemplateID, + } + pfopRet, e := opManager.PfopV2(context.Background(), &pfopRequest) + if e != nil { + return "", data.ConvertError(e) + } + return pfopRet.PersistentID, nil } func getOperationManager(bucket string) (*storage.OperationManager, *data.CodeError) { diff --git a/iqshell/storage/object/operations/fop.go b/iqshell/storage/object/operations/fop.go index 0ed44cc6..9a9d5aa0 100644 --- a/iqshell/storage/object/operations/fop.go +++ b/iqshell/storage/object/operations/fop.go @@ -33,16 +33,16 @@ func PreFopStatus(cfg *iqshell.Config, info PreFopStatusInfo) { }) if err != nil { data.SetCmdStatusError() - log.ErrorF("pre fog status error:%v", err) + log.ErrorF("prefop status error:%v", err) return } log.Alert(ret.String()) } -type PreFopInfo object.PreFopApiInfo +type PfopInfo object.PfopApiInfo -func (info *PreFopInfo) Check() *data.CodeError { +func (info *PfopInfo) Check() *data.CodeError { if len(info.Bucket) == 0 { return alert.CannotEmptyError("Bucket", "") } @@ -51,23 +51,23 @@ func (info *PreFopInfo) Check() *data.CodeError { return alert.CannotEmptyError("Key", "") } - if len(info.Fops) == 0 { + if len(info.Fops) == 0 && len(info.WorkflowTemplateID) == 0 { return alert.CannotEmptyError("Fops", "") } return nil } -func PreFop(cfg *iqshell.Config, info PreFopInfo) { +func Pfop(cfg *iqshell.Config, info PfopInfo) { if shouldContinue := iqshell.CheckAndLoad(cfg, iqshell.CheckAndLoadInfo{ Checker: &info, }); !shouldContinue { return } - persistentId, err := object.PreFop(object.PreFopApiInfo(info)) + persistentId, err := object.Pfop(object.PfopApiInfo(info)) if err != nil { data.SetCmdStatusError() - log.ErrorF("pre fog error:%v", err) + log.ErrorF("pfop error:%v", err) return } log.Alert(persistentId) diff --git a/iqshell/storage/object/operations/share.go b/iqshell/storage/object/operations/share.go new file mode 100644 index 00000000..1767610a --- /dev/null +++ b/iqshell/storage/object/operations/share.go @@ -0,0 +1,504 @@ +package operations + +import ( + "bufio" + "context" + "encoding/json" + "math/rand" + "net/url" + "os" + "path/filepath" + "strings" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/aws/aws-sdk-go/service/s3/s3manager" + "github.com/qiniu/go-sdk/v7/storagev2/apis" + "github.com/qiniu/go-sdk/v7/storagev2/apis/create_share" + "github.com/qiniu/go-sdk/v7/storagev2/apis/verify_share" + "github.com/qiniu/qshell/v2/iqshell" + "github.com/qiniu/qshell/v2/iqshell/common/alert" + "github.com/qiniu/qshell/v2/iqshell/common/data" + "github.com/qiniu/qshell/v2/iqshell/common/log" + "github.com/qiniu/qshell/v2/iqshell/common/utils" + "github.com/qiniu/qshell/v2/iqshell/storage/bucket" +) + +type CreateShareInfo struct { + apis.CreateShareRequest + + OutputPath string +} + +func (info *CreateShareInfo) Check() *data.CodeError { + if len(info.Bucket) == 0 { + return alert.CannotEmptyError("Bucket", "") + } + if len(info.Prefix) == 0 { + return alert.CannotEmptyError("Prefix", "") + } + return nil +} + +const randomExtractCodeChars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789" + +func CreateShare(cfg *iqshell.Config, info CreateShareInfo) { + if err := createShare(cfg, &info); err != nil { + data.SetCmdStatusError() + log.ErrorF("Create share Failed, [%s:%s], Error: %v", info.Bucket, info.Prefix, err) + } +} + +func createShare(cfg *iqshell.Config, info *CreateShareInfo) error { + if shouldContinue := iqshell.CheckAndLoad(cfg, iqshell.CheckAndLoadInfo{ + Checker: info, + }); !shouldContinue { + return nil + } + + storagev2Client, codeErr := bucket.GetStorageV2() + if codeErr != nil { + return codeErr + } + + if info.DurationSeconds == 0 { + info.DurationSeconds = 15 * 60 + } + if info.ExtractCode == "" { + bytes := []byte(randomExtractCodeChars) + extractCodeBytes := make([]byte, 0, 6) + for i := 0; i < 6; i++ { + n := rand.Intn(len(bytes)) + extractCodeBytes = append(extractCodeBytes, bytes[n]) + } + info.ExtractCode = string(extractCodeBytes) + } + + response, err := storagev2Client.CreateShare(context.Background(), &info.CreateShareRequest, nil) + if err != nil { + return err + } + + body, err := newCreateShareReponseBody(cfg, info, response) + if err != nil { + return err + } + + if info.OutputPath != "" { + createdFile, err := os.Create(info.OutputPath) + if err != nil { + return err + } + defer createdFile.Close() + if err := json.NewEncoder(createdFile).Encode(&body); err != nil { + return err + } + } else { + log.AlertF("Link:\n%s", body.Link) + log.AlertF("Extract Code:\n%s", body.ExtractCode) + log.AlertF("Expire:\n%s", body.WillExpireAt.Local().Format(time.DateTime+" -0700")) + } + return nil +} + +type createShareReponseBody struct { + Link string `json:"link"` + ExtractCode string `json:"extract_code"` + WillExpireAt time.Time `json:"will_expire_at"` +} + +func newCreateShareReponseBody(cfg *iqshell.Config, info *CreateShareInfo, response *create_share.Response) (*createShareReponseBody, error) { + expires, err := time.Parse(time.RFC3339, response.Expires) + if err != nil { + return nil, err + } + urlString := cfg.CmdCfg.GetPortalHost() + if !strings.Contains(urlString, "://") { + if cfg.CmdCfg.IsUseHttps() { + urlString = "https://" + urlString + } else { + urlString = "http://" + urlString + } + } + urlString += "/kodo-shares/verify" + parsedUrl, err := url.Parse(urlString) + if err != nil { + return nil, err + } + query := parsedUrl.Query() + query.Set("id", response.Id) + query.Set("token", response.Token) + parsedUrl.RawQuery = query.Encode() + return &createShareReponseBody{Link: parsedUrl.String(), ExtractCode: info.ExtractCode, WillExpireAt: expires}, nil +} + +type ListShareInfo struct { + LinkURL string + ExtractCode string + Prefix string + Limit int64 + Marker string +} + +func (info *ListShareInfo) Check() *data.CodeError { + if len(info.LinkURL) == 0 { + return alert.CannotEmptyError("Link", "") + } + if len(info.ExtractCode) == 0 { + return alert.CannotEmptyError("ExtractCode", "") + } + if info.Limit < 0 { + return alert.Error("Limit should not be negative", "") + } + return nil +} + +func ListShare(cfg *iqshell.Config, info ListShareInfo) { + if err := listShare(cfg, &info); err != nil { + data.SetCmdStatusError() + log.ErrorF("List share Failed, [%s], Error: %v", info.LinkURL, err) + } +} + +func listShare(cfg *iqshell.Config, info *ListShareInfo) error { + if !strings.HasPrefix(info.LinkURL, "http://") && !strings.HasPrefix(info.LinkURL, "https://") { + linkUrl, extractCode, err := readLinkURLFromPath(info.LinkURL) + if err != nil { + return err + } + info.LinkURL = linkUrl + if info.ExtractCode == "" { + info.ExtractCode = extractCode + } + } + if info.ExtractCode == "" { + info.ExtractCode = promptExtractCode() + } + + if shouldContinue := iqshell.CheckAndLoad(cfg, iqshell.CheckAndLoadInfo{ + Checker: info, + }); !shouldContinue { + return nil + } + + storagev2Client, codeErr := bucket.GetStorageV2() + if codeErr != nil { + return codeErr + } + + parsedUrl, err := url.Parse(info.LinkURL) + if err != nil { + return err + } + + response, err := storagev2Client.VerifyShare(context.Background(), &verify_share.Request{ + ShareId: parsedUrl.Query().Get("id"), + Token: parsedUrl.Query().Get("token"), + ExtractCode: info.ExtractCode, + }, nil) + if err != nil { + return err + } + + s3Svc, err := getS3Service(response, cfg) + if err != nil { + return err + } + prefix := info.Prefix + if prefix == "" { + prefix = response.Prefix + } + input := s3.ListObjectsV2Input{ + Bucket: aws.String(response.BucketId), + } + if prefix != "" { + input.Prefix = aws.String(prefix) + } + if info.Marker != "" { + input.ContinuationToken = aws.String(info.Marker) + } + + var ( + stats = listedStats{prefix: info.Prefix} + restListed = info.Limit + listOutput *s3.ListObjectsV2Output + ) + for restListed > 0 || info.Limit == 0 { + if restListed > 0 && restListed < 1000 { + input.MaxKeys = aws.Int64(restListed) + } + listOutput, err = s3Svc.ListObjectsV2(&input) + if err != nil { + return err + } + for _, s3Object := range listOutput.Contents { + if restListed > 0 { + restListed -= 1 + } + if strings.HasSuffix(aws.StringValue(s3Object.Key), "/") && aws.Int64Value(s3Object.Size) == 0 { + printListedS3Directory(s3Object) + stats.directoryNumbers += 1 + } else { + printListedS3Object(s3Object) + stats.objectNumbers += 1 + stats.totalSize += aws.Int64Value(s3Object.Size) + } + } + if listOutput.NextContinuationToken == nil { + break + } + input.ContinuationToken = listOutput.NextContinuationToken + } + if listOutput.NextContinuationToken != nil { + log.AlertF("Marker: %s", aws.StringValue(listOutput.NextContinuationToken)) + } + printListedStats(&stats) + return nil +} + +func printListedS3Object(object *s3.Object) { + log.AlertF("%s\t%d\t%s\t%s", aws.StringValue(object.Key), aws.Int64Value(object.Size), aws.StringValue(object.StorageClass), aws.TimeValue(object.LastModified)) +} + +func printListedS3Directory(object *s3.Object) { + log.AlertF("%s", aws.StringValue(object.Key)) +} + +type listedStats struct { + prefix string + totalSize int64 + directoryNumbers int64 + objectNumbers int64 +} + +func printListedStats(info *listedStats) { + if info.prefix == "" { + log.AlertF("Total size: %s", utils.FormatFileSize(info.totalSize)) + } else { + log.AlertF("Total size of prefix [%s]: %s", info.prefix, utils.FormatFileSize(info.totalSize)) + } + log.AlertF("Folder number: %d", info.directoryNumbers) + log.AlertF("File number: %d", info.objectNumbers) +} + +type CopyShareInfo struct { + FromPath string + ToPath string + LinkURL string + ExtractCode string + Recursive bool +} + +func (info *CopyShareInfo) Check() *data.CodeError { + if len(info.ToPath) == 0 { + return alert.CannotEmptyError("ToURL", "") + } + if len(info.LinkURL) == 0 { + return alert.CannotEmptyError("Link", "") + } + if len(info.ExtractCode) == 0 { + return alert.CannotEmptyError("ExtractCode", "") + } + return nil +} + +func CopyShare(cfg *iqshell.Config, info CopyShareInfo) { + if err := copyShare(cfg, &info); err != nil { + data.SetCmdStatusError() + log.ErrorF("Copy share Failed, [%s], Error: %v", info.LinkURL, err) + } +} + +func copyShare(cfg *iqshell.Config, info *CopyShareInfo) error { + if !strings.HasPrefix(info.LinkURL, "http://") && !strings.HasPrefix(info.LinkURL, "https://") { + linkUrl, extractCode, err := readLinkURLFromPath(info.LinkURL) + if err != nil { + return err + } + info.LinkURL = linkUrl + if info.ExtractCode == "" { + info.ExtractCode = extractCode + } + } + if info.ExtractCode == "" { + info.ExtractCode = promptExtractCode() + } + + if shouldContinue := iqshell.CheckAndLoad(cfg, iqshell.CheckAndLoadInfo{ + Checker: info, + }); !shouldContinue { + return nil + } + + storagev2Client, codeErr := bucket.GetStorageV2() + if codeErr != nil { + return codeErr + } + + parsedLinkUrl, err := url.Parse(info.LinkURL) + if err != nil { + return err + } + + response, err := storagev2Client.VerifyShare(context.Background(), &verify_share.Request{ + ShareId: parsedLinkUrl.Query().Get("id"), + Token: parsedLinkUrl.Query().Get("token"), + ExtractCode: info.ExtractCode, + }, nil) + if err != nil { + return err + } + + s3Svc, err := getS3Service(response, cfg) + if err != nil { + return err + } + + s3session, err := session.NewSession(&s3Svc.Config) + if err != nil { + return err + } + s3Downloader := s3manager.NewDownloader(s3session) + + fromPrefix := info.FromPath + if fromPrefix == "" { + fromPrefix = response.Prefix + } + + toPath := strings.TrimPrefix(info.ToPath, "file://") + if strings.Contains(toPath, "://") { + return err + } + + if info.Recursive && strings.HasSuffix(fromPrefix, "/") { + fromParentDictionaryPrefix := fromPrefix[:strings.LastIndex(strings.TrimSuffix(fromPrefix, "/"), "/")+1] + input := s3.ListObjectsV2Input{ + Bucket: aws.String(response.BucketId), + } + if fromPrefix != "" { + input.Prefix = aws.String(fromPrefix) + } + for { + listOutput, err := s3Svc.ListObjectsV2(&input) + if err != nil { + return err + } + for _, s3Object := range listOutput.Contents { + relativePath := strings.TrimPrefix(aws.StringValue(s3Object.Key), fromParentDictionaryPrefix) + if filepath.Separator != '/' { + relativePath = strings.Replace(relativePath, "/", string(filepath.Separator), -1) + } + downloadPath := toPath + if relativePath != "" { + downloadPath = filepath.Join(toPath, relativePath) + } + if strings.HasSuffix(aws.StringValue(s3Object.Key), "/") && aws.Int64Value(s3Object.Size) == 0 { + err = os.MkdirAll(downloadPath, 0700) + } else { + if err = os.MkdirAll(filepath.Dir(downloadPath), 0700); err != nil { + return err + } + err = s3DownloadObjectToPath(s3Downloader, response.BucketId, aws.StringValue(s3Object.Key), downloadPath) + } + if err != nil { + return err + } + } + if listOutput.NextContinuationToken == nil { + break + } + input.ContinuationToken = listOutput.NextContinuationToken + } + } else { + if err = s3StatObject(s3Svc, response.BucketId, fromPrefix); err != nil { + return err + } + downloadPath := toPath + onlyMkDir := strings.HasSuffix(fromPrefix, "/") + fromPrefix = strings.TrimSuffix(fromPrefix, "/") + downloadPath = filepath.Join(downloadPath, fromPrefix[(strings.LastIndex(fromPrefix, "/")+1):]) + if onlyMkDir { + if err = os.MkdirAll(downloadPath, 0700); err != nil { + return err + } + } else { + if err = os.MkdirAll(filepath.Dir(downloadPath), 0700); err != nil { + return err + } + if err = s3DownloadObjectToPath(s3Downloader, response.BucketId, fromPrefix, downloadPath); err != nil { + return err + } + } + } + return nil +} + +func readLinkURLFromPath(path string) (string, string, error) { + path = strings.TrimPrefix(path, "file://") + b, err := os.ReadFile(path) + if err != nil { + return "", "", err + } + var body createShareReponseBody + if err = json.Unmarshal(b, &body); err != nil { + return "", "", err + } + return body.Link, body.ExtractCode, nil +} + +func promptExtractCode() string { + log.AlertF("Input Extract Code:") + scanner := bufio.NewScanner(os.Stdin) + scanner.Scan() + return strings.TrimSpace(scanner.Text()) +} + +func s3StatObject(s3Service *s3.S3, fromBucketId, key string) error { + _, err := s3Service.HeadObject(&s3.HeadObjectInput{ + Bucket: aws.String(fromBucketId), + Key: aws.String(key), + }) + return err +} + +func s3DownloadObjectToPath(downloader *s3manager.Downloader, fromBucketId, key, downloadPath string) error { + file, err := os.OpenFile(downloadPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600) + if err != nil { + return err + } + defer file.Close() + + _, err = downloader.Download(file, &s3.GetObjectInput{ + Bucket: aws.String(fromBucketId), + Key: aws.String(key), + }) + return err +} + +func getAwsConfig(response *verify_share.Response, cfg *iqshell.Config) *aws.Config { + config := aws.NewConfig() + if !cfg.CmdCfg.IsUseHttps() { + config.WithDisableSSL(true) + } + config.WithEndpoint(response.Endpoint) + config.WithRegion(response.Region) + config.WithCredentials(credentials.NewStaticCredentials(response.FederatedAk, response.FederatedSk, response.SessionToken)) + if cfg.DebugEnable { + config.WithLogLevel(aws.LogDebug) + } else if cfg.DDebugEnable { + config.WithLogLevel(aws.LogDebugWithHTTPBody) + } + return config +} + +func getS3Service(response *verify_share.Response, cfg *iqshell.Config) (*s3.S3, error) { + s3session, err := session.NewSession(getAwsConfig(response, cfg)) + if err != nil { + return nil, err + } + s3service := s3.New(s3session) + return s3service, nil +} diff --git a/iqshell/storage/object/upload/operations/batch_data.go b/iqshell/storage/object/upload/operations/batch_data.go index 322ef5d5..67f170a5 100644 --- a/iqshell/storage/object/upload/operations/batch_data.go +++ b/iqshell/storage/object/upload/operations/batch_data.go @@ -41,7 +41,8 @@ type UploadConfig struct { DisableForm bool `json:"disable_form,omitempty"` WorkerCount int `json:"work_count,omitempty"` // 分片上传并发数 RecordRoot string `json:"record_root,omitempty"` - SequentialReadFile bool `json:"sequential_read_file"` // 文件顺序读 + SequentialReadFile bool `json:"sequential_read_file"` // 文件顺序读 + Accelerate bool `json:"uploading_acceleration"` // 开启上传加速 // 唯一属主标识。特殊场景下非常有用,例如根据 App-Client 标识给图片或视频打水印。 EndUser string `json:"end_user,omitempty"` diff --git a/iqshell/storage/object/upload/uploader.go b/iqshell/storage/object/upload/uploader.go index 0ab87e5b..b49e7139 100644 --- a/iqshell/storage/object/upload/uploader.go +++ b/iqshell/storage/object/upload/uploader.go @@ -30,6 +30,7 @@ type ApiInfo struct { CheckSize bool `json:"-"` // 是否检查文件大小,检查是会对比服务端文件大小 Overwrite bool `json:"-"` // 当遇到服务端文件已存在时,是否使用本地文件覆盖之服务端的文件 UpHost string `json:"up_host"` // 上传使用的域名 + Accelerate bool `json:"upload_acceleration"` // 启用上传加速 TokenProvider func() string `json:"-"` // token provider TryTimes int `json:"-"` // 失败时,最多重试次数【可选】 TryInterval time.Duration `json:"-"` // 重试间隔时间 【可选】 @@ -202,6 +203,7 @@ func uploadSource(info *ApiInfo) (*ApiResult, *data.CodeError) { }) }) storageCfg := workspace.GetStorageConfig() + storageCfg.AccelerateUploading = info.Accelerate var up Uploader if utils.IsNetworkSource(info.FilePath) { up = networkSourceUploader(info, storageCfg)