From e22a1d39f1ba7b4478a9a5e2994f1c3e746a880d Mon Sep 17 00:00:00 2001 From: YangSen-qn Date: Wed, 12 Jul 2023 18:12:31 +0800 Subject: [PATCH 01/23] uc client: use client v1 --- internal/clientv2/client.go | 34 +++++++++++++++++++++++++++++++++- storage/region.go | 5 ++++- storage/uc.go | 1 + 3 files changed, 38 insertions(+), 2 deletions(-) diff --git a/internal/clientv2/client.go b/internal/clientv2/client.go index e8e693a0..5b716f53 100644 --- a/internal/clientv2/client.go +++ b/internal/clientv2/client.go @@ -19,7 +19,13 @@ type client struct { func NewClient(cli Client, interceptors ...Interceptor) Client { if cli == nil { - cli = http.DefaultClient + if clientV1.DefaultClient.Client != nil { + cli = NewClientWithClientV1(&clientV1.DefaultClient) + } else if http.DefaultClient != nil { + cli = http.DefaultClient + } else { + cli = &http.Client{} + } } var is interceptorList = interceptors @@ -96,3 +102,29 @@ func DoAndDecodeJsonResponse(c Client, options RequestParams, ret interface{}) ( return resp, nil } + +type clientV1Wrapper struct { + c *clientV1.Client +} + +func (c *clientV1Wrapper) Do(req *http.Request) (*http.Response, error) { + return c.c.Do(req.Context(), req) +} + +func NewClientWithClientV1(c *clientV1.Client) Client { + if c == nil { + c = &clientV1.DefaultClient + } + + if c.Client == nil { + if clientV1.DefaultClient.Client != nil { + c.Client = clientV1.DefaultClient.Client + } else { + c.Client = &http.Client{} + } + } + + return &clientV1Wrapper{ + c: c, + } +} diff --git a/storage/region.go b/storage/region.go index c84180a6..35eae1ef 100644 --- a/storage/region.go +++ b/storage/region.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "github.com/qiniu/go-sdk/v7/auth" + "github.com/qiniu/go-sdk/v7/client" "github.com/qiniu/go-sdk/v7/internal/clientv2" "github.com/qiniu/go-sdk/v7/internal/hostprovider" "strings" @@ -334,6 +335,8 @@ type ucClientConfig struct { // 主备域名冻结时间（默认：600s），当一个域名请求失败（单个域名会被重试 TryTimes 次），会被冻结一段时间，使用备用域名进行重试，在冻结时间内，域名不能被使用，当一个操作中所有域名竣备冻结操作不在进行重试，返回最后一次操作的错误。 HostFreezeDuration time.Duration + + Client *client.Client } func getUCClient(config ucClientConfig, mac *auth.Credentials) clientv2.Client { @@ -375,5 +378,5 @@ func getUCClient(config ucClientConfig, mac *auth.Credentials) clientv2.Client { })) } - return clientv2.NewClient(nil, is...) + return clientv2.NewClient(clientv2.NewClientWithClientV1(config.Client), is...) } diff --git a/storage/uc.go b/storage/uc.go index 1da4111a..ea44649a 100644 --- a/storage/uc.go +++ b/storage/uc.go @@ -743,5 +743,6 @@ func (m *BucketManager) getUCClient() clientv2.Client { IsUcQueryApi: false, RetryMax: m.options.RetryMax, HostFreezeDuration: m.options.HostFreezeDuration, + Client: m.Client, }, m.Mac) } From 8c2eede541fac28e9b067bb52d16dcbdcc24a76a Mon Sep 17 00:00:00 2001 From: YangSen-qn Date: Thu, 13 Jul 2023 10:14:18 +0800 Subject: [PATCH 02/23] only test --- internal/clientv2/interceptor_retry_simple.go | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/internal/clientv2/interceptor_retry_simple.go b/internal/clientv2/interceptor_retry_simple.go index 9ea7237a..ebcd13e0 100644 --- a/internal/clientv2/interceptor_retry_simple.go +++ b/internal/clientv2/interceptor_retry_simple.go @@ -1,6 +1,7 @@ package clientv2 import ( + "fmt" clientv1 "github.com/qiniu/go-sdk/v7/client" "io" "math/rand" @@ -8,7 +9,6 @@ import ( "net/http" "net/url" "os" - "strings" "syscall" "time" ) @@ -167,23 +167,25 @@ func isNetworkErrorWithOpError(err *net.OpError) bool { return false } - desc := err.Error() - if strings.Contains(desc, "connection reset by peer") { - return true - } - if strings.Contains(desc, "use of closed network connection") { - return true - } + //desc := err.Error() + //if strings.Contains(desc, "connection reset by peer") { + // return true + //} + //if strings.Contains(desc, "use of closed network connection") { + // return true + //} switch t := err.Err.(type) { case *net.DNSError: return true case *os.SyscallError: if errno, ok := t.Err.(syscall.Errno); ok { - return errno == syscall.ECONNABORTED || + hit := errno == syscall.ECONNABORTED || errno == syscall.ECONNRESET || errno == syscall.ECONNREFUSED || errno == syscall.ETIMEDOUT + fmt.Printf("error code:%d hit:%t", errno, hit) + return hit } } From f3d962e157164de8ce0c68d603029daddf0f3afe Mon Sep 17 00:00:00 2001 From: YangSen-qn Date: Thu, 13 Jul 2023 10:54:33 +0800 Subject: [PATCH 03/23] only test --- internal/clientv2/interceptor_retry_simple.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/clientv2/interceptor_retry_simple.go b/internal/clientv2/interceptor_retry_simple.go index ebcd13e0..181fdb5c 100644 --- a/internal/clientv2/interceptor_retry_simple.go +++ b/internal/clientv2/interceptor_retry_simple.go @@ -148,6 +148,7 @@ func IsErrorRetryable(err error) bool { return false } + fmt.Printf("IsErrorRetryable: error：%+v \n", err) switch t := err.(type) { case *net.OpError: return isNetworkErrorWithOpError(t) @@ -184,7 +185,7 @@ func isNetworkErrorWithOpError(err *net.OpError) bool { errno == syscall.ECONNRESET || errno == syscall.ECONNREFUSED || errno == syscall.ETIMEDOUT - fmt.Printf("error code:%d hit:%t", errno, hit) + fmt.Printf("OpError error code:%d hit:%t \n", errno, hit) return hit } } From 072504d1eeafa88f0201fa0df4ec2e7e565c5d5c Mon Sep 17 00:00:00 2001 From: YangSen-qn Date: Mon, 17 Jul 2023 15:51:41 +0800 Subject: [PATCH 04/23] clientv2: retry when happy io.EOF --- internal/clientv2/interceptor_retry_simple.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/internal/clientv2/interceptor_retry_simple.go b/internal/clientv2/interceptor_retry_simple.go index 181fdb5c..e0d1e2c8 100644 --- a/internal/clientv2/interceptor_retry_simple.go +++ b/internal/clientv2/interceptor_retry_simple.go @@ -159,6 +159,9 @@ func IsErrorRetryable(err error) bool { case *clientv1.ErrorInfo: return isStatusCodeRetryable(t.Code) default: + if err == io.EOF { + return true + } return false } } From fded3c90c8057d64dc2beb5353cf241ff115c18c Mon Sep 17 00:00:00 2001 From: YangSen-qn Date: Tue, 18 Jul 2023 16:56:13 +0800 Subject: [PATCH 05/23] only test --- internal/clientv2/interceptor_retry_simple.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/internal/clientv2/interceptor_retry_simple.go b/internal/clientv2/interceptor_retry_simple.go index e0d1e2c8..731fe5eb 100644 --- a/internal/clientv2/interceptor_retry_simple.go +++ b/internal/clientv2/interceptor_retry_simple.go @@ -162,6 +162,7 @@ func IsErrorRetryable(err error) bool { if err == io.EOF { return true } + fmt.Printf("IsErrorRetryable default error type:%+v \n", t) return false } } @@ -191,6 +192,8 @@ func isNetworkErrorWithOpError(err *net.OpError) bool { fmt.Printf("OpError error code:%d hit:%t \n", errno, hit) return hit } + default: + fmt.Printf("isNetworkErrorWithOpError default error type:%+v \n", t) } return false From 006a8fe33a24ed70845557a91de352f9916d9f86 Mon Sep 17 00:00:00 2001 From: YangSen-qn Date: Wed, 19 Jul 2023 14:13:16 +0800 Subject: [PATCH 06/23] change uc host retry order --- storage/region.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/storage/region.go b/storage/region.go index 35eae1ef..8e3b7beb 100644 --- a/storage/region.go +++ b/storage/region.go @@ -202,8 +202,8 @@ var regionMap = map[RegionID]Region{ const ( defaultApiHost = "api.qiniu.com" - defaultUcHost0 = "uc.qbox.me" - defaultUcHost1 = "kodo-config.qiniuapi.com" + defaultUcHost0 = "kodo-config.qiniuapi.com" + defaultUcHost1 = "uc.qbox.me" ) // UcHost 为查询空间相关域名的 API 服务地址 From 30b1fa0c0ea0f3bb9ca82cf6e389ec24d8a45bc4 Mon Sep 17 00:00:00 2001 From: YangSen-qn Date: Wed, 19 Jul 2023 16:01:39 +0800 Subject: [PATCH 07/23] only test --- internal/clientv2/interceptor_retry_simple.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/internal/clientv2/interceptor_retry_simple.go b/internal/clientv2/interceptor_retry_simple.go index 731fe5eb..cf826e8b 100644 --- a/internal/clientv2/interceptor_retry_simple.go +++ b/internal/clientv2/interceptor_retry_simple.go @@ -2,7 +2,6 @@ package clientv2 import ( "fmt" - clientv1 "github.com/qiniu/go-sdk/v7/client" "io" "math/rand" "net" @@ -11,6 +10,8 @@ import ( "os" "syscall" "time" + + clientv1 "github.com/qiniu/go-sdk/v7/client" ) type RetryConfig struct { @@ -192,6 +193,7 @@ func isNetworkErrorWithOpError(err *net.OpError) bool { fmt.Printf("OpError error code:%d hit:%t \n", errno, hit) return hit } + fmt.Printf("isNetworkErrorWithOpError SyscallError type:%v \n", t.Err) default: fmt.Printf("isNetworkErrorWithOpError default error type:%+v \n", t) } From 323417ffc8ec4cb8aab83af91d1acca4826e73a8 Mon Sep 17 00:00:00 2001 From: YangSen-qn Date: Fri, 21 Jul 2023 14:46:37 +0800 Subject: [PATCH 08/23] only test --- internal/clientv2/interceptor_retry_simple.go | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/internal/clientv2/interceptor_retry_simple.go b/internal/clientv2/interceptor_retry_simple.go index cf826e8b..62058aea 100644 --- a/internal/clientv2/interceptor_retry_simple.go +++ b/internal/clientv2/interceptor_retry_simple.go @@ -149,7 +149,6 @@ func IsErrorRetryable(err error) bool { return false } - fmt.Printf("IsErrorRetryable: error：%+v \n", err) switch t := err.(type) { case *net.OpError: return isNetworkErrorWithOpError(t) @@ -163,17 +162,17 @@ func IsErrorRetryable(err error) bool { if err == io.EOF { return true } - fmt.Printf("IsErrorRetryable default error type:%+v \n", t) return false } } func isNetworkErrorWithOpError(err *net.OpError) bool { - if err == nil { + if err == nil || err.Err == nil { return false } - //desc := err.Error() + fmt.Printf("isNetworkErrorWithOpError: %#v", err.Err) + //desc := err.Err.Error() //if strings.Contains(desc, "connection reset by peer") { // return true //} @@ -186,16 +185,11 @@ func isNetworkErrorWithOpError(err *net.OpError) bool { return true case *os.SyscallError: if errno, ok := t.Err.(syscall.Errno); ok { - hit := errno == syscall.ECONNABORTED || + return errno == syscall.ECONNABORTED || errno == syscall.ECONNRESET || errno == syscall.ECONNREFUSED || errno == syscall.ETIMEDOUT - fmt.Printf("OpError error code:%d hit:%t \n", errno, hit) - return hit } - fmt.Printf("isNetworkErrorWithOpError SyscallError type:%v \n", t.Err) - default: - fmt.Printf("isNetworkErrorWithOpError default error type:%+v \n", t) } return false From 499cb0601809bdf5058525c65a81ec6a9bb9a113 Mon Sep 17 00:00:00 2001 From: YangSen-qn Date: Fri, 21 Jul 2023 15:03:58 +0800 Subject: [PATCH 09/23] only test --- internal/clientv2/interceptor_retry_simple.go | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/internal/clientv2/interceptor_retry_simple.go b/internal/clientv2/interceptor_retry_simple.go index 62058aea..b02f619f 100644 --- a/internal/clientv2/interceptor_retry_simple.go +++ b/internal/clientv2/interceptor_retry_simple.go @@ -1,13 +1,13 @@ package clientv2 import ( - "fmt" "io" "math/rand" "net" "net/http" "net/url" "os" + "strings" "syscall" "time" @@ -171,15 +171,6 @@ func isNetworkErrorWithOpError(err *net.OpError) bool { return false } - fmt.Printf("isNetworkErrorWithOpError: %#v", err.Err) - //desc := err.Err.Error() - //if strings.Contains(desc, "connection reset by peer") { - // return true - //} - //if strings.Contains(desc, "use of closed network connection") { - // return true - //} - switch t := err.Err.(type) { case *net.DNSError: return true @@ -190,6 +181,13 @@ func isNetworkErrorWithOpError(err *net.OpError) bool { errno == syscall.ECONNREFUSED || errno == syscall.ETIMEDOUT } + case *net.OpError: + return isNetworkErrorWithOpError(t) + default: + desc := err.Err.Error() + if strings.Contains(desc, "use of closed network connection") { + return true + } } return false From 3910f54c36a506d2162dc6dabf44a7a4a4f4006e Mon Sep 17 00:00:00 2001 From: YangSen-qn Date: Wed, 23 Aug 2023 10:56:58 +0800 Subject: [PATCH 10/23] version to v7.17.1 --- CHANGELOG.md | 6 ++++++ README.md | 2 +- conf/conf.go | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8afd0e5a..4e644772 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,10 @@ # Changelog +## 7.17.1 +* 优化 + * 调整在获取 Bucket 所在区域服务域名时的主备域名顺序 +* 调整 + * 移除内置的亚太-首尔区域 + ## 7.17.0 * 优化 * 对象存储，UC 服务相关请求支持主备重试 diff --git a/README.md b/README.md index 9fe77c31..9fc2ed20 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ github.com/qiniu/go-sdk 在您的项目中的 `go.mod` 文件内添加这行代码 ``` -require github.com/qiniu/go-sdk/v7 v7.17.0 +require github.com/qiniu/go-sdk/v7 v7.17.1 ``` 并且在项目中使用 `"github.com/qiniu/go-sdk/v7"` 引用 Qiniu Go SDK。 diff --git a/conf/conf.go b/conf/conf.go index b8421d16..1da4c7d2 100644 --- a/conf/conf.go +++ b/conf/conf.go @@ -5,7 +5,7 @@ import ( "strings" ) -const Version = "7.17.0" +const Version = "7.17.1" const ( CONTENT_TYPE_JSON = "application/json" From b1c8269ef3ea2ae6cb7cb6910797ac50780ef681 Mon Sep 17 00:00:00 2001 From: YangSen-qn Date: Wed, 13 Sep 2023 15:50:04 +0800 Subject: [PATCH 11/23] uc api handle host free duration --- storage/region.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/region.go b/storage/region.go index 679bdae6..39d6da4f 100644 --- a/storage/region.go +++ b/storage/region.go @@ -364,7 +364,7 @@ func getUCClient(config ucClientConfig, mac *auth.Credentials) clientv2.Client { ShouldRetry: nil, }, ShouldFreezeHost: nil, - HostFreezeDuration: 0, + HostFreezeDuration: config.HostFreezeDuration, HostProvider: hostprovider.NewWithHosts(hosts), }), clientv2.NewSimpleRetryInterceptor(clientv2.RetryConfig{ From 02bcf46a3350be57003eb43de7dec1d381cd1239 Mon Sep 17 00:00:00 2001 From: YangSen-qn Date: Fri, 13 Oct 2023 15:22:48 +0800 Subject: [PATCH 12/23] version to 7.18.0 --- CHANGELOG.md | 8 ++++++++ README.md | 2 +- conf/conf.go | 2 +- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4e644772..43a67c10 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,12 @@ # Changelog +## 7.18.0 +* 新增 + * BucketManager 支持下载 + * BucketManager 支持获取和修改对象 meta data + * 上传和下载支持单链限速 +* 优化 + * 区域缓存信息 ID 和查询域名相关 + ## 7.17.1 * 优化 * 调整在获取 Bucket 所在区域服务域名时的主备域名顺序 diff --git a/README.md b/README.md index d0bc57d9..18128abf 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ github.com/qiniu/go-sdk 在您的项目中的 `go.mod` 文件内添加这行代码 ``` -require github.com/qiniu/go-sdk/v7 v7.17.1 +require github.com/qiniu/go-sdk/v7 v7.18.0 ``` 并且在项目中使用 `"github.com/qiniu/go-sdk/v7"` 引用 Qiniu Go SDK。 diff --git a/conf/conf.go b/conf/conf.go index 1da4c7d2..345eb610 100644 --- a/conf/conf.go +++ b/conf/conf.go @@ -5,7 +5,7 @@ import ( "strings" ) -const Version = "7.17.1" +const Version = "7.18.0" const ( CONTENT_TYPE_JSON = "application/json" From dea8712711141ddf952ea42ffefa2b72d5b5353b Mon Sep 17 00:00:00 2001 From: YangSen-qn Date: Fri, 13 Oct 2023 15:46:54 +0800 Subject: [PATCH 13/23] change ci --- .github/workflows/ci-test.yml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index 64a6a320..979146d3 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -34,9 +34,14 @@ jobs: # new package name don't work in non-module mode rm -rf $GITHUB_WORKSPACE/src/github.com/go-playground/validator/v10 && git clone -b v10.9.0 --depth 1 https://github.com/go-playground/validator.git $GITHUB_WORKSPACE/src/github.com/go-playground/validator/v10 rm -rf $GITHUB_WORKSPACE/src/github.com/universal-translator && git clone -b v0.18.0 --depth 1 https://github.com/go-playground/universal-translator.git $GITHUB_WORKSPACE/src/github.com/go-playground/universal-translator + + rm -rf $GITHUB_WORKSPACE/src/golang.org/x/crypto && git clone -b v0.10.0 --depth 1 https://go.googlesource.com/crypto $GITHUB_WORKSPACE/src/golang.org/x/crypto + # GOPATH=$GITHUB_WORKSPACE go get golang.org/x/crypto/sha3 + + rm -rf $GITHUB_WORKSPACE/src/golang.org/x/text && git clone -b v0.10.0 --depth 1 https://cs.opensource.google/go/x/text $GITHUB_WORKSPACE/src/golang.org/x/text + # GOPATH=$GITHUB_WORKSPACE go get golang.org/x/text/language + GOPATH=$GITHUB_WORKSPACE go get github.com/leodido/go-urn - GOPATH=$GITHUB_WORKSPACE go get golang.org/x/crypto/sha3 - GOPATH=$GITHUB_WORKSPACE go get golang.org/x/text/language GOPATH=$GITHUB_WORKSPACE go get github.com/go-playground/locales # github.com/stretchr/testify @@ -45,7 +50,7 @@ jobs: GOPATH=$GITHUB_WORKSPACE go get github.com/davecgh/go-spew/spew GOPATH=$GITHUB_WORKSPACE go get github.com/pmezard/go-difflib/difflib GOPATH=$GITHUB_WORKSPACE go get gopkg.in/yaml.v3 - + GOPATH=$GITHUB_WORKSPACE make unittest working-directory: src/github.com/qiniu/go-sdk go-mod-test: From 25f7734c6d795c20d061453a7df2671cfe53281c Mon Sep 17 00:00:00 2001 From: YangSen-qn Date: Fri, 13 Oct 2023 17:06:09 +0800 Subject: [PATCH 14/23] change ci --- .github/workflows/ci-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index 979146d3..4f095f35 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -38,7 +38,7 @@ jobs: rm -rf $GITHUB_WORKSPACE/src/golang.org/x/crypto && git clone -b v0.10.0 --depth 1 https://go.googlesource.com/crypto $GITHUB_WORKSPACE/src/golang.org/x/crypto # GOPATH=$GITHUB_WORKSPACE go get golang.org/x/crypto/sha3 - rm -rf $GITHUB_WORKSPACE/src/golang.org/x/text && git clone -b v0.10.0 --depth 1 https://cs.opensource.google/go/x/text $GITHUB_WORKSPACE/src/golang.org/x/text + rm -rf $GITHUB_WORKSPACE/src/golang.org/x/text && git clone -b v0.10.0 --depth 1 https://github.com/golang/text $GITHUB_WORKSPACE/src/golang.org/x/text # GOPATH=$GITHUB_WORKSPACE go get golang.org/x/text/language GOPATH=$GITHUB_WORKSPACE go get github.com/leodido/go-urn From c19efb9ebfd4222213e34f6a340955c3e321a82a Mon Sep 17 00:00:00 2001 From: YangSen-qn Date: Fri, 13 Oct 2023 17:47:34 +0800 Subject: [PATCH 15/23] change client --- internal/clientv2/interceptor_retry_hosts.go | 3 +++ internal/clientv2/interceptor_retry_simple.go | 2 ++ 2 files changed, 5 insertions(+) diff --git a/internal/clientv2/interceptor_retry_hosts.go b/internal/clientv2/interceptor_retry_hosts.go index 758bdf5d..fc28a8d9 100644 --- a/internal/clientv2/interceptor_retry_hosts.go +++ b/internal/clientv2/interceptor_retry_hosts.go @@ -1,6 +1,8 @@ package clientv2 import ( + "io" + "io/ioutil" "net/http" "net/url" "strings" @@ -111,6 +113,7 @@ func (interceptor *hostsRetryInterceptor) Intercept(req *http.Request, handler H req = reqBefore if resp != nil && resp.Body != nil { + io.Copy(ioutil.Discard, resp.Body) resp.Body.Close() } diff --git a/internal/clientv2/interceptor_retry_simple.go b/internal/clientv2/interceptor_retry_simple.go index 42e4a756..e19b839d 100644 --- a/internal/clientv2/interceptor_retry_simple.go +++ b/internal/clientv2/interceptor_retry_simple.go @@ -2,6 +2,7 @@ package clientv2 import ( "io" + "io/ioutil" "math/rand" "net" "net/http" @@ -84,6 +85,7 @@ func (interceptor *simpleRetryInterceptor) Intercept(req *http.Request, handler } if resp != nil && resp.Body != nil { + io.Copy(ioutil.Discard, resp.Body) resp.Body.Close() } From 1534c008a546708caf788370d2f5dc0d7962fa57 Mon Sep 17 00:00:00 2001 From: YangSen-qn Date: Tue, 17 Oct 2023 14:26:14 +0800 Subject: [PATCH 16/23] PutPolicy support set -1 & version to 7.18.1 --- CHANGELOG.md | 4 ++++ README.md | 2 +- conf/conf.go | 2 +- storage/token.go | 15 ++++++++++----- 4 files changed, 16 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 43a67c10..873c5377 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,8 @@ # Changelog +## 7.18.1 +* 调整 + * PutPolicy 的 DetectMime 支持配置为 -1 + ## 7.18.0 * 新增 * BucketManager 支持下载 diff --git a/README.md b/README.md index 18128abf..aeb18b31 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ github.com/qiniu/go-sdk 在您的项目中的 `go.mod` 文件内添加这行代码 ``` -require github.com/qiniu/go-sdk/v7 v7.18.0 +require github.com/qiniu/go-sdk/v7 v7.18.1 ``` 并且在项目中使用 `"github.com/qiniu/go-sdk/v7"` 引用 Qiniu Go SDK。 diff --git a/conf/conf.go b/conf/conf.go index 345eb610..2518a644 100644 --- a/conf/conf.go +++ b/conf/conf.go @@ -5,7 +5,7 @@ import ( "strings" ) -const Version = "7.18.0" +const Version = "7.18.1" const ( CONTENT_TYPE_JSON = "application/json" diff --git a/storage/token.go b/storage/token.go index cceaa749..2061827e 100644 --- a/storage/token.go +++ b/storage/token.go @@ -91,11 +91,16 @@ type PutPolicy struct { FsizeLimit int64 `json:"fsizeLimit,omitempty"` // 开启 MimeType 侦测功能，并按照下述规则进行侦测；如不能侦测出正确的值，会默认使用 application/octet-stream 。 - // 设为非 0 值，则忽略上传端传递的文件 MimeType 信息，并按如下顺序侦测 MimeType 值： - // 1. 侦测内容； 2. 检查文件扩展名； 3. 检查 Key 扩展名。 - // 默认设为 0 值，如上传端指定了 MimeType 则直接使用该值，否则按如下顺序侦测 MimeType 值： - // 1. 检查文件扩展名； 2. 检查 Key 扩展名； 3. 侦测内容。 - DetectMime uint8 `json:"detectMime,omitempty"` + // 默认设为 0 时：如上传端指定了 MimeType 则直接使用该值，否则按如下顺序侦测 MimeType 值： + // 1. 检查文件扩展名； + // 2. 检查 Key 扩展名； + // 3. 侦测内容。 + // 设为 1 时：则忽略上传端传递的文件 MimeType 信息，并按如下顺序侦测 MimeType 值： + // 1. 侦测内容； + // 2. 检查文件扩展名； + // 3. 检查 Key 扩展名。 + // 设为 -1 时：无论上传端指定了何值直接使用该值。 + DetectMime int `json:"detectMime,omitempty"` // 限定用户上传的文件类型。指定本字段值，七牛服务器会侦测文件内容以判断 MimeType，再用判断值跟指定值进行匹配，匹配成功则允许上传，匹配失败则返回 403 状态码。示例： // image/* 表示只允许上传图片类型 From 4bd47ad961ba9e9a3096a1ddd9db40d7b23828af Mon Sep 17 00:00:00 2001 From: YangSen-qn Date: Thu, 23 Nov 2023 15:55:40 +0800 Subject: [PATCH 17/23] change bucket manager get Zone --- storage/bucket.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/storage/bucket.go b/storage/bucket.go index 839b1b38..e2ac8a31 100644 --- a/storage/bucket.go +++ b/storage/bucket.go @@ -889,8 +889,8 @@ func (m *BucketManager) ApiHost(bucket string) (apiHost string, err error) { func (m *BucketManager) Zone(bucket string) (z *Zone, err error) { - if m.Cfg.Zone != nil { - z = m.Cfg.Zone + z = m.Cfg.GetRegion() + if z != nil { return } From 675a946a86277db55dda243a4c70c92d13ef4d9c Mon Sep 17 00:00:00 2001 From: YangSen-qn Date: Thu, 23 Nov 2023 17:30:35 +0800 Subject: [PATCH 18/23] add archive ir --- storage/bucket.go | 11 +++++++++++ storage/bucket_list.go | 4 +++- storage/bucket_test.go | 8 +++++--- storage/uc.go | 8 ++++++++ 4 files changed, 27 insertions(+), 4 deletions(-) diff --git a/storage/bucket.go b/storage/bucket.go index e2ac8a31..d9a3213f 100644 --- a/storage/bucket.go +++ b/storage/bucket.go @@ -47,6 +47,7 @@ type FileInfo struct { * 1 表示低频存储 * 2 表示归档存储 * 3 表示深度归档存储 + * 4 表示归档存储直读 */ Type int `json:"type"` @@ -102,6 +103,16 @@ type FileInfo struct { */ TransitionToIA int64 `json:"transitionToIA"` + /** + * 文件生命周期中转为归档直读存储的日期，int64 类型，Unix 时间戳格式 ，具体日期计算参考 生命周期管理。 + * 文件在设置转归档直读后才会返回该字段（通过生命周期规则设置文件转归档直读，仅对该功能发布后满足规则条件新上传文件返回该字段； + * 历史文件想要返回该字段需要在功能发布后可通过 修改文件生命周期 API 指定转归档直读时间；对于已经设置过转归档直读时间的历史文 + * 件，到期都会正常执行，只是服务端没有该字段返回) + * + * 例如：值为1568736000的时间，表示文件会在2019/9/18当天转为归档直读存储类型。 + */ + TransitionToArchiveIR int64 `json:"transitionToArchiveIR"` + /** * 文件生命周期中转为归档存储的日期，int64 类型，Unix 时间戳格式 ，具体日期计算参考 生命周期管理。 * 文件在设置转归档后才会返回该字段（通过生命周期规则设置文件转归档，仅对该功能发布后满足规则条件新上传文件返回该字段； diff --git a/storage/bucket_list.go b/storage/bucket_list.go index 6a50f580..59f9fda8 100644 --- a/storage/bucket_list.go +++ b/storage/bucket_list.go @@ -4,9 +4,10 @@ import ( "context" "errors" "fmt" - "github.com/qiniu/go-sdk/v7/auth" "net/url" "strconv" + + "github.com/qiniu/go-sdk/v7/auth" ) // ListItem 为文件列举的返回值 @@ -38,6 +39,7 @@ type ListItem struct { * 1 表示低频存储 * 2 表示归档存储 * 3 表示深度归档存储 + * 4 表示归档存储直读 */ Type int `json:"type"` diff --git a/storage/bucket_test.go b/storage/bucket_test.go index f969f8e8..83ab73c6 100644 --- a/storage/bucket_test.go +++ b/storage/bucket_test.go @@ -783,6 +783,7 @@ func TestBucketLifeCycleRule(t *testing.T) { Prefix: "testPutFileKey", DeleteAfterDays: 13, ToLineAfterDays: 1, + ToArchiveIRAfterDays: 2, ToArchiveAfterDays: 6, ToDeepArchiveAfterDays: 10, }) @@ -798,7 +799,7 @@ func TestBucketLifeCycleRule(t *testing.T) { ruleExists := false for _, r := range rules { if r.Name == "golangIntegrationTest" && r.Prefix == "testPutFileKey" && r.DeleteAfterDays == 13 && - r.ToLineAfterDays == 1 && r.ToArchiveAfterDays == 6 && r.ToDeepArchiveAfterDays == 10 { + r.ToLineAfterDays == 1 && r.ToArchiveIRAfterDays == 2 && r.ToArchiveAfterDays == 6 && r.ToDeepArchiveAfterDays == 10 { ruleExists = true break } @@ -812,6 +813,7 @@ func TestBucketLifeCycleRule(t *testing.T) { Prefix: "testPutFileKey", DeleteAfterDays: 22, ToLineAfterDays: 11, + ToArchiveIRAfterDays: 12, ToArchiveAfterDays: 16, ToDeepArchiveAfterDays: 20, }) @@ -827,7 +829,7 @@ func TestBucketLifeCycleRule(t *testing.T) { ruleExists = false for _, r := range rules { if r.Name == "golangIntegrationTest" && r.Prefix == "testPutFileKey" && r.DeleteAfterDays == 22 && - r.ToLineAfterDays == 11 && r.ToArchiveAfterDays == 16 && r.ToDeepArchiveAfterDays == 20 { + r.ToLineAfterDays == 11 && r.ToArchiveIRAfterDays == 12 && r.ToArchiveAfterDays == 16 && r.ToDeepArchiveAfterDays == 20 { ruleExists = true break } @@ -979,7 +981,7 @@ func TestMakeURL(t *testing.T) { "": "", "abc_def.mp4": "abc_def.mp4", "/ab/cd": "/ab/cd", - "ab/中文/de": "ab/%E4%B8%AD%E6%96%87/de", + "ab/中文/de": "ab/%E4%B8%AD%E6%96%87/de", // "ab+-*de f": "ab%2B-%2Ade%20f", "ab:cd": "ab%3Acd", // "ab@cd": "ab%40cd", diff --git a/storage/uc.go b/storage/uc.go index 888d7b47..db15658e 100644 --- a/storage/uc.go +++ b/storage/uc.go @@ -293,6 +293,12 @@ type BucketLifeCycleRule struct { // > 0 表示多少天后转低频存储 ToLineAfterDays int `json:"to_line_after_days"` + // 指定文件上传多少天后转归档直读存储。 + // 0 表示不转归档直读存储， + // < 0 表示上传的文件立即变归档直读存储 + // > 0 表示多少天后转归档直读存储 + ToArchiveIRAfterDays int `json:"to_archive_ir_after_days"` + // 指定文件上传多少天后转归档存储。 // 0 表示不转归档存储， // < 0 表示上传的文件立即变归档存储 @@ -316,6 +322,7 @@ func (m *BucketManager) AddBucketLifeCycleRule(bucketName string, lifeCycleRule params["delete_after_days"] = []string{strconv.Itoa(lifeCycleRule.DeleteAfterDays)} params["to_ia_after_days"] = []string{strconv.Itoa(lifeCycleRule.ToLineAfterDays)} params["to_archive_after_days"] = []string{strconv.Itoa(lifeCycleRule.ToArchiveAfterDays)} + params["to_archive_ir_after_days"] = []string{strconv.Itoa(lifeCycleRule.ToArchiveIRAfterDays)} params["to_deep_archive_after_days"] = []string{strconv.Itoa(lifeCycleRule.ToDeepArchiveAfterDays)} reqURL := getUcHost(m.Cfg.UseHTTPS) + "/rules/add" @@ -355,6 +362,7 @@ func (m *BucketManager) UpdateBucketLifeCycleRule(bucketName string, rule *Bucke params["delete_after_days"] = []string{strconv.Itoa(rule.DeleteAfterDays)} params["to_line_after_days"] = []string{strconv.Itoa(rule.ToLineAfterDays)} params["to_archive_after_days"] = []string{strconv.Itoa(rule.ToArchiveAfterDays)} + params["to_archive_ir_after_days"] = []string{strconv.Itoa(rule.ToArchiveIRAfterDays)} params["to_deep_archive_after_days"] = []string{strconv.Itoa(rule.ToDeepArchiveAfterDays)} reqURL := getUcHost(m.Cfg.UseHTTPS) + "/rules/update" From 091556950dce01ac96f140c30c51f96a79220fe2 Mon Sep 17 00:00:00 2001 From: YangSen-qn Date: Fri, 24 Nov 2023 11:54:09 +0800 Subject: [PATCH 19/23] change some doc --- storage/bucket.go | 13 ++++++++++++- storage/token.go | 2 +- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/storage/bucket.go b/storage/bucket.go index d9a3213f..921f71e8 100644 --- a/storage/bucket.go +++ b/storage/bucket.go @@ -187,6 +187,7 @@ type BatchOpRet struct { * 1 表示低频存储 * 2 表示归档存储 * 3 表示深度归档存储 + * 4 表示归档直读存储 */ Type int `json:"type"` @@ -237,6 +238,16 @@ type BatchOpRet struct { */ TransitionToIA *int64 `json:"transitionToIA"` + /** + * 文件生命周期中转为归档直读存储的日期，int64 类型，Unix 时间戳格式 ，具体日期计算参考 生命周期管理。 + * 文件在设置转归档直读后才会返回该字段（通过生命周期规则设置文件转归档直读，仅对该功能发布后满足规则条件新上传文件返回该字段； + * 历史文件想要返回该字段需要在功能发布后可通过 修改文件生命周期 API 指定转归档直读时间；对于已经设置过转归档直读时间的历史文 + * 件，到期都会正常执行，只是服务端没有该字段返回) + * + * 例如：值为1568736000的时间，表示文件会在2019/9/18当天转为归档直读存储类型。 + */ + TransitionToArchiveIR int64 `json:"transitionToArchiveIR"` + /** * 文件生命周期中转为归档存储的日期，Unix 时间戳格式 ，具体日期计算参考 生命周期管理。 * 文件在设置转归档后才会返回该字段（通过生命周期规则设置文件转归档，仅对该功能发布后满足规则条件新上传文件返回该字段； @@ -544,7 +555,7 @@ func (m *BucketManager) ChangeMimeAndMeta(bucket, key, newMime string, metas map return } -// ChangeType 用来更新文件的存储类型，0 表示普通存储，1 表示低频存储，2 表示归档存储，3 表示深度归档存储 +// ChangeType 用来更新文件的存储类型，0 表示普通存储，1 表示低频存储，2 表示归档存储，3 表示深度归档存储，4 表示归档直读存储 func (m *BucketManager) ChangeType(bucket, key string, fileType int) (err error) { reqHost, reqErr := m.RsReqHost(bucket) if reqErr != nil { diff --git a/storage/token.go b/storage/token.go index 2061827e..5f3f97a2 100644 --- a/storage/token.go +++ b/storage/token.go @@ -108,7 +108,7 @@ type PutPolicy struct { // !application/json;text/plain 表示禁止上传 json 文本和纯文本。注意最前面的感叹号！ MimeLimit string `json:"mimeLimit,omitempty"` - // 资源的存储类型，0表示标准存储，1 表示低频存储，2 表示归档存储，3 表示深度归档存储。 + // 资源的存储类型，0表示标准存储，1 表示低频存储，2 表示归档存储，3 表示深度归档存储，4 表示归档直读存储。 FileType int `json:"fileType,omitempty"` CallbackFetchKey uint8 `json:"callbackFetchKey,omitempty"` From a65d80b9f9c3f0a385a6065e4f4dde923fa9a9ef Mon Sep 17 00:00:00 2001 From: YangSen-qn Date: Wed, 29 Nov 2023 16:20:53 +0800 Subject: [PATCH 20/23] fmt --- storage/bucket_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/bucket_test.go b/storage/bucket_test.go index 83ab73c6..aa8fdd57 100644 --- a/storage/bucket_test.go +++ b/storage/bucket_test.go @@ -981,7 +981,7 @@ func TestMakeURL(t *testing.T) { "": "", "abc_def.mp4": "abc_def.mp4", "/ab/cd": "/ab/cd", - "ab/中文/de": "ab/%E4%B8%AD%E6%96%87/de", + "ab/中文/de": "ab/%E4%B8%AD%E6%96%87/de", // "ab+-*de f": "ab%2B-%2Ade%20f", "ab:cd": "ab%3Acd", // "ab@cd": "ab%40cd", From 8b61abf3b12ccff585d08fd54a073dc40a537862 Mon Sep 17 00:00:00 2001 From: YangSen-qn Date: Thu, 30 Nov 2023 14:52:14 +0800 Subject: [PATCH 21/23] change test case --- storage/bucket_test.go | 12 +++++++----- storage/uc.go | 5 +---- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/storage/bucket_test.go b/storage/bucket_test.go index aa8fdd57..89e49222 100644 --- a/storage/bucket_test.go +++ b/storage/bucket_test.go @@ -216,10 +216,11 @@ func TestStat(t *testing.T) { if e := bucketManager.AddBucketLifeCycleRule(testBucket, &BucketLifeCycleRule{ Name: ruleName, Prefix: "", - ToLineAfterDays: 1, - ToArchiveAfterDays: 2, - ToDeepArchiveAfterDays: 3, - DeleteAfterDays: 4, + ToLineAfterDays: 10, + ToArchiveIRAfterDays: 20, + ToArchiveAfterDays: 30, + ToDeepArchiveAfterDays: 40, + DeleteAfterDays: 50, }); e != nil { t.Logf("Stat AddBucketLifeCycleRule() error, %s", e) t.Fail() @@ -235,7 +236,8 @@ func TestStat(t *testing.T) { } client.DebugMode = true if info, e := bucketManager.Stat(testBucket, copyKey); e != nil || - len(info.Hash) == 0 || info.Expiration == 0 { + len(info.Hash) == 0 || info.Expiration == 0 || info.TransitionToArchive == 0 || + info.TransitionToIA == 0 || info.TransitionToArchiveIR == 0 || info.TransitionToDeepArchive == 0 { t.Logf("3 Stat() error, %v", e) t.Fail() } else { diff --git a/storage/uc.go b/storage/uc.go index db15658e..1f913dad 100644 --- a/storage/uc.go +++ b/storage/uc.go @@ -289,24 +289,21 @@ type BucketLifeCycleRule struct { // 在多少天后转低频存储 // 0 - 表示不转低频 - // < 0 表示上传的文件立即使用低频存储 // > 0 表示多少天后转低频存储 ToLineAfterDays int `json:"to_line_after_days"` // 指定文件上传多少天后转归档直读存储。 // 0 表示不转归档直读存储， - // < 0 表示上传的文件立即变归档直读存储 // > 0 表示多少天后转归档直读存储 ToArchiveIRAfterDays int `json:"to_archive_ir_after_days"` // 指定文件上传多少天后转归档存储。 // 0 表示不转归档存储， - // < 0 表示上传的文件立即变归档存储 // > 0 表示多少天后转归档存储 ToArchiveAfterDays int `json:"to_archive_after_days"` // 指定文件上传多少天后转深度归档存储。 - // < 0 表示上传的文件立即变深度归档存储 + // 0 表示不转深度归档存储 // > 0 表示多少天后转深度归档存储 ToDeepArchiveAfterDays int `json:"to_deep_archive_after_days"` } From 77d06b5567638d636d35101c9e0c99592451b865 Mon Sep 17 00:00:00 2001 From: YangSen-qn Date: Fri, 1 Dec 2023 10:50:36 +0800 Subject: [PATCH 22/23] FileInfo string add more info & close response body while error --- client/client.go | 9 ++++++++- storage/bucket.go | 21 +++++++++++++++------ storage/bucket_get_test.go | 4 ++++ 3 files changed, 27 insertions(+), 7 deletions(-) diff --git a/client/client.go b/client/client.go index 3f46ea3d..b1a03734 100644 --- a/client/client.go +++ b/client/client.go @@ -246,11 +246,18 @@ func parseError(e *ErrorInfo, r io.Reader) { } func ResponseError(resp *http.Response) error { - e := &ErrorInfo{ Reqid: resp.Header.Get("X-Reqid"), Code: resp.StatusCode, } + + defer func() { + if resp.Body != nil { + io.Copy(ioutil.Discard, resp.Body) + resp.Body.Close() + } + }() + if resp.StatusCode > 299 { if resp.ContentLength != 0 { ct, ok := resp.Header["Content-Type"] diff --git a/storage/bucket.go b/storage/bucket.go index 921f71e8..47fdf905 100644 --- a/storage/bucket.go +++ b/storage/bucket.go @@ -139,12 +139,21 @@ type FileInfo struct { func (f *FileInfo) String() string { str := "" - str += fmt.Sprintf("Hash: %s\n", f.Hash) - str += fmt.Sprintf("Fsize: %d\n", f.Fsize) - str += fmt.Sprintf("PutTime: %d\n", f.PutTime) - str += fmt.Sprintf("MimeType: %s\n", f.MimeType) - str += fmt.Sprintf("Type: %d\n", f.Type) - str += fmt.Sprintf("Status: %d\n", f.Status) + str += fmt.Sprintf("Hash: %s\n", f.Hash) + str += fmt.Sprintf("Fsize: %d\n", f.Fsize) + str += fmt.Sprintf("PutTime: %d\n", f.PutTime) + str += fmt.Sprintf("MimeType: %s\n", f.MimeType) + str += fmt.Sprintf("Type: %d\n", f.Type) + str += fmt.Sprintf("RestoreStatus: %d\n", f.RestoreStatus) + str += fmt.Sprintf("Status: %d\n", f.Status) + str += fmt.Sprintf("Md5: %s\n", f.Md5) + str += fmt.Sprintf("EndUser: %s\n", f.EndUser) + str += fmt.Sprintf("Expiration: %d\n", f.Expiration) + str += fmt.Sprintf("TransitionToIA: %d\n", f.TransitionToIA) + str += fmt.Sprintf("TransitionToArchiveIR: %d\n", f.TransitionToArchiveIR) + str += fmt.Sprintf("TransitionToArchive: %d\n", f.TransitionToArchive) + str += fmt.Sprintf("TransitionToDeepArchive: %d\n", f.TransitionToDeepArchive) + str += fmt.Sprintf("MetaData: %d\n", f.MetaData) return str } diff --git a/storage/bucket_get_test.go b/storage/bucket_get_test.go index ceae9b99..65575584 100644 --- a/storage/bucket_get_test.go +++ b/storage/bucket_get_test.go @@ -72,6 +72,10 @@ func TestGet(t *testing.T) { t.Logf("Get test error, %s", err) } + defer func() { + resp.Close() + }() + body, err = io.ReadAll(resp.Body) if err != nil { t.Logf("Get test read body error, %s", err) From cc8eb4b2c86119de4abc92a88aa6877b48979e0d Mon Sep 17 00:00:00 2001 From: YangSen-qn Date: Fri, 1 Dec 2023 14:06:45 +0800 Subject: [PATCH 23/23] change file info string --- storage/bucket.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/bucket.go b/storage/bucket.go index 47fdf905..f750e657 100644 --- a/storage/bucket.go +++ b/storage/bucket.go @@ -153,7 +153,7 @@ func (f *FileInfo) String() string { str += fmt.Sprintf("TransitionToArchiveIR: %d\n", f.TransitionToArchiveIR) str += fmt.Sprintf("TransitionToArchive: %d\n", f.TransitionToArchive) str += fmt.Sprintf("TransitionToDeepArchive: %d\n", f.TransitionToDeepArchive) - str += fmt.Sprintf("MetaData: %d\n", f.MetaData) + str += fmt.Sprintf("MetaData: %s\n", f.MetaData) return str }