Skip to content

Add archive ir #120

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 30 commits into from
Dec 6, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
30 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion client/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -246,11 +246,18 @@ func parseError(e *ErrorInfo, r io.Reader) {
}

func ResponseError(resp *http.Response) error {

e := &ErrorInfo{
Reqid: resp.Header.Get("X-Reqid"),
Code: resp.StatusCode,
}

defer func() {
if resp.Body != nil {
io.Copy(ioutil.Discard, resp.Body)
resp.Body.Close()
}
}()

if resp.StatusCode > 299 {
if resp.ContentLength != 0 {
ct, ok := resp.Header["Content-Type"]
Expand Down
49 changes: 40 additions & 9 deletions storage/bucket.go
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ type FileInfo struct {
* 1 表示低频存储
* 2 表示归档存储
* 3 表示深度归档存储
* 4 表示归档存储直读
*/
Type int `json:"type"`

Expand Down Expand Up @@ -102,6 +103,16 @@ type FileInfo struct {
*/
TransitionToIA int64 `json:"transitionToIA"`

/**
* 文件生命周期中转为归档直读存储的日期,int64 类型,Unix 时间戳格式 ,具体日期计算参考 生命周期管理。
* 文件在设置转归档直读后才会返回该字段(通过生命周期规则设置文件转归档直读,仅对该功能发布后满足规则条件新上传文件返回该字段;
* 历史文件想要返回该字段需要在功能发布后可通过 修改文件生命周期 API 指定转归档直读时间;对于已经设置过转归档直读时间的历史文
* 件,到期都会正常执行,只是服务端没有该字段返回)
*
* 例如:值为1568736000的时间,表示文件会在2019/9/18当天转为归档直读存储类型。
*/
TransitionToArchiveIR int64 `json:"transitionToArchiveIR"`

/**
* 文件生命周期中转为归档存储的日期,int64 类型,Unix 时间戳格式 ,具体日期计算参考 生命周期管理。
* 文件在设置转归档后才会返回该字段(通过生命周期规则设置文件转归档,仅对该功能发布后满足规则条件新上传文件返回该字段;
Expand All @@ -128,12 +139,21 @@ type FileInfo struct {

func (f *FileInfo) String() string {
str := ""
str += fmt.Sprintf("Hash: %s\n", f.Hash)
str += fmt.Sprintf("Fsize: %d\n", f.Fsize)
str += fmt.Sprintf("PutTime: %d\n", f.PutTime)
str += fmt.Sprintf("MimeType: %s\n", f.MimeType)
str += fmt.Sprintf("Type: %d\n", f.Type)
str += fmt.Sprintf("Status: %d\n", f.Status)
str += fmt.Sprintf("Hash: %s\n", f.Hash)
str += fmt.Sprintf("Fsize: %d\n", f.Fsize)
str += fmt.Sprintf("PutTime: %d\n", f.PutTime)
str += fmt.Sprintf("MimeType: %s\n", f.MimeType)
str += fmt.Sprintf("Type: %d\n", f.Type)
str += fmt.Sprintf("RestoreStatus: %d\n", f.RestoreStatus)
str += fmt.Sprintf("Status: %d\n", f.Status)
str += fmt.Sprintf("Md5: %s\n", f.Md5)
str += fmt.Sprintf("EndUser: %s\n", f.EndUser)
str += fmt.Sprintf("Expiration: %d\n", f.Expiration)
str += fmt.Sprintf("TransitionToIA: %d\n", f.TransitionToIA)
str += fmt.Sprintf("TransitionToArchiveIR: %d\n", f.TransitionToArchiveIR)
str += fmt.Sprintf("TransitionToArchive: %d\n", f.TransitionToArchive)
str += fmt.Sprintf("TransitionToDeepArchive: %d\n", f.TransitionToDeepArchive)
str += fmt.Sprintf("MetaData: %s\n", f.MetaData)
return str
}

Expand Down Expand Up @@ -176,6 +196,7 @@ type BatchOpRet struct {
* 1 表示低频存储
* 2 表示归档存储
* 3 表示深度归档存储
* 4 表示归档直读存储
*/
Type int `json:"type"`

Expand Down Expand Up @@ -226,6 +247,16 @@ type BatchOpRet struct {
*/
TransitionToIA *int64 `json:"transitionToIA"`

/**
* 文件生命周期中转为归档直读存储的日期,int64 类型,Unix 时间戳格式 ,具体日期计算参考 生命周期管理。
* 文件在设置转归档直读后才会返回该字段(通过生命周期规则设置文件转归档直读,仅对该功能发布后满足规则条件新上传文件返回该字段;
* 历史文件想要返回该字段需要在功能发布后可通过 修改文件生命周期 API 指定转归档直读时间;对于已经设置过转归档直读时间的历史文
* 件,到期都会正常执行,只是服务端没有该字段返回)
*
* 例如:值为1568736000的时间,表示文件会在2019/9/18当天转为归档直读存储类型。
*/
TransitionToArchiveIR int64 `json:"transitionToArchiveIR"`

/**
* 文件生命周期中转为归档存储的日期,Unix 时间戳格式 ,具体日期计算参考 生命周期管理。
* 文件在设置转归档后才会返回该字段(通过生命周期规则设置文件转归档,仅对该功能发布后满足规则条件新上传文件返回该字段;
Expand Down Expand Up @@ -533,7 +564,7 @@ func (m *BucketManager) ChangeMimeAndMeta(bucket, key, newMime string, metas map
return
}

// ChangeType 用来更新文件的存储类型,0 表示普通存储,1 表示低频存储,2 表示归档存储,3 表示深度归档存储
// ChangeType 用来更新文件的存储类型,0 表示普通存储,1 表示低频存储,2 表示归档存储,3 表示深度归档存储,4 表示归档直读存储
func (m *BucketManager) ChangeType(bucket, key string, fileType int) (err error) {
reqHost, reqErr := m.RsReqHost(bucket)
if reqErr != nil {
Expand Down Expand Up @@ -889,8 +920,8 @@ func (m *BucketManager) ApiHost(bucket string) (apiHost string, err error) {

func (m *BucketManager) Zone(bucket string) (z *Zone, err error) {

if m.Cfg.Zone != nil {
z = m.Cfg.Zone
z = m.Cfg.GetRegion()
if z != nil {
return
}

Expand Down
4 changes: 4 additions & 0 deletions storage/bucket_get_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,10 @@ func TestGet(t *testing.T) {
t.Logf("Get test error, %s", err)
}

defer func() {
resp.Close()
}()

body, err = io.ReadAll(resp.Body)
if err != nil {
t.Logf("Get test read body error, %s", err)
Expand Down
4 changes: 3 additions & 1 deletion storage/bucket_list.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,10 @@ import (
"context"
"errors"
"fmt"
"github.com/qiniu/go-sdk/v7/auth"
"net/url"
"strconv"

"github.com/qiniu/go-sdk/v7/auth"
)

// ListItem 为文件列举的返回值
Expand Down Expand Up @@ -38,6 +39,7 @@ type ListItem struct {
* 1 表示低频存储
* 2 表示归档存储
* 3 表示深度归档存储
* 4 表示归档存储直读
*/
Type int `json:"type"`

Expand Down
18 changes: 11 additions & 7 deletions storage/bucket_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -216,10 +216,11 @@ func TestStat(t *testing.T) {
if e := bucketManager.AddBucketLifeCycleRule(testBucket, &BucketLifeCycleRule{
Name: ruleName,
Prefix: "",
ToLineAfterDays: 1,
ToArchiveAfterDays: 2,
ToDeepArchiveAfterDays: 3,
DeleteAfterDays: 4,
ToLineAfterDays: 10,
ToArchiveIRAfterDays: 20,
ToArchiveAfterDays: 30,
ToDeepArchiveAfterDays: 40,
DeleteAfterDays: 50,
}); e != nil {
t.Logf("Stat AddBucketLifeCycleRule() error, %s", e)
t.Fail()
Expand All @@ -235,7 +236,8 @@ func TestStat(t *testing.T) {
}
client.DebugMode = true
if info, e := bucketManager.Stat(testBucket, copyKey); e != nil ||
len(info.Hash) == 0 || info.Expiration == 0 {
len(info.Hash) == 0 || info.Expiration == 0 || info.TransitionToArchive == 0 ||
info.TransitionToIA == 0 || info.TransitionToArchiveIR == 0 || info.TransitionToDeepArchive == 0 {
t.Logf("3 Stat() error, %v", e)
t.Fail()
} else {
Expand Down Expand Up @@ -783,6 +785,7 @@ func TestBucketLifeCycleRule(t *testing.T) {
Prefix: "testPutFileKey",
DeleteAfterDays: 13,
ToLineAfterDays: 1,
ToArchiveIRAfterDays: 2,
ToArchiveAfterDays: 6,
ToDeepArchiveAfterDays: 10,
})
Expand All @@ -798,7 +801,7 @@ func TestBucketLifeCycleRule(t *testing.T) {
ruleExists := false
for _, r := range rules {
if r.Name == "golangIntegrationTest" && r.Prefix == "testPutFileKey" && r.DeleteAfterDays == 13 &&
r.ToLineAfterDays == 1 && r.ToArchiveAfterDays == 6 && r.ToDeepArchiveAfterDays == 10 {
r.ToLineAfterDays == 1 && r.ToArchiveIRAfterDays == 2 && r.ToArchiveAfterDays == 6 && r.ToDeepArchiveAfterDays == 10 {
ruleExists = true
break
}
Expand All @@ -812,6 +815,7 @@ func TestBucketLifeCycleRule(t *testing.T) {
Prefix: "testPutFileKey",
DeleteAfterDays: 22,
ToLineAfterDays: 11,
ToArchiveIRAfterDays: 12,
ToArchiveAfterDays: 16,
ToDeepArchiveAfterDays: 20,
})
Expand All @@ -827,7 +831,7 @@ func TestBucketLifeCycleRule(t *testing.T) {
ruleExists = false
for _, r := range rules {
if r.Name == "golangIntegrationTest" && r.Prefix == "testPutFileKey" && r.DeleteAfterDays == 22 &&
r.ToLineAfterDays == 11 && r.ToArchiveAfterDays == 16 && r.ToDeepArchiveAfterDays == 20 {
r.ToLineAfterDays == 11 && r.ToArchiveIRAfterDays == 12 && r.ToArchiveAfterDays == 16 && r.ToDeepArchiveAfterDays == 20 {
ruleExists = true
break
}
Expand Down
2 changes: 1 addition & 1 deletion storage/token.go
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,7 @@ type PutPolicy struct {
// !application/json;text/plain 表示禁止上传 json 文本和纯文本。注意最前面的感叹号!
MimeLimit string `json:"mimeLimit,omitempty"`

// 资源的存储类型,0表示标准存储,1 表示低频存储,2 表示归档存储,3 表示深度归档存储。
// 资源的存储类型,0表示标准存储,1 表示低频存储,2 表示归档存储,3 表示深度归档存储,4 表示归档直读存储
FileType int `json:"fileType,omitempty"`

CallbackFetchKey uint8 `json:"callbackFetchKey,omitempty"`
Expand Down
11 changes: 8 additions & 3 deletions storage/uc.go
Original file line number Diff line number Diff line change
Expand Up @@ -289,18 +289,21 @@ type BucketLifeCycleRule struct {

// 在多少天后转低频存储
// 0 - 表示不转低频
// < 0 表示上传的文件立即使用低频存储
// > 0 表示多少天后转低频存储
ToLineAfterDays int `json:"to_line_after_days"`

// 指定文件上传多少天后转归档直读存储。
// 0 表示不转归档直读存储,
// > 0 表示多少天后转归档直读存储
ToArchiveIRAfterDays int `json:"to_archive_ir_after_days"`

// 指定文件上传多少天后转归档存储。
// 0 表示不转归档存储,
// < 0 表示上传的文件立即变归档存储
// > 0 表示多少天后转归档存储
ToArchiveAfterDays int `json:"to_archive_after_days"`

// 指定文件上传多少天后转深度归档存储。
// < 0 表示上传的文件立即变深度归档存储
// 0 表示不转深度归档存储
// > 0 表示多少天后转深度归档存储
ToDeepArchiveAfterDays int `json:"to_deep_archive_after_days"`
}
Expand All @@ -316,6 +319,7 @@ func (m *BucketManager) AddBucketLifeCycleRule(bucketName string, lifeCycleRule
params["delete_after_days"] = []string{strconv.Itoa(lifeCycleRule.DeleteAfterDays)}
params["to_ia_after_days"] = []string{strconv.Itoa(lifeCycleRule.ToLineAfterDays)}
params["to_archive_after_days"] = []string{strconv.Itoa(lifeCycleRule.ToArchiveAfterDays)}
params["to_archive_ir_after_days"] = []string{strconv.Itoa(lifeCycleRule.ToArchiveIRAfterDays)}
params["to_deep_archive_after_days"] = []string{strconv.Itoa(lifeCycleRule.ToDeepArchiveAfterDays)}

reqURL := getUcHost(m.Cfg.UseHTTPS) + "/rules/add"
Expand Down Expand Up @@ -355,6 +359,7 @@ func (m *BucketManager) UpdateBucketLifeCycleRule(bucketName string, rule *Bucke
params["delete_after_days"] = []string{strconv.Itoa(rule.DeleteAfterDays)}
params["to_line_after_days"] = []string{strconv.Itoa(rule.ToLineAfterDays)}
params["to_archive_after_days"] = []string{strconv.Itoa(rule.ToArchiveAfterDays)}
params["to_archive_ir_after_days"] = []string{strconv.Itoa(rule.ToArchiveIRAfterDays)}
params["to_deep_archive_after_days"] = []string{strconv.Itoa(rule.ToDeepArchiveAfterDays)}

reqURL := getUcHost(m.Cfg.UseHTTPS) + "/rules/update"
Expand Down