From a8adeaec19e96fb6800eb8b5c88a07442fffed05 Mon Sep 17 00:00:00 2001 From: Shubhendu Ram Tripathi Date: Fri, 2 Feb 2024 23:33:44 +0530 Subject: [PATCH 1/5] Enable --expired-object-all-versions Signed-off-by: Shubhendu Ram Tripathi --- cmd/ilm-rule-add.go | 4 ++++ cmd/ilm/options.go | 8 +++++++- cmd/ilm/parse.go | 8 ++++++-- go.mod | 6 ++++-- go.sum | 4 ++-- 5 files changed, 23 insertions(+), 7 deletions(-) diff --git a/cmd/ilm-rule-add.go b/cmd/ilm-rule-add.go index 6b5c37a198..9f939209ca 100644 --- a/cmd/ilm-rule-add.go +++ b/cmd/ilm-rule-add.go @@ -182,6 +182,10 @@ var ilmAddFlags = []cli.Flag{ Name: "noncurrent-transition-tier", Usage: "remote tier name to transition", }, + cli.BoolFlag{ + Name: "expired-object-all-versions", + Usage: "expire all object versions", + }, } type ilmAddMessage struct { diff --git a/cmd/ilm/options.go b/cmd/ilm/options.go index ccd46f5e2a..47d4218e99 100644 --- a/cmd/ilm/options.go +++ b/cmd/ilm/options.go @@ -79,6 +79,7 @@ type LifecycleOptions struct { NoncurrentVersionTransitionDays *int NewerNoncurrentTransitionVersions *int NoncurrentVersionTransitionStorageClass *string + ExpiredObjectAllversions *bool } // Filter returns lifecycle.Filter appropriate for opts @@ -149,7 +150,7 @@ func (opts LifecycleOptions) ToILMRule() (lifecycle.Rule, *probe.Error) { return "Enabled" }() - expiry, err := parseExpiry(opts.ExpiryDate, opts.ExpiryDays, opts.ExpiredObjectDeleteMarker) + expiry, err := parseExpiry(opts.ExpiryDate, opts.ExpiryDays, opts.ExpiredObjectDeleteMarker, opts.ExpiredObjectAllversions) if err != nil { return lifecycle.Rule{}, err } @@ -241,6 +242,7 @@ func GetLifecycleOptions(ctx *cli.Context) (LifecycleOptions, *probe.Error) { noncurrentVersionTransitionDays *int newerNoncurrentTransitionVersions *int noncurrentTier *string + expiredObjectAllversions *bool ) id = ctx.String("id") @@ -368,6 +370,9 @@ func GetLifecycleOptions(ctx *cli.Context) (LifecycleOptions, *probe.Error) { if f := "noncurrent-transition-newer"; ctx.IsSet(f) { newerNoncurrentTransitionVersions = intPtr(ctx.Int(f)) } + if ctx.IsSet("expired-object-all-versions") { + expiredObjectAllversions = boolPtr(ctx.Bool("expired-object-all-versions")) + } return LifecycleOptions{ ID: id, @@ -387,6 +392,7 @@ func GetLifecycleOptions(ctx *cli.Context) (LifecycleOptions, *probe.Error) { NoncurrentVersionTransitionDays: noncurrentVersionTransitionDays, NewerNoncurrentTransitionVersions: newerNoncurrentTransitionVersions, NoncurrentVersionTransitionStorageClass: noncurrentTier, + ExpiredObjectAllversions: expiredObjectAllversions, }, nil } diff --git a/cmd/ilm/parse.go b/cmd/ilm/parse.go index 89abb3c7f1..8a22adb1a1 100644 --- a/cmd/ilm/parse.go +++ b/cmd/ilm/parse.go @@ -104,7 +104,7 @@ func validateExpiration(rule lifecycle.Rule) error { if !rule.Expiration.IsDateNull() { i++ } - if rule.Expiration.IsDeleteMarkerExpirationEnabled() { + if rule.Expiration.IsDeleteMarkerExpirationEnabled() || rule.Expiration.DeleteAll.IsEnabled() { i++ } if i > 1 { @@ -242,7 +242,7 @@ func parseExpiryDays(expiryDayStr string) (lifecycle.ExpirationDays, *probe.Erro } // Returns lifecycleExpiration to be included in lifecycleRule -func parseExpiry(expiryDate, expiryDays *string, expiredDeleteMarker *bool) (lfcExp lifecycle.Expiration, err *probe.Error) { +func parseExpiry(expiryDate, expiryDays *string, expiredDeleteMarker, expiredObjectAllVersions *bool) (lfcExp lifecycle.Expiration, err *probe.Error) { if expiryDate != nil { date, err := parseExpiryDate(*expiryDate) if err != nil { @@ -263,5 +263,9 @@ func parseExpiry(expiryDate, expiryDays *string, expiredDeleteMarker *bool) (lfc lfcExp.DeleteMarker = lifecycle.ExpireDeleteMarker(*expiredDeleteMarker) } + if expiredObjectAllVersions != nil { + lfcExp.DeleteAll = lifecycle.ExpirationBoolean(*expiredObjectAllVersions) + } + return lfcExp, nil } diff --git a/go.mod b/go.mod index b1cb25ebcf..be10c28faf 100644 --- a/go.mod +++ b/go.mod @@ -27,7 +27,7 @@ require ( github.com/pkg/xattr v0.4.9 github.com/posener/complete v1.2.3 github.com/prometheus/client_golang v1.17.0 - github.com/prometheus/prom2json v1.3.3 + github.com/prometheus/prom2json v1.3.3 // indirect github.com/rjeczalik/notify v0.9.3 github.com/rs/xid v1.5.0 github.com/shirou/gopsutil/v3 v3.23.11 @@ -51,7 +51,6 @@ require ( github.com/muesli/reflow v0.3.0 github.com/navidys/tvxwidgets v0.4.1 github.com/olekukonko/tablewriter v0.0.5 - github.com/prometheus/client_model v0.5.0 github.com/rivo/tview v0.0.0-20231206124440-5f078138442e github.com/vbauerster/mpb/v8 v8.7.1 golang.org/x/term v0.15.0 @@ -65,6 +64,7 @@ require ( github.com/kr/pretty v0.3.1 // indirect github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect github.com/minio/mux v1.9.0 // indirect + github.com/prometheus/client_model v0.5.0 // indirect github.com/rogpeppe/go-internal v1.10.0 // indirect github.com/safchain/ethtool v0.3.0 // indirect github.com/shoenig/go-m1cpu v0.1.6 // indirect @@ -131,3 +131,5 @@ require ( google.golang.org/protobuf v1.31.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect ) + +replace github.com/minio/minio-go/v7 v7.0.66 => github.com/shtripat/minio-go/v7 v7.0.0-20240202175226-e4e2f9e24339 diff --git a/go.sum b/go.sum index 7c54a087f5..527cb21ecb 100644 --- a/go.sum +++ b/go.sum @@ -146,8 +146,6 @@ github.com/minio/madmin-go/v3 v3.0.41 h1:9pU7xW51cn8Ixy8ftaRfp4SbHtnhEPlwU6mMA9K github.com/minio/madmin-go/v3 v3.0.41/go.mod h1:4QN2NftLSV7MdlT50dkrenOMmNVHluxTvlqJou3hte8= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= -github.com/minio/minio-go/v7 v7.0.66 h1:bnTOXOHjOqv/gcMuiVbN9o2ngRItvqE774dG9nq0Dzw= -github.com/minio/minio-go/v7 v7.0.66/go.mod h1:DHAgmyQEGdW3Cif0UooKOyrT3Vxs82zNdV6tkKhRtbs= github.com/minio/mux v1.9.0 h1:dWafQFyEfGhJvK6AwLOt83bIG5bxKxKJnKMCi0XAaoA= github.com/minio/mux v1.9.0/go.mod h1:1pAare17ZRL5GpmNL+9YmqHoWnLmMZF9C/ioUCfy0BQ= github.com/minio/pkg/v2 v2.0.7 h1:vJZ+XUTDeUe/cHpPZSyG/+54252dg6RQKU5K1jXfy/A= @@ -224,6 +222,8 @@ github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFt github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= +github.com/shtripat/minio-go/v7 v7.0.0-20240202175226-e4e2f9e24339 h1:TO/xo3GtXfv4/ILPhjWIweWwAO8souprDHtrzPLII/o= +github.com/shtripat/minio-go/v7 v7.0.0-20240202175226-e4e2f9e24339/go.mod h1:+UXocnUeZ3wHvVh5s95gcrA4YjMIbccT6ubB+1m054A= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= From 985a3ad44e7b57a9c2e4f0cd9bbebf27fe21e682 Mon Sep 17 00:00:00 2001 From: Shubhendu Ram Tripathi Date: Mon, 5 Feb 2024 19:47:58 +0530 Subject: [PATCH 2/5] remove unwanted check Signed-off-by: Shubhendu Ram Tripathi --- cmd/ilm/parse.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/ilm/parse.go b/cmd/ilm/parse.go index 8a22adb1a1..8bc7832c0e 100644 --- a/cmd/ilm/parse.go +++ b/cmd/ilm/parse.go @@ -104,7 +104,7 @@ func validateExpiration(rule lifecycle.Rule) error { if !rule.Expiration.IsDateNull() { i++ } - if rule.Expiration.IsDeleteMarkerExpirationEnabled() || rule.Expiration.DeleteAll.IsEnabled() { + if rule.Expiration.IsDeleteMarkerExpirationEnabled() { i++ } if i > 1 { From 8c8f19f8ec9b2e02c97569fdfd2718b56c3ad8fc Mon Sep 17 00:00:00 2001 From: Shubhendu Ram Tripathi Date: Tue, 6 Feb 2024 16:20:42 +0530 Subject: [PATCH 3/5] apply delete all flag as well Signed-off-by: Shubhendu Ram Tripathi --- cmd/ilm/options.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cmd/ilm/options.go b/cmd/ilm/options.go index 47d4218e99..be0441e5cf 100644 --- a/cmd/ilm/options.go +++ b/cmd/ilm/options.go @@ -435,6 +435,12 @@ func ApplyRuleFields(dest *lifecycle.Rule, opts LifecycleOptions) *probe.Error { dest.Expiration.DeleteMarker = lifecycle.ExpireDeleteMarker(*opts.ExpiredObjectDeleteMarker) dest.Expiration.Days = 0 dest.Expiration.Date = lifecycle.ExpirationDate{} + // safe to check for delete here as delete marker would at least always come as false + if opts.ExpiredObjectAllversions != nil { + dest.Expiration.DeleteAll = lifecycle.ExpirationBoolean(*opts.ExpiredObjectAllversions) + dest.Expiration.Days = 0 + dest.Expiration.Date = lifecycle.ExpirationDate{} + } } if opts.TransitionDate != nil { From 36540a92a23617bb7a2f79d953a646275a6e5f92 Mon Sep 17 00:00:00 2001 From: Shubhendu Ram Tripathi Date: Thu, 15 Feb 2024 09:55:18 +0530 Subject: [PATCH 4/5] use latest minio-go Signed-off-by: Shubhendu Ram Tripathi --- go.mod | 4 +--- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index be10c28faf..237d866228 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/minio/colorjson v1.0.6 github.com/minio/filepath v1.0.0 github.com/minio/md5-simd v1.1.2 // indirect - github.com/minio/minio-go/v7 v7.0.66 + github.com/minio/minio-go/v7 v7.0.67 github.com/minio/selfupdate v0.6.0 github.com/minio/sha256-simd v1.0.1 github.com/mitchellh/go-homedir v1.1.0 @@ -131,5 +131,3 @@ require ( google.golang.org/protobuf v1.31.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect ) - -replace github.com/minio/minio-go/v7 v7.0.66 => github.com/shtripat/minio-go/v7 v7.0.0-20240202175226-e4e2f9e24339 diff --git a/go.sum b/go.sum index 527cb21ecb..25428acf41 100644 --- a/go.sum +++ b/go.sum @@ -146,6 +146,8 @@ github.com/minio/madmin-go/v3 v3.0.41 h1:9pU7xW51cn8Ixy8ftaRfp4SbHtnhEPlwU6mMA9K github.com/minio/madmin-go/v3 v3.0.41/go.mod h1:4QN2NftLSV7MdlT50dkrenOMmNVHluxTvlqJou3hte8= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.67 h1:BeBvZWAS+kRJm1vGTMJYVjKUNoo0FoEt/wUWdUtfmh8= +github.com/minio/minio-go/v7 v7.0.67/go.mod h1:+UXocnUeZ3wHvVh5s95gcrA4YjMIbccT6ubB+1m054A= github.com/minio/mux v1.9.0 h1:dWafQFyEfGhJvK6AwLOt83bIG5bxKxKJnKMCi0XAaoA= github.com/minio/mux v1.9.0/go.mod h1:1pAare17ZRL5GpmNL+9YmqHoWnLmMZF9C/ioUCfy0BQ= github.com/minio/pkg/v2 v2.0.7 h1:vJZ+XUTDeUe/cHpPZSyG/+54252dg6RQKU5K1jXfy/A= @@ -222,8 +224,6 @@ github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFt github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= -github.com/shtripat/minio-go/v7 v7.0.0-20240202175226-e4e2f9e24339 h1:TO/xo3GtXfv4/ILPhjWIweWwAO8souprDHtrzPLII/o= -github.com/shtripat/minio-go/v7 v7.0.0-20240202175226-e4e2f9e24339/go.mod h1:+UXocnUeZ3wHvVh5s95gcrA4YjMIbccT6ubB+1m054A= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= From c13ab497907eb8096a4033e584bf36cc2c3875e1 Mon Sep 17 00:00:00 2001 From: Shubhendu Ram Tripathi Date: Tue, 20 Feb 2024 10:19:03 +0530 Subject: [PATCH 5/5] review comments Signed-off-by: Shubhendu Ram Tripathi --- cmd/ilm-rule-add.go | 2 +- cmd/ilm/options.go | 12 ++++-------- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/cmd/ilm-rule-add.go b/cmd/ilm-rule-add.go index 9f939209ca..e77d3b28c2 100644 --- a/cmd/ilm-rule-add.go +++ b/cmd/ilm-rule-add.go @@ -183,7 +183,7 @@ var ilmAddFlags = []cli.Flag{ Usage: "remote tier name to transition", }, cli.BoolFlag{ - Name: "expired-object-all-versions", + Name: "expire-all-object-versions", Usage: "expire all object versions", }, } diff --git a/cmd/ilm/options.go b/cmd/ilm/options.go index be0441e5cf..21b3c117a7 100644 --- a/cmd/ilm/options.go +++ b/cmd/ilm/options.go @@ -370,8 +370,8 @@ func GetLifecycleOptions(ctx *cli.Context) (LifecycleOptions, *probe.Error) { if f := "noncurrent-transition-newer"; ctx.IsSet(f) { newerNoncurrentTransitionVersions = intPtr(ctx.Int(f)) } - if ctx.IsSet("expired-object-all-versions") { - expiredObjectAllversions = boolPtr(ctx.Bool("expired-object-all-versions")) + if ctx.IsSet("expire-all-object-versions") { + expiredObjectAllversions = boolPtr(ctx.Bool("expire-all-object-versions")) } return LifecycleOptions{ @@ -435,12 +435,8 @@ func ApplyRuleFields(dest *lifecycle.Rule, opts LifecycleOptions) *probe.Error { dest.Expiration.DeleteMarker = lifecycle.ExpireDeleteMarker(*opts.ExpiredObjectDeleteMarker) dest.Expiration.Days = 0 dest.Expiration.Date = lifecycle.ExpirationDate{} - // safe to check for delete here as delete marker would at least always come as false - if opts.ExpiredObjectAllversions != nil { - dest.Expiration.DeleteAll = lifecycle.ExpirationBoolean(*opts.ExpiredObjectAllversions) - dest.Expiration.Days = 0 - dest.Expiration.Date = lifecycle.ExpirationDate{} - } + } else if opts.ExpiredObjectAllversions != nil { + dest.Expiration.DeleteAll = lifecycle.ExpirationBoolean(*opts.ExpiredObjectAllversions) } if opts.TransitionDate != nil {