From 98d19549602dd88d00652de99e4ce425ea7ba1e7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=A9r=C3=B4me=20LOYET?=
 <822436+fatpat@users.noreply.github.com>
Date: Wed, 6 Dec 2023 06:49:08 +0100
Subject: [PATCH] [delete] bench object from listing the bucket

backport listing feature from get to delete

when doing a delete bench, during the prepare phase, instead of putting
objects in the bucket use the objects already in there. This can be used
to bench a specific use case with an already filled in bucket.

New options:
--list-existing to activate the listing bucket (current
  behaviour remains)
--list-flat: disable recursive listing
---
 README.md           |  9 +++++--
 cli/delete.go       | 11 +++++++++
 pkg/bench/delete.go | 60 +++++++++++++++++++++++++++++++++++++++++++--
 3 files changed, 76 insertions(+), 4 deletions(-)

diff --git a/README.md b/README.md
index 85c9dcf7..ea4821b4 100644
--- a/README.md
+++ b/README.md
@@ -328,13 +328,18 @@ It is possible by forcing md5 checksums on data by using the `--md5` option.
 
 ## DELETE
 
-Benchmarking delete operations will upload `--objects` objects of size `--obj.size` and attempt to
-delete as many it can within `--duration`.
+Benchmarking delete operations will attempt to delete as many objects it can within `--duration`.
+
+By default, `--objects` objects of size `--obj.size` are uploaded beforing doin the actual bench.
 
 The delete operations are done in `--batch` objects per request in `--concurrent` concurrently running requests.
 
 If there are no more objects left the benchmark will end.
 
+Using `--list-existing` will list at most `--objects` from the bucket and delete them instead of
+deleting random objects (set it to 0 to use all objects from the lsiting).
+Listing is restricted to `--prefix` if it is set and recursive listing can be disabled by setting `--list-flat`.
+
 The analysis will include the upload stats as `PUT` operations and the `DELETE` operations.
 
 ```
diff --git a/cli/delete.go b/cli/delete.go
index bc6f42f9..f5d486b8 100644
--- a/cli/delete.go
+++ b/cli/delete.go
@@ -39,6 +39,14 @@ var deleteFlags = []cli.Flag{
 		Value: 100,
 		Usage: "Number of DELETE operations per batch.",
 	},
+	cli.BoolFlag{
+		Name:  "list-existing",
+		Usage: "Instead of preparing the bench by PUTing some objects, only use objects already in the bucket",
+	},
+	cli.BoolFlag{
+		Name:  "list-flat",
+		Usage: "When using --list-existing, do not use recursive listing",
+	},
 }
 
 var deleteCmd = cli.Command{
@@ -68,6 +76,9 @@ func mainDelete(ctx *cli.Context) error {
 		Common:        getCommon(ctx, newGenSource(ctx, "obj.size")),
 		CreateObjects: ctx.Int("objects"),
 		BatchSize:     ctx.Int("batch"),
+		ListExisting:  ctx.Bool("list-existing"),
+		ListFlat:      ctx.Bool("list-flat"),
+		ListPrefix:    ctx.String("prefix"),
 	}
 	return runBench(ctx, &b)
 }
diff --git a/pkg/bench/delete.go b/pkg/bench/delete.go
index 529658fd..34c58c6f 100644
--- a/pkg/bench/delete.go
+++ b/pkg/bench/delete.go
@@ -37,11 +37,68 @@ type Delete struct {
 
 	CreateObjects int
 	BatchSize     int
+	ListExisting  bool
+	ListFlat      bool
+	ListPrefix    string
 }
 
 // Prepare will create an empty bucket or delete any content already there
 // and upload a number of objects.
 func (d *Delete) Prepare(ctx context.Context) error {
+	var groupErr error
+
+	// prepare the bench by listing object from the bucket
+	if d.ListExisting {
+		cl, done := d.Client()
+
+		// ensure the bucket exist
+		found, err := cl.BucketExists(ctx, d.Bucket)
+		if err != nil {
+			return err
+		}
+		if !found {
+			return fmt.Errorf("bucket %s does not exist and --list-existing has been set", d.Bucket)
+		}
+
+		// list all objects
+		ctx, cancel := context.WithCancel(ctx)
+		defer cancel()
+		objectCh := cl.ListObjects(ctx, d.Bucket, minio.ListObjectsOptions{
+			Prefix:       d.ListPrefix,
+			Recursive:    !d.ListFlat,
+		})
+
+		for object := range objectCh {
+			if object.Err != nil {
+				return object.Err
+			}
+			obj := generator.Object{
+				Name: object.Key,
+				Size: object.Size,
+			}
+
+			d.objects = append(d.objects, obj)
+
+			// limit to ListingMaxObjects
+			if d.CreateObjects > 0 && len(d.objects) >= d.CreateObjects {
+				break
+			}
+		}
+		if len(d.objects) == 0 {
+			return (fmt.Errorf("no objects found for bucket %s", d.Bucket))
+		}
+		done()
+		d.Collector = NewCollector()
+
+		// Shuffle objects.
+		// Benchmark will pick from slice in order.
+		a := d.objects
+		rand.Shuffle(len(a), func(i, j int) {
+			a[i], a[j] = a[j], a[i]
+		})
+		return groupErr
+	}
+
 	if err := d.createEmptyBucket(ctx); err != nil {
 		return err
 	}
@@ -57,7 +114,6 @@ func (d *Delete) Prepare(ctx context.Context) error {
 	}
 	close(obj)
 	var mu sync.Mutex
-	var groupErr error
 	for i := 0; i < d.Concurrency; i++ {
 		go func(i int) {
 			defer wg.Done()
@@ -217,7 +273,7 @@ func (d *Delete) Start(ctx context.Context, wait chan struct{}) (Operations, err
 
 // Cleanup deletes everything uploaded to the bucket.
 func (d *Delete) Cleanup(ctx context.Context) {
-	if len(d.objects) > 0 {
+	if len(d.objects) > 0 && !d.ListExisting{
 		d.deleteAllInBucket(ctx, d.objects.Prefixes()...)
 	}
 }