Add versions to get/stat runs (#203)

Allow testing reads with different version count.

README.md

@@ -269,6 +269,9 @@ A similar benchmark is called `versioned` which operates on versioned objects.
 Benchmarking get operations will upload `--objects` objects of size `--obj.size` 
 and attempt to download as many it can within `--duration`.
 
+If versioned listing should be tested, it is possible by setting `--versions=n` (default 1),
+which will add multiple versions of each object and request individual versions.
+
 Objects will be uploaded with `--concurrent` different prefixes, 
 except if `--noprefix` is specified. Downloads are chosen randomly between all uploaded data.
 
@@ -337,6 +340,9 @@ Throughput, split into 59 x 1s:
 Benchmarking list operations will upload `--objects` objects of size `--obj.size` with `--concurrent` prefixes. 
 The list operations are done per prefix.
 
+If versioned listing should be tested, it is possible by setting `--versions=N` (default 1), 
+which will add multiple versions of each object and use `ListObjectVersions` for listing.
+
 The analysis will include the upload stats as `PUT` operations and the `LIST` operations separately. 
 The time from request start to first object is recorded as well and can be accessed using the `--analyze.v` parameter.
 
@@ -355,6 +361,9 @@ Throughput, split into 59 x 1s:
 Benchmarking [stat object](https://docs.min.io/docs/golang-client-api-reference#StatObject) operations 
 will upload `--objects` objects of size `--obj.size` with `--concurrent` prefixes.
 
+If versioned listing should be tested, it is possible by setting `--versions=n` (default 1),
+which will add multiple versions of each object and request information for individual versions.
+
 The main benchmark will do individual requests to get object information for the uploaded objects.
 
 Since the object size is of little importance, only objects per second is reported.

cli/get.go

@@ -40,6 +40,11 @@ var (
 			Name:  "range",
 			Usage: "Do ranged get operations. Will request with random offset and length.",
 		},
+		cli.IntFlag{
+			Name:  "versions",
+			Value: 1,
+			Usage: "Number of versions to upload. If more than 1, versioned listing will be benchmarked",
+		},
 	}
 )
 
@@ -75,6 +80,7 @@ func mainGet(ctx *cli.Context) error {
 			Location:    "",
 			PutOpts:     putOpts(ctx),
 		},
+		Versions:      ctx.Int("versions"),
 		RandomRanges:  ctx.Bool("range"),
 		CreateObjects: ctx.Int("objects"),
 		GetOpts:       minio.GetObjectOptions{ServerSideEncryption: sse},
@@ -86,7 +92,9 @@ func checkGetSyntax(ctx *cli.Context) {
 	if ctx.NArg() > 0 {
 		console.Fatal("Command takes no arguments")
 	}
-
+	if ctx.Int("versions") < 1 {
+		console.Fatal("At least one version must be tested")
+	}
 	checkAnalyze(ctx)
 	checkBenchmark(ctx)
 }

cli/stat.go

@@ -36,6 +36,11 @@ var (
 			Value: "1KB",
 			Usage: "Size of each generated object. Can be a number or 10KB/MB/GB. All sizes are base 2 binary.",
 		},
+		cli.IntFlag{
+			Name:  "versions",
+			Value: 1,
+			Usage: "Number of versions to upload. If more than 1, versioned listing will be benchmarked",
+		},
 	}
 )
 
@@ -72,6 +77,7 @@ func mainStat(ctx *cli.Context) error {
 			Location:    "",
 			PutOpts:     putOpts(ctx),
 		},
+		Versions:      ctx.Int("versions"),
 		CreateObjects: ctx.Int("objects"),
 		StatOpts: minio.StatObjectOptions{
 			ServerSideEncryption: sse,
@@ -84,7 +90,9 @@ func checkStatSyntax(ctx *cli.Context) {
 	if ctx.NArg() > 0 {
 		console.Fatal("Command takes no arguments")
 	}
-
+	if ctx.Int("versions") < 1 {
+		console.Fatal("At least one version must be tested")
+	}
 	checkAnalyze(ctx)
 	checkBenchmark(ctx)
 }

pkg/bench/get.go

@@ -38,6 +38,7 @@ type Get struct {
 	RandomRanges  bool
 	Collector     *Collector
 	objects       generator.Objects
+	Versions      int
 
 	// Default Get options.
 	GetOpts minio.GetObjectOptions
@@ -50,16 +51,32 @@ func (g *Get) Prepare(ctx context.Context) error {
 	if err := g.createEmptyBucket(ctx); err != nil {
 		return err
 	}
-	src := g.Source()
+	if g.Versions > 1 {
+		cl, done := g.Client()
+		if !g.Versioned {
+			err := cl.EnableVersioning(ctx, g.Bucket)
+			if err != nil {
+				return err
+			}
+			g.Versioned = true
+		}
+		done()
+	}
 	console.Eraseline()
-	console.Info("\rUploading ", g.CreateObjects, " objects of ", src.String())
+	x := ""
+	if g.Versions > 1 {
+		x = fmt.Sprintf(" with %d versions each", g.Versions)
+	}
+	console.Info("\rUploading ", g.CreateObjects, " objects", x)
+
 	var wg sync.WaitGroup
 	wg.Add(g.Concurrency)
 	g.Collector = NewCollector()
 	obj := make(chan struct{}, g.CreateObjects)
 	for i := 0; i < g.CreateObjects; i++ {
 		obj <- struct{}{}
 	}
+	rcv := g.Collector.rcv
 	close(obj)
 	var groupErr error
 	var mu sync.Mutex
@@ -68,58 +85,63 @@ func (g *Get) Prepare(ctx context.Context) error {
 		go func(i int) {
 			defer wg.Done()
 			src := g.Source()
-			for range obj {
-				opts := g.PutOpts
-				rcv := g.Collector.Receiver()
-				done := ctx.Done()
+			opts := g.PutOpts
 
+			for range obj {
 				select {
-				case <-done:
+				case <-ctx.Done():
 					return
 				default:
 				}
 				obj := src.Object()
-				client, cldone := g.Client()
-				op := Operation{
-					OpType:   http.MethodPut,
-					Thread:   uint16(i),
-					Size:     obj.Size,
-					File:     obj.Name,
-					ObjPerOp: 1,
-					Endpoint: client.EndpointURL().String(),
-				}
-				opts.ContentType = obj.ContentType
-				op.Start = time.Now()
-				res, err := client.PutObject(ctx, g.Bucket, obj.Name, obj.Reader, obj.Size, opts)
-				op.End = time.Now()
-				if err != nil {
-					err := fmt.Errorf("upload error: %w", err)
-					g.Error(err)
-					mu.Lock()
-					if groupErr == nil {
-						groupErr = err
+
+				name := obj.Name
+				for ver := 0; ver < g.Versions; ver++ {
+					// New input for each version
+					obj := src.Object()
+					obj.Name = name
+					client, cldone := g.Client()
+					op := Operation{
+						OpType:   http.MethodPut,
+						Thread:   uint16(i),
+						Size:     obj.Size,
+						File:     obj.Name,
+						ObjPerOp: 1,
+						Endpoint: client.EndpointURL().String(),
 					}
-					mu.Unlock()
-					return
-				}
-				obj.VersionID = res.VersionID
-				if res.Size != obj.Size {
-					err := fmt.Errorf("short upload. want: %d, got %d", obj.Size, res.Size)
-					g.Error(err)
-					mu.Lock()
-					if groupErr == nil {
-						groupErr = err
+					opts.ContentType = obj.ContentType
+					op.Start = time.Now()
+					res, err := client.PutObject(ctx, g.Bucket, obj.Name, obj.Reader, obj.Size, opts)
+					op.End = time.Now()
+					if err != nil {
+						err := fmt.Errorf("upload error: %w", err)
+						g.Error(err)
+						mu.Lock()
+						if groupErr == nil {
+							groupErr = err
+						}
+						mu.Unlock()
+						return
 					}
+					obj.VersionID = res.VersionID
+					if res.Size != obj.Size {
+						err := fmt.Errorf("short upload. want: %d, got %d", obj.Size, res.Size)
+						g.Error(err)
+						mu.Lock()
+						if groupErr == nil {
+							groupErr = err
+						}
+						mu.Unlock()
+						return
+					}
+					cldone()
+					mu.Lock()
+					obj.Reader = nil
+					g.objects = append(g.objects, *obj)
+					g.prepareProgress(float64(len(g.objects)) / float64(g.CreateObjects*g.Versions))
 					mu.Unlock()
-					return
+					rcv <- op
 				}
-				cldone()
-				mu.Lock()
-				obj.Reader = nil
-				g.objects = append(g.objects, *obj)
-				g.prepareProgress(float64(len(g.objects)) / float64(g.CreateObjects))
-				mu.Unlock()
-				rcv <- op
 			}
 		}(i)
 	}
@@ -194,7 +216,9 @@ func (g *Get) Start(ctx context.Context, wait chan struct{}) (Operations, error)
 				}
 				op.Start = time.Now()
 				var err error
-				opts.VersionID = obj.VersionID
+				if g.Versions > 1 {
+					opts.VersionID = obj.VersionID
+				}
 				o, err := client.GetObject(nonTerm, g.Bucket, obj.Name, opts)
 				if err != nil {
 					g.Error("download error:", err)