From 3fafe4396dfafc03c1a7dfba42b9008f9703f59a Mon Sep 17 00:00:00 2001 From: wubx Date: Mon, 17 Oct 2022 22:06:29 +0800 Subject: [PATCH 1/3] add readNums --- go.mod | 7 +++++++ go.sum | 23 +++++++++++++++++++++++ s3bench.go | 13 +++++++++---- 3 files changed, 39 insertions(+), 4 deletions(-) create mode 100644 go.mod create mode 100644 go.sum diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..8965598 --- /dev/null +++ b/go.mod @@ -0,0 +1,7 @@ +module s3bench + +go 1.18 + +require github.com/aws/aws-sdk-go v1.44.116 + +require github.com/jmespath/go-jmespath v0.4.0 // indirect diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..247c22b --- /dev/null +++ b/go.sum @@ -0,0 +1,23 @@ +github.com/aws/aws-sdk-go v1.44.116 h1:NpLIhcvLWXJZAEwvPj3TDHeqp7DleK6ZUVYyW01WNHY= +github.com/aws/aws-sdk-go v1.44.116/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd h1:O7DYs+zxREGLKzKoMQrtrEacpb0ZVXA5rIwylE2Xchk= +golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/s3bench.go b/s3bench.go index 76af88f..5199736 100644 --- a/s3bench.go +++ b/s3bench.go @@ -37,6 +37,7 @@ func main() { objectSize := flag.Int64("objectSize", 80*1024*1024, "size of individual requests in bytes (must be smaller than main memory)") numClients := flag.Int("numClients", 40, "number of concurrent clients") numSamples := flag.Int("numSamples", 200, "total number of requests to send") + readNums := flag.Int("readNums", 10, "total number of requests read") skipCleanup := flag.Bool("skipCleanup", false, "skip deleting objects created by this tool at the end of the run") verbose := flag.Bool("verbose", false, "print verbose per thread status") @@ -92,16 +93,20 @@ func main() { writeResult := params.Run(opWrite) fmt.Println() - fmt.Printf("Running %s test...\n", opRead) - readResult := params.Run(opRead) - fmt.Println() + fmt.Printf("Running %s test...\n", opRead) + fmt.Println() // Repeating the parameters of the test followed by the results fmt.Println(params) fmt.Println() fmt.Println(writeResult) fmt.Println() - fmt.Println(readResult) + + for i := 1; i <= *readNums; i++ { + fmt.Printf("Read %d Round...\n", i) + readResult := params.Run(opRead) + fmt.Println(readResult) + } // Do cleanup if required if !*skipCleanup { From aee21c49407963206f183c0093d310bc22aba048 Mon Sep 17 00:00:00 2001 From: wubx Date: Tue, 18 Oct 2022 11:01:42 +0800 Subject: [PATCH 2/3] make random string as filename --- s3bench.go | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/s3bench.go b/s3bench.go index 5199736..840670a 100644 --- a/s3bench.go +++ b/s3bench.go @@ -11,6 +11,7 @@ import ( "sort" "strings" "time" + "encoding/hex" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/credentials" @@ -26,8 +27,19 @@ const ( ) var bufferBytes []byte +var l []string + +// randToken generates a random hex value. +func randToken(n int) (string, error) { + bytes := make([]byte, n) + if _, err := rand.Read(bytes); err != nil { + return "", err + } + return hex.EncodeToString(bytes), nil +} func main() { + startTime := time.Now() endpoint := flag.String("endpoint", "", "S3 endpoint(s) comma separated - http://IP:PORT,http://IP:PORT") region := flag.String("region", "igneous-test", "AWS region to use, eg: us-west-1|us-east-1, etc") accessKey := flag.String("accessKey", "", "the S3 access key") @@ -54,6 +66,14 @@ func main() { os.Exit(1) } + // Generate the suffix + + + for i :=0; i<=*numSamples; i++ { + token, _ := randToken(16) + l = append(l, token) + } + // Setup and print summary of the accepted parameters params := Params{ requests: make(chan Req), @@ -69,6 +89,7 @@ func main() { fmt.Println(params) fmt.Println() + // Generate the data from which we will do the writting fmt.Printf("Generating in-memory sample data... ") timeGenData := time.Now() @@ -120,7 +141,7 @@ func main() { keyList := make([]*s3.ObjectIdentifier, 0, commitSize) for i := 0; i < *numSamples; i++ { bar := s3.ObjectIdentifier{ - Key: aws.String(fmt.Sprintf("%s%d", *objectNamePrefix, i)), + Key: aws.String(fmt.Sprintf("%s%s", *objectNamePrefix, l[i])), } keyList = append(keyList, &bar) if len(keyList) == commitSize || i == *numSamples-1 { @@ -143,6 +164,7 @@ func main() { } fmt.Printf("Successfully deleted %d/%d objects in %s\n", numSuccessfullyDeleted, *numSamples, time.Since(delStartTime)) } + fmt.Printf("Total running time %s\n", time.Since(startTime)) } func (params *Params) Run(op string) Result { @@ -180,7 +202,7 @@ func (params *Params) Run(op string) Result { func (params *Params) submitLoad(op string) { bucket := aws.String(params.bucketName) for i := 0; i < params.numSamples; i++ { - key := aws.String(fmt.Sprintf("%s%d", params.objectNamePrefix, i)) + key := aws.String(fmt.Sprintf("%s%s", params.objectNamePrefix, l[i])) if op == opWrite { params.requests <- &s3.PutObjectInput{ Bucket: bucket, From 7ec955900a7b60a4473aea953e33e326273a84c2 Mon Sep 17 00:00:00 2001 From: wubx Date: Tue, 18 Oct 2022 11:09:42 +0800 Subject: [PATCH 3/3] Update README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index f0939b8..21958f9 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,8 @@ Run the following command to build the binary. ``` go get github.com/igneous-systems/s3bench +or +go build s3bench.go ``` The binary will be placed under $GOPATH/bin/s3bench.