Skip to content

Commit

Permalink
cmp: Compare request times (#195)
Browse files Browse the repository at this point in the history 
Before:
```
λ warp cmp before.csv.zst after.csv.zst
274918 operations loaded... Done!
366324 operations loaded... Done!
-------------------
Operation: DELETE
Operations: 27495 -> 36629
* Average: +33.21% (+30.4) obj/s
-------------------
Operation: GET
Operations: 123710 -> 164845
* Average: +33.25% (+0.1 MiB/s) throughput, +33.25% (+137.1) obj/s
* First Byte: Average: -3.951011ms (-17%), Median: -757.1µs (-27%), Best: +1.8µs (+0%), Worst: -506.2678ms (-16%)
-------------------
Operation: PUT
Operations: 41237 -> 54949
* Average: +33.26% (+0.0 MiB/s) throughput, +33.26% (+45.7) obj/s
-------------------
Operation: STAT
Operations: 82476 -> 109901
* Average: +33.26% (+91.4) obj/s
```

After:
```
λ warp cmp before.csv.zst after.csv.zst
274918 operations loaded... Done!
366324 operations loaded... Done!
-------------------
Operation: DELETE
Operations: 27495 -> 36629
* Average: +33.21% (+30.4) obj/s
* Requests: Avg: -24.6ms (-35%), P50: -1.9756ms (-28%), P99: -330.2085ms (-26%), Best: -56.6µs (-3%), Worst: -628.0938ms (-7%)
-------------------
Operation: GET
Operations: 123710 -> 164845
* Average: +33.25% (+0.1 MiB/s) throughput, +33.25% (+137.1) obj/s
* Requests: Avg: -3.95ms (-17%), P50: -795.3µs (-28%), P99: -54.9509ms (-7%), Best: -500ns (-0%), Worst: -506.2678ms (-16%)
* TTFB: Avg: -3.95ms (-17%), P50: -757.1µs (-27%), P99: -54.9509ms (-7%), Best: +1.8µs (+0%), Worst: -506.2678ms (-16%)
-------------------
Operation: PUT
Operations: 41237 -> 54949
* Average: +33.26% (+0.0 MiB/s) throughput, +33.26% (+45.7) obj/s
* Requests: Avg: -23.7ms (-33%), P50: -10.3µs (-0%), P99: -317.7879ms (-26%), Best: -301.8µs (-10%), Worst: +716.2566ms (+9%)
-------------------
Operation: STAT
Operations: 82476 -> 109901
* Average: +33.26% (+91.4) obj/s
* Requests: Avg: -3.1ms (-14%), P50: -216µs (-10%), P99: -24.2622ms (-3%), Best: -2.1µs (-0%), Worst: -1.7413943s (-44%)
```
  • Loading branch information
@klauspost
klauspost authored Nov 9, 2021
1 parent b8830bc commit 8f72511
Show file tree
Hide file tree
Showing 2 changed files with 93 additions and 3 deletions.
7 changes: 6 additions & 1 deletion cli/cmp.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -135,9 +135,14 @@ func printCompare(ctx *cli.Context, before, after bench.Operations) {
if timeDur(before) != timeDur(after) {
console.Println("Duration:", timeDur(before), "->", timeDur(after))
}
if cmp.Reqs.Before.AvgObjSize != cmp.Reqs.After.AvgObjSize {
console.Println("Object size: %d->%d, ", cmp.Reqs.Before.AvgObjSize, cmp.Reqs.After.AvgObjSize)
}
console.Println("* Average:", cmp.Average)
console.Println("* Requests:", cmp.Reqs.String())

if cmp.TTFB != nil {
console.Println("* First Byte:", cmp.TTFB)
console.Println("* TTFB:", cmp.TTFB)
}
if !isMultiOp {
console.SetColor("Print", color.New(color.FgWhite))
Expand Down
89 changes: 87 additions & 2 deletions pkg/bench/compare.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ type Comparison struct {
Op string

TTFB *TTFBCmp
Reqs CmpReqs

Average CmpSegment
Fastest CmpSegment
Expand All @@ -44,6 +45,85 @@ type CmpSegment struct {
OpsEndedPerSec float64
}

type CmpReqs struct {
CmpRequests
Before, After CmpRequests
}

func (c *CmpReqs) Compare(before, after Operations) {
c.Before.fill(before)
c.After.fill(after)
a := c.After
b := c.Before
c.CmpRequests =
CmpRequests{
Average: a.Average - b.Average,
Worst: a.Worst - b.Worst,
Best: a.Best - b.Best,
Median: a.Median - b.Median,
P25: a.P25 - b.P25,
P75: a.P75 - b.P75,
P90: a.P90 - b.P90,
P99: a.P99 - b.P99,
}
}

type CmpRequests struct {
AvgObjSize int64
Requests int
Average time.Duration
Best time.Duration
P25 time.Duration
Median time.Duration
P75 time.Duration
P90 time.Duration
P99 time.Duration
Worst time.Duration
}

func (c *CmpRequests) fill(ops Operations) {
ops.SortByDuration()
c.Requests = len(ops)
c.AvgObjSize = ops.AvgSize()
c.Average = ops.AvgDuration()
c.Best = ops.Median(0).Duration()
c.P25 = ops.Median(0.25).Duration()
c.Median = ops.Median(0.5).Duration()
c.P75 = ops.Median(0.75).Duration()
c.P90 = ops.Median(0.9).Duration()
c.P99 = ops.Median(0.99).Duration()
c.Worst = ops.Median(1).Duration()
}

// String returns a human readable representation of the TTFB comparison.
func (c *CmpReqs) String() string {
if c == nil {
return ""
}
return fmt.Sprintf("Avg: %s%v (%s%.f%%), P50: %s%v (%s%.f%%), P99: %s%v (%s%.f%%), Best: %s%v (%s%.f%%), Worst: %s%v (%s%.f%%)",
plusPositiveD(c.Average),
c.Average.Round(time.Millisecond/20),
plusPositiveD(c.Average),
100*(float64(c.After.Average)-float64(c.Before.Average))/float64(c.Before.Average),
plusPositiveD(c.Median),
c.Median,
plusPositiveD(c.Median),
100*(float64(c.After.Median)-float64(c.Before.Median))/float64(c.Before.Median),
plusPositiveD(c.P99),
c.P99,
plusPositiveD(c.P99),
100*(float64(c.After.P99)-float64(c.Before.P99))/float64(c.Before.P99),
plusPositiveD(c.Best),
c.Best,
plusPositiveD(c.Best),
100*(float64(c.After.Best)-float64(c.Before.Best))/float64(c.Before.Best),
plusPositiveD(c.Worst),
c.Worst,
plusPositiveD(c.Worst),
100*(float64(c.After.Worst)-float64(c.Before.Worst))/float64(c.Before.Worst),
)
}

// Compare sets c to a comparison between before and after.
func (c *CmpSegment) Compare(before, after Segment) {
c.Before = &before
Expand Down Expand Up @@ -118,15 +198,19 @@ func (t *TTFBCmp) String() string {
if t == nil {
return ""
}
return fmt.Sprintf("Average: %s%v (%s%.f%%), Median: %s%v (%s%.f%%), Best: %s%v (%s%.f%%), Worst: %s%v (%s%.f%%)",
return fmt.Sprintf("Avg: %s%v (%s%.f%%), P50: %s%v (%s%.f%%), P99: %s%v (%s%.f%%), Best: %s%v (%s%.f%%), Worst: %s%v (%s%.f%%)",
plusPositiveD(t.Average),
t.Average,
t.Average.Round(time.Millisecond/20),
plusPositiveD(t.Average),
100*(float64(t.After.Average)-float64(t.Before.Average))/float64(t.Before.Average),
plusPositiveD(t.Median),
t.Median,
plusPositiveD(t.Median),
100*(float64(t.After.Median)-float64(t.Before.Median))/float64(t.Before.Median),
plusPositiveD(t.P99),
t.P99,
plusPositiveD(t.P99),
100*(float64(t.After.P99)-float64(t.Before.P99))/float64(t.Before.P99),
plusPositiveD(t.Best),
t.Best,
plusPositiveD(t.Best),
Expand Down Expand Up @@ -192,6 +276,7 @@ func Compare(before, after Operations, analysis time.Duration, allThreads bool)

beforeTotals, beforeTTFB := before.Total(allThreads), before.TTFB(before.TimeRange())
afterTotals, afterTTFB := after.Total(allThreads), after.TTFB(after.TimeRange())
res.Reqs.Compare(before, after)

res.Average.Compare(beforeTotals, afterTotals)
res.TTFB = beforeTTFB.Compare(afterTTFB)
Expand Down

0 comments on commit 8f72511

Please sign in to comment.