Skip to content

Commit

Permalink
Add host file input
Browse files Browse the repository at this point in the history
Allow file as host input for benchmark targets or clients.

Example `warp-hosts.txt`:

```
127.0.0.1:{9001...9004}

127.1.2.3
giga.abc.com
```

Debug printing loaded hosts:
```
λ warp stat --host=file:warp-hosts.txt
HOSTS: [127.0.0.1:9001 127.0.0.1:9002 127.0.0.1:9003 127.0.0.1:9004 127.1.2.3 giga.abc.com]

λ warp stat --warp-client=file:warp-hosts.txt
hosts [127.0.0.1:9001 127.0.0.1:9002 127.0.0.1:9003 127.0.0.1:9004 127.1.2.3 giga.abc.com]
```

Slightly more generic version of minio#297 (and avoids another flag)
  • Loading branch information
klauspost committed Jan 26, 2024
1 parent 2e5e05a commit 9cdfd13
Show file tree
Hide file tree
Showing 3 changed files with 57 additions and 2 deletions.
4 changes: 4 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,9 @@ Multiple S3 hosts can be specified as comma-separated values, for instance
Alternatively numerical ranges can be specified using `--host=10.0.0.{1...10}:9000` which will add
`10.0.0.1` through `10.0.0.10`. This syntax can be used for any part of the host name and port.

A file with newline separated hosts can also be specified using `file:` prefix and a file name.
For distributed tests the file will be read locally and sent to each client.

By default a host is chosen between the hosts that have the least number of requests running
and with the longest time since the last request finished. This will ensure that in cases where
hosts operate at different speeds that the fastest servers will get the most requests.
Expand Down Expand Up @@ -112,6 +115,7 @@ Each client will also save its own data locally.

Enabling server mode is done by adding `--warp-client=client-{1...10}:7761`
or a comma separated list of warp client hosts.
Finally, a file with newline separated hosts can also be specified using `file:` prefix and a file name.
If no host port is specified the default is added.

Example:
Expand Down
20 changes: 19 additions & 1 deletion cli/benchserver.go
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,20 @@ func runServerBenchmark(ctx *cli.Context, b bench.Benchmark) (bool, error) {
"syncstart": {},
"analyze.out": {},
}
transformFlags := map[string]func(flag cli.Flag) (string, error){
// Special handling for hosts, we read files and expand it.
"host": func(flag cli.Flag) (string, error) {
hostsIn := flag.String()
if !strings.Contains(hostsIn, "file:") {
return flagToJSON(ctx, flag)
}
// This is a file, we will read it locally and expand.
hosts := parseHosts(hostsIn, false)
// Rejoin
return strings.Join(hosts, ","), nil
},
}

req := serverRequest{
Operation: serverReqBenchmark,
}
Expand All @@ -130,7 +144,11 @@ func runServerBenchmark(ctx *cli.Context, b bench.Benchmark) (bool, error) {
}
if ctx.IsSet(flag.GetName()) {
var err error
req.Benchmark.Flags[flag.GetName()], err = flagToJSON(ctx, flag)
if t := transformFlags[flag.GetName()]; t != nil {
req.Benchmark.Flags[flag.GetName()], err = t(flag)
} else {
req.Benchmark.Flags[flag.GetName()], err = flagToJSON(ctx, flag)
}
if err != nil {
return true, err
}
Expand Down
35 changes: 34 additions & 1 deletion cli/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,11 @@
package cli

import (
"bufio"
"crypto/tls"
"crypto/x509"
"errors"
"fmt"
"log"
"math"
"math/rand"
Expand Down Expand Up @@ -227,7 +229,38 @@ func parseHosts(h string, resolveDNS bool) []string {
var dst []string
for _, host := range hosts {
if !ellipses.HasEllipses(host) {
dst = append(dst, host)
if !strings.HasPrefix(host, "file:") {
dst = append(dst, host)
continue
}
// If host starts with file:, then it is a file containing hosts.
f, err := os.Open(strings.TrimPrefix(host, "file:"))
if err != nil {
fatalIf(probe.NewError(err), "Unable to open host file")
}
defer f.Close()
scanner := bufio.NewScanner(f)
for scanner.Scan() {
host := strings.TrimSpace(scanner.Text())
if len(host) == 0 {
continue
}
if !ellipses.HasEllipses(host) {
dst = append(dst, host)
continue
}
patterns, perr := ellipses.FindEllipsesPatterns(host)
if perr != nil {
fatalIf(probe.NewError(perr), fmt.Sprintf("Unable to parse host parameter: %s", host))
log.Fatal(perr.Error())
}
for _, lbls := range patterns.Expand() {
dst = append(dst, strings.Join(lbls, ""))
}
}
if err := scanner.Err(); err != nil {
fatalIf(probe.NewError(err), "Unable to read host file")
}
continue
}
patterns, perr := ellipses.FindEllipsesPatterns(host)
Expand Down

0 comments on commit 9cdfd13

Please sign in to comment.