diff --git a/README.md b/README.md index 05152df3..85c9dcf7 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,9 @@ Multiple S3 hosts can be specified as comma-separated values, for instance Alternatively numerical ranges can be specified using `--host=10.0.0.{1...10}:9000` which will add `10.0.0.1` through `10.0.0.10`. This syntax can be used for any part of the host name and port. +A file with newline separated hosts can also be specified using `file:` prefix and a file name. +For distributed tests the file will be read locally and sent to each client. + By default a host is chosen between the hosts that have the least number of requests running and with the longest time since the last request finished. This will ensure that in cases where hosts operate at different speeds that the fastest servers will get the most requests. @@ -112,6 +115,7 @@ Each client will also save its own data locally. Enabling server mode is done by adding `--warp-client=client-{1...10}:7761` or a comma separated list of warp client hosts. +Finally, a file with newline separated hosts can also be specified using `file:` prefix and a file name. If no host port is specified the default is added. Example: diff --git a/cli/benchserver.go b/cli/benchserver.go index 066fe67c..a1129c7d 100644 --- a/cli/benchserver.go +++ b/cli/benchserver.go @@ -117,6 +117,20 @@ func runServerBenchmark(ctx *cli.Context, b bench.Benchmark) (bool, error) { "syncstart": {}, "analyze.out": {}, } + transformFlags := map[string]func(flag cli.Flag) (string, error){ + // Special handling for hosts, we read files and expand it. + "host": func(flag cli.Flag) (string, error) { + hostsIn := flag.String() + if !strings.Contains(hostsIn, "file:") { + return flagToJSON(ctx, flag) + } + // This is a file, we will read it locally and expand. + hosts := parseHosts(hostsIn, false) + // Rejoin + return strings.Join(hosts, ","), nil + }, + } + req := serverRequest{ Operation: serverReqBenchmark, } @@ -130,7 +144,11 @@ func runServerBenchmark(ctx *cli.Context, b bench.Benchmark) (bool, error) { } if ctx.IsSet(flag.GetName()) { var err error - req.Benchmark.Flags[flag.GetName()], err = flagToJSON(ctx, flag) + if t := transformFlags[flag.GetName()]; t != nil { + req.Benchmark.Flags[flag.GetName()], err = t(flag) + } else { + req.Benchmark.Flags[flag.GetName()], err = flagToJSON(ctx, flag) + } if err != nil { return true, err } diff --git a/cli/client.go b/cli/client.go index eaa1582f..23498e0a 100644 --- a/cli/client.go +++ b/cli/client.go @@ -18,9 +18,11 @@ package cli import ( + "bufio" "crypto/tls" "crypto/x509" "errors" + "fmt" "log" "math" "math/rand" @@ -227,7 +229,38 @@ func parseHosts(h string, resolveDNS bool) []string { var dst []string for _, host := range hosts { if !ellipses.HasEllipses(host) { - dst = append(dst, host) + if !strings.HasPrefix(host, "file:") { + dst = append(dst, host) + continue + } + // If host starts with file:, then it is a file containing hosts. + f, err := os.Open(strings.TrimPrefix(host, "file:")) + if err != nil { + fatalIf(probe.NewError(err), "Unable to open host file") + } + defer f.Close() + scanner := bufio.NewScanner(f) + for scanner.Scan() { + host := strings.TrimSpace(scanner.Text()) + if len(host) == 0 { + continue + } + if !ellipses.HasEllipses(host) { + dst = append(dst, host) + continue + } + patterns, perr := ellipses.FindEllipsesPatterns(host) + if perr != nil { + fatalIf(probe.NewError(perr), fmt.Sprintf("Unable to parse host parameter: %s", host)) + log.Fatal(perr.Error()) + } + for _, lbls := range patterns.Expand() { + dst = append(dst, strings.Join(lbls, "")) + } + } + if err := scanner.Err(); err != nil { + fatalIf(probe.NewError(err), "Unable to read host file") + } continue } patterns, perr := ellipses.FindEllipsesPatterns(host)