Skip to content

Commit

Permalink
Add json reporting. (#10)
Browse files Browse the repository at this point in the history 
* Add JSON reporting.

Signed-off-by: Yury-Fridlyand <[email protected]>

* Fix for #26.

Signed-off-by: Yury-Fridlyand <[email protected]>

* Update java/benchmarks/src/main/java/javababushka/benchmarks/utils/Benchmarking.java

Signed-off-by: Yury-Fridlyand <[email protected]>

Co-authored-by: Andrew Carbonetto <[email protected]>

* Use `Optional`.

Signed-off-by: Yury-Fridlyand <[email protected]>

* Address PR feedback.

Signed-off-by: Yury-Fridlyand <[email protected]>

---------

Signed-off-by: Yury-Fridlyand <[email protected]>
Co-authored-by: Andrew Carbonetto <[email protected]>
  • Loading branch information
@Yury-Fridlyand @acarbonetto
Yury-Fridlyand and acarbonetto authored Oct 20, 2023
1 parent 540f49a commit c3d235a
Show file tree
Hide file tree
Showing 7 changed files with 159 additions and 66 deletions.
2 changes: 1 addition & 1 deletion benchmarks/utilities/csv_exporter.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@

json_file_name = os.path.basename(json_file_full_path)

languages = ["csharp", "node", "python", "rust"]
languages = ["csharp", "node", "python", "rust", "java"]
language = next(
(language for language in languages if language in json_file_name), None
)
Expand Down
4 changes: 4 additions & 0 deletions java/benchmarks/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,10 @@ dependencies {
implementation 'io.lettuce:lettuce-core:6.2.6.RELEASE'
implementation 'commons-cli:commons-cli:1.5.0'
implementation group: 'org.apache.commons', name: 'commons-lang3', version: '3.13.0'
implementation group: 'com.google.code.gson', name: 'gson', version: '2.10.1'

compileOnly 'org.projectlombok:lombok:1.18.30'
annotationProcessor 'org.projectlombok:lombok:1.18.30'
}

// Apply a specific Java toolchain to ease working on different environments.
Expand Down
19 changes: 2 additions & 17 deletions java/benchmarks/src/main/java/javababushka/benchmarks/BenchmarkingApp.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,6 @@

import static javababushka.benchmarks.utils.Benchmarking.testClientSetGet;

import java.io.FileWriter;
import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import java.util.Optional;
Expand Down Expand Up @@ -57,14 +55,6 @@ public static void main(String[] args) {
break;
}
}

if (runConfiguration.resultsFile.isPresent()) {
try {
runConfiguration.resultsFile.get().close();
} catch (IOException ioException) {
System.out.println("Error closing results file");
}
}
}

private static Options getOptions() {
Expand Down Expand Up @@ -97,12 +87,7 @@ private static RunConfiguration verifyOptions(CommandLine line) throws ParseExce
}

if (line.hasOption("resultsFile")) {
try {
runConfiguration.resultsFile =
Optional.of(new FileWriter(line.getOptionValue("resultsFile")));
} catch (IOException e) {
throw new ParseException("Unable to write to resultsFile.");
}
runConfiguration.resultsFile = Optional.ofNullable(line.getOptionValue("resultsFile"));
}

if (line.hasOption("dataSize")) {
Expand Down Expand Up @@ -210,8 +195,8 @@ public boolean isEqual(String other) {

public static class RunConfiguration {
public String configuration;
public Optional<FileWriter> resultsFile;
public int dataSize;
public Optional<String> resultsFile;
public List<Integer> concurrentTasks;
public ClientName[] clients;
public String host;
Expand Down
72 changes: 28 additions & 44 deletions java/benchmarks/src/main/java/javababushka/benchmarks/utils/Benchmarking.java
View file
Original file line number Diff line number Diff line change
@@ -1,18 +1,15 @@
package javababushka.benchmarks.utils;

import java.io.FileWriter;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Supplier;
import java.util.stream.Collectors;
import javababushka.benchmarks.AsyncClient;
import javababushka.benchmarks.BenchmarkingApp;
import javababushka.benchmarks.Client;
Expand All @@ -26,6 +23,8 @@ public class Benchmarking {
static final int SIZE_GET_KEYSPACE = 3750000;
static final int SIZE_SET_KEYSPACE = 3000000;
static final int ASYNC_OPERATION_TIMEOUT_SEC = 1;
// measurements are done in nano-seconds, but it should be converted to seconds later
static final double SECONDS_IN_NANO = 1e-9;

private static ChosenAction randomAction() {
if (Math.random() > PROB_GET) {
Expand All @@ -50,7 +49,7 @@ public interface Operation {
void go() throws Exception;
}

private static Pair<ChosenAction, Long> getLatency(Map<ChosenAction, Operation> actions) {
public static Pair<ChosenAction, Long> measurePerformance(Map<ChosenAction, Operation> actions) {
var action = randomAction();
long before = System.nanoTime();
try {
Expand Down Expand Up @@ -84,56 +83,29 @@ private static double stdDeviation(ArrayList<Long> latencies, Double avgLatency)
// This has the side-effect of sorting each latencies ArrayList
public static Map<ChosenAction, LatencyResults> calculateResults(
Map<ChosenAction, ArrayList<Long>> actionLatencies) {
Map<ChosenAction, LatencyResults> results = new HashMap<ChosenAction, LatencyResults>();
Map<ChosenAction, LatencyResults> results = new HashMap<>();

for (Map.Entry<ChosenAction, ArrayList<Long>> entry : actionLatencies.entrySet()) {
ChosenAction action = entry.getKey();
ArrayList<Long> latencies = entry.getValue();

Double avgLatency =
latencies.stream().collect(Collectors.summingLong(Long::longValue))
/ Double.valueOf(latencies.size());
double avgLatency =
SECONDS_IN_NANO * latencies.stream().mapToLong(Long::longValue).sum() / latencies.size();

Collections.sort(latencies);
results.put(
action,
new LatencyResults(
avgLatency,
percentile(latencies, 50),
percentile(latencies, 90),
percentile(latencies, 99),
stdDeviation(latencies, avgLatency)));
SECONDS_IN_NANO * percentile(latencies, 50),
SECONDS_IN_NANO * percentile(latencies, 90),
SECONDS_IN_NANO * percentile(latencies, 99),
SECONDS_IN_NANO * stdDeviation(latencies, avgLatency)));
}

return results;
}

public static void printResults(
Map<ChosenAction, LatencyResults> calculatedResults, Optional<FileWriter> resultsFile) {
if (resultsFile.isPresent()) {
printResults(calculatedResults, resultsFile.get());
} else {
printResults(calculatedResults);
}
}

public static void printResults(
Map<ChosenAction, LatencyResults> resultsMap, FileWriter resultsFile) {
for (Map.Entry<ChosenAction, LatencyResults> entry : resultsMap.entrySet()) {
ChosenAction action = entry.getKey();
LatencyResults results = entry.getValue();

try {
resultsFile.write("Avg. time in ms per " + action + ": " + results.avgLatency / 1000000.0);
resultsFile.write(action + " p50 latency in ms: " + results.p50Latency / 1000000.0);
resultsFile.write(action + " p90 latency in ms: " + results.p90Latency / 1000000.0);
resultsFile.write(action + " p99 latency in ms: " + results.p99Latency / 1000000.0);
resultsFile.write(action + " std dev in ms: " + results.stdDeviation / 1000000.0);
} catch (Exception ignored) {
}
}
}

public static void printResults(Map<ChosenAction, LatencyResults> resultsMap) {
for (Map.Entry<ChosenAction, LatencyResults> entry : resultsMap.entrySet()) {
ChosenAction action = entry.getKey();
Expand Down Expand Up @@ -189,9 +161,10 @@ public static void testClientSetGet(
"> iteration = %d/%d, client# = %d/%d%n",
iterationIncrement + 1, iterations, clientIndex + 1, clientCount);
}

var actions = getActionMap(clients.get(clientIndex), config.dataSize, async);
// operate and calculate tik-tok
Pair<ChosenAction, Long> result =
measurePerformance(clients.get(clientIndex), config.dataSize, async);
Pair<ChosenAction, Long> result = measurePerformance(actions);
actionResults.get(result.getLeft()).add(result.getRight());

iterationIncrement = iterationCounter.getAndIncrement();
Expand All @@ -204,6 +177,7 @@ public static void testClientSetGet(
"===> concurrentNum = %d, clientNum = %d, tasks = %d%n",
concurrentNum, clientCount, tasks.size());
}
long started = System.nanoTime();
tasks.stream()
.map(CompletableFuture::runAsync)
.forEach(
Expand All @@ -215,14 +189,25 @@ public static void testClientSetGet(
}
});

printResults(calculateResults(actionResults), config.resultsFile);
var calculatedResults = calculateResults(actionResults);
if (config.resultsFile.isPresent()) {
JsonWriter.Write(
calculatedResults,
config.resultsFile.get(),
config.dataSize,
clientCreator.get().getName(),
clientCount,
concurrentNum,
iterationCounter.get() * 1e9 / (System.nanoTime() - started));
}
printResults(calculatedResults);
}
}

System.out.println();
}

public static Pair<ChosenAction, Long> measurePerformance(
public static Map<ChosenAction, Operation> getActionMap(
Client client, int dataSize, boolean async) {

String value = RandomStringUtils.randomAlphanumeric(dataSize);
Expand Down Expand Up @@ -251,7 +236,6 @@ public static Pair<ChosenAction, Long> measurePerformance(
.asyncSet(generateKeySet(), value)
.get(ASYNC_OPERATION_TIMEOUT_SEC, TimeUnit.SECONDS)
: () -> ((SyncClient) client).set(generateKeySet(), value));

return getLatency(actions);
return actions;
}
}
103 changes: 103 additions & 0 deletions java/benchmarks/src/main/java/javababushka/benchmarks/utils/JsonWriter.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,103 @@
package javababushka.benchmarks.utils;

import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.reflect.TypeToken;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Map;
import lombok.Builder;
import lombok.Getter;

public class JsonWriter {

public static void Write(
Map<ChosenAction, LatencyResults> calculatedResults,
String resultsFile,
int dataSize,
String client,
int clientCount,
int numOfTasks,
double tps) {

try {
Gson gson = new GsonBuilder().setPrettyPrinting().serializeNulls().create();
Collection<Measurements> recordings = new ArrayList<>();

Path path = Path.of(resultsFile);
if (Files.exists(path)) {
TypeToken<Collection<Measurements>> collectionType = new TypeToken<>() {};
var json = new String(Files.readAllBytes(path));
recordings = gson.fromJson(json, collectionType);
}
var data =
new Measurements.MeasurementsBuilder()
// TODO: is_cluster
.data_size(dataSize)
.client(client)
.client_count(clientCount)
.num_of_tasks(numOfTasks)
.tps(tps)
.get_existing_average_latency(
calculatedResults.get(ChosenAction.GET_EXISTING).avgLatency)
.get_existing_p50_latency(calculatedResults.get(ChosenAction.GET_EXISTING).p50Latency)
.get_existing_p90_latency(calculatedResults.get(ChosenAction.GET_EXISTING).p90Latency)
.get_existing_p99_latency(calculatedResults.get(ChosenAction.GET_EXISTING).p99Latency)
.get_existing_std_dev(calculatedResults.get(ChosenAction.GET_EXISTING).stdDeviation)
.get_non_existing_average_latency(
calculatedResults.get(ChosenAction.GET_NON_EXISTING).avgLatency)
.get_non_existing_p50_latency(
calculatedResults.get(ChosenAction.GET_NON_EXISTING).p50Latency)
.get_non_existing_p90_latency(
calculatedResults.get(ChosenAction.GET_NON_EXISTING).p90Latency)
.get_non_existing_p99_latency(
calculatedResults.get(ChosenAction.GET_NON_EXISTING).p99Latency)
.get_non_existing_std_dev(
calculatedResults.get(ChosenAction.GET_NON_EXISTING).stdDeviation)
.set_average_latency(calculatedResults.get(ChosenAction.SET).avgLatency)
.set_p50_latency(calculatedResults.get(ChosenAction.SET).p50Latency)
.set_p90_latency(calculatedResults.get(ChosenAction.SET).p90Latency)
.set_p99_latency(calculatedResults.get(ChosenAction.SET).p99Latency)
.set_std_dev(calculatedResults.get(ChosenAction.SET).stdDeviation)
.build();

recordings.add(data);

Files.write(path, gson.toJson(recordings).getBytes());
} catch (IOException e) {
System.out.printf(
"Failed to write measurement results into a file '%s': %s%n",
resultsFile, e.getMessage());
e.printStackTrace();
}
}

@Getter
@Builder
public static class Measurements {
private String client;
private int client_count;
private int data_size;
private double get_existing_average_latency;
private double get_existing_p50_latency;
private double get_existing_p90_latency;
private double get_existing_p99_latency;
private double get_existing_std_dev;
private double get_non_existing_average_latency;
private double get_non_existing_p50_latency;
private double get_non_existing_p90_latency;
private double get_non_existing_p99_latency;
private double get_non_existing_std_dev;
private boolean is_cluster;
private int num_of_tasks;
private double set_average_latency;
private double set_p50_latency;
private double set_p90_latency;
private double set_p99_latency;
private double set_std_dev;
private double tps;
}
}
12 changes: 8 additions & 4 deletions java/benchmarks/src/main/java/javababushka/benchmarks/utils/LatencyResults.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,17 @@
// Raw timing results in nanoseconds
public class LatencyResults {
public final double avgLatency;
public final long p50Latency;
public final long p90Latency;
public final long p99Latency;
public final double p50Latency;
public final double p90Latency;
public final double p99Latency;
public final double stdDeviation;

public LatencyResults(
double avgLatency, long p50Latency, long p90Latency, long p99Latency, double stdDeviation) {
double avgLatency,
double p50Latency,
double p90Latency,
double p99Latency,
double stdDeviation) {
this.avgLatency = avgLatency;
this.p50Latency = p50Latency;
this.p90Latency = p90Latency;
Expand Down
13 changes: 13 additions & 0 deletions java/benchmarks/src/test/java/javababushka/benchmarks/jedis/JedisClientIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@

import static org.junit.jupiter.api.Assertions.assertTrue;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map;
import javababushka.benchmarks.utils.Benchmarking;
Expand Down Expand Up @@ -54,5 +55,17 @@ public void testResourceSetGet() {
actions.put(
ChosenAction.GET_NON_EXISTING, () -> jedisClient.get(Benchmarking.generateKeyGet()));
actions.put(ChosenAction.SET, () -> jedisClient.set(Benchmarking.generateKeySet(), value));

Map<ChosenAction, ArrayList<Long>> latencies =
Map.of(
ChosenAction.GET_EXISTING, new ArrayList<>(),
ChosenAction.GET_NON_EXISTING, new ArrayList<>(),
ChosenAction.SET, new ArrayList<>());
for (int i = 0; i < iterations; i++) {
var latency = Benchmarking.measurePerformance(actions);
latencies.get(latency.getKey()).add(latency.getValue());
}

Benchmarking.printResults(Benchmarking.calculateResults(latencies));
}
}

0 comments on commit c3d235a

Please sign in to comment.