Add json reporting.

benchmarks/utilities/csv_exporter.py

@@ -39,7 +39,7 @@
 
             json_file_name = os.path.basename(json_file_full_path)
 
-            languages = ["csharp", "node", "python", "rust"]
+            languages = ["csharp", "node", "python", "rust", "java"]
             language = next(
                 (language for language in languages if language in json_file_name), None
             )

java/benchmarks/build.gradle

@@ -18,6 +18,10 @@ dependencies {
     implementation 'io.lettuce:lettuce-core:6.2.6.RELEASE'
     implementation 'commons-cli:commons-cli:1.5.0'
     implementation group: 'org.apache.commons', name: 'commons-lang3', version: '3.13.0'
+    implementation group: 'com.google.code.gson', name: 'gson', version: '2.10.1'
+
+    compileOnly 'org.projectlombok:lombok:1.18.30'
+    annotationProcessor 'org.projectlombok:lombok:1.18.30'
 }
 
 // Apply a specific Java toolchain to ease working on different environments.

java/benchmarks/src/main/java/javababushka/benchmarks/BenchmarkingApp.java

@@ -2,8 +2,6 @@
 
 import static javababushka.benchmarks.utils.Benchmarking.testClientSetGet;
 
-import java.io.FileWriter;
-import java.io.IOException;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Optional;
@@ -57,14 +55,6 @@ public static void main(String[] args) {
           break;
       }
     }
-
-    if (runConfiguration.resultsFile.isPresent()) {
-      try {
-        runConfiguration.resultsFile.get().close();
-      } catch (IOException ioException) {
-        System.out.println("Error closing results file");
-      }
-    }
   }
 
   private static Options getOptions() {
@@ -97,12 +87,7 @@ private static RunConfiguration verifyOptions(CommandLine line) throws ParseExce
     }
 
     if (line.hasOption("resultsFile")) {
-      try {
-        runConfiguration.resultsFile =
-            Optional.of(new FileWriter(line.getOptionValue("resultsFile")));
-      } catch (IOException e) {
-        throw new ParseException("Unable to write to resultsFile.");
-      }
+      runConfiguration.resultsFile = Optional.ofNullable(line.getOptionValue("resultsFile"));
     }
 
     if (line.hasOption("dataSize")) {
@@ -210,8 +195,8 @@ public boolean isEqual(String other) {
 
   public static class RunConfiguration {
     public String configuration;
-    public Optional<FileWriter> resultsFile;
     public int dataSize;
+    public Optional<String> resultsFile;
     public List<Integer> concurrentTasks;
     public ClientName[] clients;
     public String host;

java/benchmarks/src/main/java/javababushka/benchmarks/utils/Benchmarking.java

@@ -1,18 +1,15 @@
 package javababushka.benchmarks.utils;
 
-import java.io.FileWriter;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
-import java.util.Optional;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.Supplier;
-import java.util.stream.Collectors;
 import javababushka.benchmarks.AsyncClient;
 import javababushka.benchmarks.BenchmarkingApp;
 import javababushka.benchmarks.Client;
@@ -26,6 +23,8 @@ public class Benchmarking {
   static final int SIZE_GET_KEYSPACE = 3750000;
   static final int SIZE_SET_KEYSPACE = 3000000;
   static final int ASYNC_OPERATION_TIMEOUT_SEC = 1;
+  // measurements are done in nano-seconds, but it should be converted to seconds later
+  static final double SECONDS_IN_NANO = 1e-9;
 
   private static ChosenAction randomAction() {
     if (Math.random() > PROB_GET) {
@@ -50,7 +49,7 @@ public interface Operation {
     void go() throws Exception;
   }
 
-  private static Pair<ChosenAction, Long> getLatency(Map<ChosenAction, Operation> actions) {
+  public static Pair<ChosenAction, Long> measurePerformance(Map<ChosenAction, Operation> actions) {
     var action = randomAction();
     long before = System.nanoTime();
     try {
@@ -84,56 +83,29 @@ private static double stdDeviation(ArrayList<Long> latencies, Double avgLatency)
   // This has the side-effect of sorting each latencies ArrayList
   public static Map<ChosenAction, LatencyResults> calculateResults(
       Map<ChosenAction, ArrayList<Long>> actionLatencies) {
-    Map<ChosenAction, LatencyResults> results = new HashMap<ChosenAction, LatencyResults>();
+    Map<ChosenAction, LatencyResults> results = new HashMap<>();
 
     for (Map.Entry<ChosenAction, ArrayList<Long>> entry : actionLatencies.entrySet()) {
       ChosenAction action = entry.getKey();
       ArrayList<Long> latencies = entry.getValue();
 
-      Double avgLatency =
-          latencies.stream().collect(Collectors.summingLong(Long::longValue))
-              / Double.valueOf(latencies.size());
+      double avgLatency =
+          SECONDS_IN_NANO * latencies.stream().mapToLong(Long::longValue).sum() / latencies.size();
 
       Collections.sort(latencies);
       results.put(
           action,
           new LatencyResults(
               avgLatency,
-              percentile(latencies, 50),
-              percentile(latencies, 90),
-              percentile(latencies, 99),
-              stdDeviation(latencies, avgLatency)));
+              SECONDS_IN_NANO * percentile(latencies, 50),
+              SECONDS_IN_NANO * percentile(latencies, 90),
+              SECONDS_IN_NANO * percentile(latencies, 99),
+              SECONDS_IN_NANO * stdDeviation(latencies, avgLatency)));
     }
 
     return results;
   }
 
-  public static void printResults(
-      Map<ChosenAction, LatencyResults> calculatedResults, Optional<FileWriter> resultsFile) {
-    if (resultsFile.isPresent()) {
-      printResults(calculatedResults, resultsFile.get());
-    } else {
-      printResults(calculatedResults);
-    }
-  }
-
-  public static void printResults(
-      Map<ChosenAction, LatencyResults> resultsMap, FileWriter resultsFile) {
-    for (Map.Entry<ChosenAction, LatencyResults> entry : resultsMap.entrySet()) {
-      ChosenAction action = entry.getKey();
-      LatencyResults results = entry.getValue();
-
-      try {
-        resultsFile.write("Avg. time in ms per " + action + ": " + results.avgLatency / 1000000.0);
-        resultsFile.write(action + " p50 latency in ms: " + results.p50Latency / 1000000.0);
-        resultsFile.write(action + " p90 latency in ms: " + results.p90Latency / 1000000.0);
-        resultsFile.write(action + " p99 latency in ms: " + results.p99Latency / 1000000.0);
-        resultsFile.write(action + " std dev in ms: " + results.stdDeviation / 1000000.0);
-      } catch (Exception ignored) {
-      }
-    }
-  }
-
   public static void printResults(Map<ChosenAction, LatencyResults> resultsMap) {
     for (Map.Entry<ChosenAction, LatencyResults> entry : resultsMap.entrySet()) {
       ChosenAction action = entry.getKey();
@@ -189,9 +161,10 @@ public static void testClientSetGet(
                         "> iteration = %d/%d, client# = %d/%d%n",
                         iterationIncrement + 1, iterations, clientIndex + 1, clientCount);
                   }
+
+                  var actions = getActionMap(clients.get(clientIndex), config.dataSize, async);
                   // operate and calculate tik-tok
-                  Pair<ChosenAction, Long> result =
-                      measurePerformance(clients.get(clientIndex), config.dataSize, async);
+                  Pair<ChosenAction, Long> result = measurePerformance(actions);
                   actionResults.get(result.getLeft()).add(result.getRight());
 
                   iterationIncrement = iterationCounter.getAndIncrement();
@@ -204,6 +177,7 @@ public static void testClientSetGet(
               "===> concurrentNum = %d, clientNum = %d, tasks = %d%n",
               concurrentNum, clientCount, tasks.size());
         }
+        long started = System.nanoTime();
         tasks.stream()
             .map(CompletableFuture::runAsync)
             .forEach(
@@ -215,14 +189,25 @@ public static void testClientSetGet(
                   }
                 });
 
-        printResults(calculateResults(actionResults), config.resultsFile);
+        var calculatedResults = calculateResults(actionResults);
+        if (config.resultsFile.isPresent()) {
+          JsonWriter.WriteJson(
+              calculatedResults,
+              config.resultsFile.get(),
+              config.dataSize,
+              clientCreator.get().getName(),
+              clientCount,
+              concurrentNum,
+              iterationCounter.get() * 1e9 / (System.nanoTime() - started));
+        }
+        printResults(calculatedResults);
       }
     }
 
     System.out.println();
   }
 
-  public static Pair<ChosenAction, Long> measurePerformance(
+  public static Map<ChosenAction, Operation> getActionMap(
       Client client, int dataSize, boolean async) {
 
     String value = RandomStringUtils.randomAlphanumeric(dataSize);
@@ -251,7 +236,6 @@ public static Pair<ChosenAction, Long> measurePerformance(
                     .asyncSet(generateKeySet(), value)
                     .get(ASYNC_OPERATION_TIMEOUT_SEC, TimeUnit.SECONDS)
             : () -> ((SyncClient) client).set(generateKeySet(), value));
-
-    return getLatency(actions);
+    return actions;
   }
 }

java/benchmarks/src/main/java/javababushka/benchmarks/utils/JsonWriter.java

@@ -0,0 +1,102 @@
+package javababushka.benchmarks.utils;
+
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.google.gson.reflect.TypeToken;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Map;
+import lombok.Getter;
+
+public class JsonWriter {
+
+  public static void WriteJson(
+      Map<ChosenAction, LatencyResults> calculatedResults,
+      String resultsFile,
+      int dataSize,
+      String client,
+      int clientCount,
+      int numOfTasks,
+      double tps) {
+
+    try {
+      Gson gson = new GsonBuilder().setPrettyPrinting().serializeNulls().create();
+      Collection<Measurements> recordings = new ArrayList<>();
+
+      Path path = Path.of(resultsFile);
+      if (Files.exists(path)) {
+        TypeToken<Collection<Measurements>> collectionType = new TypeToken<>() {};
+        var json = new String(Files.readAllBytes(path));
+        recordings = gson.fromJson(json, collectionType);
+      }
+      var data = new Measurements();
+      data.data_size = dataSize;
+      data.client = client;
+      data.client_count = clientCount;
+      data.num_of_tasks = numOfTasks;
+      data.tps = tps;
+      // TODO:  is_cluster
+
+      data.get_existing_average_latency =
+          calculatedResults.get(ChosenAction.GET_EXISTING).avgLatency;
+      data.get_existing_p50_latency = calculatedResults.get(ChosenAction.GET_EXISTING).p50Latency;
+      data.get_existing_p90_latency = calculatedResults.get(ChosenAction.GET_EXISTING).p90Latency;
+      data.get_existing_p99_latency = calculatedResults.get(ChosenAction.GET_EXISTING).p99Latency;
+      data.get_existing_std_dev = calculatedResults.get(ChosenAction.GET_EXISTING).stdDeviation;
+
+      data.get_non_existing_average_latency =
+          calculatedResults.get(ChosenAction.GET_NON_EXISTING).avgLatency;
+      data.get_non_existing_p50_latency =
+          calculatedResults.get(ChosenAction.GET_NON_EXISTING).p50Latency;
+      data.get_non_existing_p90_latency =
+          calculatedResults.get(ChosenAction.GET_NON_EXISTING).p90Latency;
+      data.get_non_existing_p99_latency =
+          calculatedResults.get(ChosenAction.GET_NON_EXISTING).p99Latency;
+      data.get_non_existing_std_dev =
+          calculatedResults.get(ChosenAction.GET_NON_EXISTING).stdDeviation;
+
+      data.set_average_latency = calculatedResults.get(ChosenAction.SET).avgLatency;
+      data.set_p50_latency = calculatedResults.get(ChosenAction.SET).p50Latency;
+      data.set_p90_latency = calculatedResults.get(ChosenAction.SET).p90Latency;
+      data.set_p99_latency = calculatedResults.get(ChosenAction.SET).p99Latency;
+      data.set_std_dev = calculatedResults.get(ChosenAction.SET).stdDeviation;
+
+      recordings.add(data);
+
+      Files.write(path, gson.toJson(recordings).getBytes());
+    } catch (IOException e) {
+      System.out.printf(
+          "Failed to write measurement results into a file '%s': %s%n",
+          resultsFile, e.getMessage());
+      e.printStackTrace();
+    }
+  }
+
+  @Getter
+  public static class Measurements {
+    private String client;
+    private int client_count;
+    private int data_size;
+    private double get_existing_average_latency;
+    private double get_existing_p50_latency;
+    private double get_existing_p90_latency;
+    private double get_existing_p99_latency;
+    private double get_existing_std_dev;
+    private double get_non_existing_average_latency;
+    private double get_non_existing_p50_latency;
+    private double get_non_existing_p90_latency;
+    private double get_non_existing_p99_latency;
+    private double get_non_existing_std_dev;
+    private boolean is_cluster;
+    private int num_of_tasks;
+    private double set_average_latency;
+    private double set_p50_latency;
+    private double set_p90_latency;
+    private double set_p99_latency;
+    private double set_std_dev;
+    private double tps;
+  }
+}

java/benchmarks/src/main/java/javababushka/benchmarks/utils/LatencyResults.java

@@ -3,13 +3,17 @@
 // Raw timing results in nanoseconds
 public class LatencyResults {
   public final double avgLatency;
-  public final long p50Latency;
-  public final long p90Latency;
-  public final long p99Latency;
+  public final double p50Latency;
+  public final double p90Latency;
+  public final double p99Latency;
   public final double stdDeviation;
 
   public LatencyResults(
-      double avgLatency, long p50Latency, long p90Latency, long p99Latency, double stdDeviation) {
+      double avgLatency,
+      double p50Latency,
+      double p90Latency,
+      double p99Latency,
+      double stdDeviation) {
     this.avgLatency = avgLatency;
     this.p50Latency = p50Latency;
     this.p90Latency = p90Latency;

java/benchmarks/src/test/java/javababushka/benchmarks/jedis/JedisClientIT.java

@@ -5,6 +5,7 @@
 
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
+import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.Map;
 import javababushka.benchmarks.utils.Benchmarking;
@@ -54,5 +55,17 @@ public void testResourceSetGet() {
     actions.put(
         ChosenAction.GET_NON_EXISTING, () -> jedisClient.get(Benchmarking.generateKeyGet()));
     actions.put(ChosenAction.SET, () -> jedisClient.set(Benchmarking.generateKeySet(), value));
+
+    Map<ChosenAction, ArrayList<Long>> latencies =
+        Map.of(
+            ChosenAction.GET_EXISTING, new ArrayList<>(),
+            ChosenAction.GET_NON_EXISTING, new ArrayList<>(),
+            ChosenAction.SET, new ArrayList<>());
+    for (int i = 0; i < iterations; i++) {
+      var latency = Benchmarking.measurePerformance(actions);
+      latencies.get(latency.getKey()).add(latency.getValue());
+    }
+
+    Benchmarking.printResults(Benchmarking.calculateResults(latencies));
   }
 }