Perf requests

aws-opentelemetry-distro/src/amazon/opentelemetry/distro/patches/_resource_detector_patches.py

@@ -17,7 +17,7 @@ def _apply_resource_detector_patches() -> None:
     def patch_ec2_aws_http_request(method, path, headers):
         with urlopen(
             Request("http://169.254.169.254" + path, headers=headers, method=method),
-            timeout=5,
+            timeout=0.005,
         ) as response:
             return response.read().decode("utf-8")
 

performance-tests/k6/performanceTest.js

@@ -1,49 +1,9 @@
 import http from "k6/http";
 import {check} from "k6";
 
-const baseUri = 'http://vehicle-service:8001/vehicle-inventory/';
-// Fake ID to trigger 400's.
-const badId = 1000;
-
 export default function() {
-    /**
-     * Calls all Vehicle Inventory Service APIs.
-     *
-     * Note that there are no modifications to the database - all POSTs and DELETEs will fail.
-     * This ensures the database does not grow over time. Data is initially added in setUp.js.
-     */
-
-    invokeApi("oops", "GET", 404)
-    invokeApi("", "GET", 200)
-    invokeApi("", "POST", 400)
-    invokeApi("1", "GET", 200)
-    invokeApi(`${badId}`, "DELETE", 404)
-    invokeApi("make/Toyota", "GET", 200)
-    invokeApi("1/image", "GET", 200)
-    invokeApi("image/toy_rav_24.png", "GET", 200)
-    invokeApi("image/", "POST", 404)
-    invokeApi("history/", "GET", 200)
-    invokeApi("history/", "POST", 400)
-    invokeApi("history/1", "GET", 200)
-    invokeApi(`history/${badId}`, "DELETE", 404)
-    invokeApi("history/1/vehicle", "GET", 200)
-
-    function invokeApi(path, method, status) {
-        const url = `${baseUri}${path}`;
-        let response;
-        switch(method) {
-            case "GET":
-                response = http.get(url);
-                break;
-            case "POST":
-                response = http.post(url, JSON.stringify({"badKey": "badValue"}));
-                break;
-            case "DELETE":
-                response = http.del(url);
-                break;
-        }
-        check(response, {
-            [`${method} ${path} response ${status}`] : (response) => response.status === status
-        });
-    }
+    let response = http.get(`http://simple-service:8080/dep`);
+    check(response, {
+        [`GET dep response 200`] : (response) => response.status === 200
+    });
 };

performance-tests/src/test/java/io/opentelemetry/OverheadTests.java

@@ -11,11 +11,7 @@
 
 import io.opentelemetry.config.Configs;
 import io.opentelemetry.config.TestConfig;
-import io.opentelemetry.containers.CollectorContainer;
-import io.opentelemetry.containers.ImageServiceContainer;
-import io.opentelemetry.containers.K6Container;
-import io.opentelemetry.containers.PostgresContainer;
-import io.opentelemetry.containers.VehicleInventoryServiceContainer;
+import io.opentelemetry.containers.*;
 import io.opentelemetry.distros.DistroConfig;
 import io.opentelemetry.results.AppPerfResults;
 import io.opentelemetry.results.MainResultsPersister;
@@ -25,7 +21,6 @@
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.time.Duration;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -38,9 +33,6 @@
 import org.slf4j.LoggerFactory;
 import org.testcontainers.containers.GenericContainer;
 import org.testcontainers.containers.Network;
-import org.testcontainers.containers.startupcheck.OneShotStartupCheckStrategy;
-import org.testcontainers.utility.DockerImageName;
-import org.testcontainers.utility.MountableFile;
 
 public class OverheadTests {
 
@@ -94,28 +86,19 @@ void runTestConfig(TestConfig config) {
   }
 
   void runAppOnce(TestConfig config, DistroConfig distroConfig) throws Exception {
-    GenericContainer<?> postgres = new PostgresContainer(NETWORK).build();
-    postgres.start();
-
-    GenericContainer<?> imageService = new ImageServiceContainer(NETWORK).build();
-    imageService.start();
-
-    GenericContainer<?> vehicleInventoryService =
-        new VehicleInventoryServiceContainer(NETWORK, collector, distroConfig, namingConventions)
+    GenericContainer<?> dependency =
+        new SimpleServiceContainer(NETWORK, collector, DistroConfig.NONE, namingConventions, 8081, false)
+            .build();
+    GenericContainer<?> application =
+        new SimpleServiceContainer(NETWORK, collector, distroConfig, namingConventions, 8080, true)
             .build();
+    dependency.start();
     long start = System.currentTimeMillis();
-    vehicleInventoryService.start();
+    application.start();
     writeStartupTimeFile(distroConfig, start);
 
-    populateDatabase();
-
-    int warmupSeconds = config.getWarmupSeconds();
-    if (warmupSeconds > 0) {
-      doWarmupPhase(warmupSeconds);
-    }
-
     long testStart = System.currentTimeMillis();
-    startRecording(distroConfig, vehicleInventoryService);
+    startRecording(distroConfig, application);
 
     GenericContainer<?> k6 =
         new K6Container(NETWORK, distroConfig, config, namingConventions).build();
@@ -124,49 +107,32 @@ void runAppOnce(TestConfig config, DistroConfig distroConfig) throws Exception {
     long runDuration = System.currentTimeMillis() - testStart;
     runDurations.put(distroConfig.getName(), runDuration);
 
-    vehicleInventoryService.stop();
-    imageService.stop();
-    postgres.stop();
+    int counter = 0;
+    while(counter++ < 120) {
+      if (!"true".equals(System.getenv("PROFILE"))) {
+        break;
+      } else if (application.getLogs().contains("Wrote flamegraph data")) {
+        logger.info("Flamegraph done.");
+        break;
+      } else {
+          logger.info("Waiting for flamegraph.");
+          sleep(1);
+      }
+    }
+
+    application.stop();
+    dependency.stop();
   }
 
-  private void startRecording(
-      DistroConfig distroConfig, GenericContainer<?> vehicleInventoryService) throws Exception {
+  private void startRecording(DistroConfig distroConfig, GenericContainer<?> service)
+      throws Exception {
     String[] command = {
       "sh",
       "executeProfiler.sh",
       namingConventions.container.performanceMetricsFileWithoutPath(distroConfig),
       namingConventions.container.root()
     };
-    vehicleInventoryService.execInContainer(command);
-  }
-
-  private void doWarmupPhase(int seconds) {
-    System.out.println("Performing warm up phase for " + seconds + " seconds.");
-    GenericContainer<?> k6 =
-        new GenericContainer<>(DockerImageName.parse("loadimpact/k6"))
-            .withNetwork(NETWORK)
-            .withCopyFileToContainer(MountableFile.forHostPath("./k6"), "/app")
-            .withCommand("run", "-u", "5", "-d", seconds + "s", "/app/performanceTest.js")
-            .withStartupCheckStrategy(
-                new OneShotStartupCheckStrategy().withTimeout(Duration.ofSeconds(seconds * 2L)));
-    k6.start();
-    sleep(seconds);
-    System.out.println("Awaiting warmup phase end.");
-    while (k6.isRunning()) {
-      System.out.println("Warmup still running.");
-      sleep(1);
-    }
-    System.out.println("Warmup complete.");
-  }
-
-  private void populateDatabase() {
-    GenericContainer<?> k6 =
-        new GenericContainer<>(DockerImageName.parse("loadimpact/k6"))
-            .withNetwork(NETWORK)
-            .withCopyFileToContainer(MountableFile.forHostPath("./k6"), "/app")
-            .withCommand("run", "/app/setUp.js")
-            .withStartupCheckStrategy(new OneShotStartupCheckStrategy());
-    k6.start();
+    service.execInContainer(command);
   }
 
   private void writeStartupTimeFile(DistroConfig distroConfig, long start) throws IOException {

performance-tests/src/test/java/io/opentelemetry/config/Configs.java

@@ -12,33 +12,26 @@
 
 /** Defines all test configurations */
 public enum Configs {
-  ALL_100_TPS(
-      TestConfig.builder()
-          .name("all-100-tps")
-          .description("Compares all DistroConfigs (100TPS test)")
-          .withDistroConfigs(DistroConfig.values())
-          .warmupSeconds(10)
-          .maxRequestRate(100)
-          .duration(System.getenv("DURATION"))
-          .concurrentConnections(System.getenv("CONCURRENCY"))
-          .build()),
-  ALL_800_TPS(
-      TestConfig.builder()
-          .name("all-800-tps")
-          .description("Compares all DistroConfigs (800TPS test)")
-          .withDistroConfigs(DistroConfig.values())
-          .warmupSeconds(10)
-          .maxRequestRate(800)
-          .duration(System.getenv("DURATION"))
-          .concurrentConnections(System.getenv("CONCURRENCY"))
-          .build());
+  ALL_TPS(buildConfig(System.getenv("TPS")));
 
   public final TestConfig config;
 
   public static Stream<TestConfig> all() {
     return Arrays.stream(Configs.values()).map(x -> x.config);
   }
 
+  private static TestConfig buildConfig(String tps) {
+    return TestConfig.builder()
+        .name(String.format("%s-tps", tps))
+        .description(String.format("Compares all DistroConfigs (%sTPS test)", tps))
+        .withDistroConfigs(DistroConfig.values())
+        .warmupSeconds(10)
+        .maxRequestRate(tps)
+        .duration(System.getenv("DURATION") + "s")
+        .concurrentConnections(System.getenv("CONCURRENCY"))
+        .build();
+  }
+
   Configs(TestConfig config) {
     this.config = config;
   }

performance-tests/src/test/java/io/opentelemetry/config/TestConfig.java

@@ -93,8 +93,9 @@ Builder withDistroConfigs(DistroConfig... distroConfigs) {
       return this;
     }
 
-    Builder maxRequestRate(int maxRequestRate) {
-      this.maxRequestRate = maxRequestRate;
+    Builder maxRequestRate(String maxRequestRate) {
+      if (maxRequestRate != null && !maxRequestRate.isEmpty())
+        this.maxRequestRate = Integer.parseInt(maxRequestRate);
       return this;
     }
 

performance-tests/src/test/java/io/opentelemetry/containers/CollectorContainer.java

@@ -18,7 +18,7 @@
 
 public class CollectorContainer {
 
-  static final int COLLECTOR_PORT = 4317;
+  static final int COLLECTOR_PORT = 4318;
   static final int COLLECTOR_HEALTH_CHECK_PORT = 13133;
 
   private static final Logger logger = LoggerFactory.getLogger(CollectorContainer.class);

performance-tests/src/test/java/io/opentelemetry/containers/K6Container.java

@@ -59,11 +59,12 @@ public GenericContainer<?> build() {
             "--summary-export",
             k6OutputFile.toString(),
             "--summary-trend-stats",
-            "avg,p(0),p(50),p(90),p(99),p(100),count",
+            "avg,p(0),p(50),p(90),p(91),p(92),p(93),p(94),p(95),p(96),p(97),p(98),p(99),p(99.9),p(100),count",
             "/app/performanceTest.js")
         .withCreateContainerCmdModifier(
             cmd -> cmd.getHostConfig().withCpusetCpus(RuntimeUtil.getNonApplicationCores()))
         .withStartupCheckStrategy(
-            new OneShotStartupCheckStrategy().withTimeout(Duration.ofMinutes(15)));
+            new OneShotStartupCheckStrategy()
+                .withTimeout(Duration.ofMinutes(90))); // set 90 mins (>1 hour) as timeout
   }
 }

performance-tests/src/test/java/io/opentelemetry/containers/SimpleRequestsServiceContainer.java

@@ -0,0 +1,86 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ * SPDX-License-Identifier: Apache-2.0
+ * Modifications Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ */
+
+package io.opentelemetry.containers;
+
+import com.github.dockerjava.api.model.Capability;
+import io.opentelemetry.distros.DistroConfig;
+import io.opentelemetry.util.NamingConventions;
+import io.opentelemetry.util.RuntimeUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testcontainers.containers.GenericContainer;
+import org.testcontainers.containers.Network;
+import org.testcontainers.containers.output.Slf4jLogConsumer;
+import org.testcontainers.containers.wait.strategy.Wait;
+import org.testcontainers.lifecycle.Startable;
+import org.testcontainers.utility.DockerImageName;
+import org.testcontainers.utility.MountableFile;
+
+public class SimpleRequestsServiceContainer {
+
+  private static final Logger logger =
+      LoggerFactory.getLogger(SimpleRequestsServiceContainer.class);
+  private static final int PORT = 8080;
+
+  private final Network network;
+  private final Startable collector;
+  private final DistroConfig distroConfig;
+  private final NamingConventions namingConventions;
+
+  public SimpleRequestsServiceContainer(
+      Network network,
+      Startable collector,
+      DistroConfig distroConfig,
+      NamingConventions namingConventions) {
+    this.network = network;
+    this.collector = collector;
+    this.distroConfig = distroConfig;
+    this.namingConventions = namingConventions;
+  }
+
+  public GenericContainer<?> build() {
+    GenericContainer<?> container =
+        new GenericContainer<>(DockerImageName.parse(distroConfig.getImageName()))
+            .withNetwork(network)
+            .withNetworkAliases("requests-service", "backend")
+            .withLogConsumer(new Slf4jLogConsumer(logger))
+            .withExposedPorts(PORT)
+            .waitingFor(Wait.forHttp("/health-check").forPort(PORT))
+            .withFileSystemBind(
+                namingConventions.localResults(), namingConventions.containerResults())
+            .withCopyFileToContainer(
+                MountableFile.forClasspathResource("run.sh"), "requests/run.sh")
+            .withCopyFileToContainer(
+                MountableFile.forClasspathResource("profiler.py"), "requests/profiler.py")
+            .withCopyFileToContainer(
+                MountableFile.forClasspathResource("executeProfiler.sh"),
+                "requests/executeProfiler.sh")
+            .withEnv(distroConfig.getAdditionalEnvVars())
+            .withEnv("TEST_NAME", distroConfig.getName())
+            .withEnv("PROFILE", System.getenv("PROFILE"))
+            .withEnv("DURATION", System.getenv("DURATION"))
+            .dependsOn(collector)
+            .withCreateContainerCmdModifier(
+                cmd -> cmd.getHostConfig()
+                    .withCpusetCpus(RuntimeUtil.getApplicationCores())
+                    .withCapAdd(Capability.SYS_PTRACE))
+            .withCommand("bash run.sh");
+
+    if (distroConfig.doInstrument()) {
+      container
+          .withEnv("DO_INSTRUMENT", "true")
+          .withEnv("OTEL_TRACES_EXPORTER", "otlp")
+          .withEnv("OTEL_EXPORTER_OTLP_PROTOCOL", "http/protobuf")
+          .withEnv("OTEL_METRICS_EXPORTER", "none")
+          .withEnv("OTEL_METRIC_EXPORT_INTERVAL", "60000")
+          .withEnv("OTEL_EXPORTER_OTLP_INSECURE", "true")
+          .withEnv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://collector:4318")
+          .withEnv("OTEL_RESOURCE_ATTRIBUTES", "service.name=requests_server");
+    }
+    return container;
+  }
+}