diff --git a/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/patches/_resource_detector_patches.py b/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/patches/_resource_detector_patches.py index 60a231da8..3474e4120 100644 --- a/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/patches/_resource_detector_patches.py +++ b/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/patches/_resource_detector_patches.py @@ -17,7 +17,7 @@ def _apply_resource_detector_patches() -> None: def patch_ec2_aws_http_request(method, path, headers): with urlopen( Request("http://169.254.169.254" + path, headers=headers, method=method), - timeout=5, + timeout=0.005, ) as response: return response.read().decode("utf-8") diff --git a/performance-tests/k6/performanceTest.js b/performance-tests/k6/performanceTest.js index 3e45cc0af..579cb1a61 100644 --- a/performance-tests/k6/performanceTest.js +++ b/performance-tests/k6/performanceTest.js @@ -1,49 +1,9 @@ import http from "k6/http"; import {check} from "k6"; -const baseUri = 'http://vehicle-service:8001/vehicle-inventory/'; -// Fake ID to trigger 400's. -const badId = 1000; - export default function() { - /** - * Calls all Vehicle Inventory Service APIs. - * - * Note that there are no modifications to the database - all POSTs and DELETEs will fail. - * This ensures the database does not grow over time. Data is initially added in setUp.js. - */ - - invokeApi("oops", "GET", 404) - invokeApi("", "GET", 200) - invokeApi("", "POST", 400) - invokeApi("1", "GET", 200) - invokeApi(`${badId}`, "DELETE", 404) - invokeApi("make/Toyota", "GET", 200) - invokeApi("1/image", "GET", 200) - invokeApi("image/toy_rav_24.png", "GET", 200) - invokeApi("image/", "POST", 404) - invokeApi("history/", "GET", 200) - invokeApi("history/", "POST", 400) - invokeApi("history/1", "GET", 200) - invokeApi(`history/${badId}`, "DELETE", 404) - invokeApi("history/1/vehicle", "GET", 200) - - function invokeApi(path, method, status) { - const url = `${baseUri}${path}`; - let response; - switch(method) { - case "GET": - response = http.get(url); - break; - case "POST": - response = http.post(url, JSON.stringify({"badKey": "badValue"})); - break; - case "DELETE": - response = http.del(url); - break; - } - check(response, { - [`${method} ${path} response ${status}`] : (response) => response.status === status - }); - } + let response = http.get(`http://simple-service:8080/dep`); + check(response, { + [`GET dep response 200`] : (response) => response.status === 200 + }); }; \ No newline at end of file diff --git a/performance-tests/src/test/java/io/opentelemetry/OverheadTests.java b/performance-tests/src/test/java/io/opentelemetry/OverheadTests.java index 79d2d7dbe..71c88f481 100644 --- a/performance-tests/src/test/java/io/opentelemetry/OverheadTests.java +++ b/performance-tests/src/test/java/io/opentelemetry/OverheadTests.java @@ -11,11 +11,7 @@ import io.opentelemetry.config.Configs; import io.opentelemetry.config.TestConfig; -import io.opentelemetry.containers.CollectorContainer; -import io.opentelemetry.containers.ImageServiceContainer; -import io.opentelemetry.containers.K6Container; -import io.opentelemetry.containers.PostgresContainer; -import io.opentelemetry.containers.VehicleInventoryServiceContainer; +import io.opentelemetry.containers.*; import io.opentelemetry.distros.DistroConfig; import io.opentelemetry.results.AppPerfResults; import io.opentelemetry.results.MainResultsPersister; @@ -25,7 +21,6 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; -import java.time.Duration; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -38,9 +33,6 @@ import org.slf4j.LoggerFactory; import org.testcontainers.containers.GenericContainer; import org.testcontainers.containers.Network; -import org.testcontainers.containers.startupcheck.OneShotStartupCheckStrategy; -import org.testcontainers.utility.DockerImageName; -import org.testcontainers.utility.MountableFile; public class OverheadTests { @@ -94,28 +86,19 @@ void runTestConfig(TestConfig config) { } void runAppOnce(TestConfig config, DistroConfig distroConfig) throws Exception { - GenericContainer postgres = new PostgresContainer(NETWORK).build(); - postgres.start(); - - GenericContainer imageService = new ImageServiceContainer(NETWORK).build(); - imageService.start(); - - GenericContainer vehicleInventoryService = - new VehicleInventoryServiceContainer(NETWORK, collector, distroConfig, namingConventions) + GenericContainer dependency = + new SimpleServiceContainer(NETWORK, collector, DistroConfig.NONE, namingConventions, 8081, false) + .build(); + GenericContainer application = + new SimpleServiceContainer(NETWORK, collector, distroConfig, namingConventions, 8080, true) .build(); + dependency.start(); long start = System.currentTimeMillis(); - vehicleInventoryService.start(); + application.start(); writeStartupTimeFile(distroConfig, start); - populateDatabase(); - - int warmupSeconds = config.getWarmupSeconds(); - if (warmupSeconds > 0) { - doWarmupPhase(warmupSeconds); - } - long testStart = System.currentTimeMillis(); - startRecording(distroConfig, vehicleInventoryService); + startRecording(distroConfig, application); GenericContainer k6 = new K6Container(NETWORK, distroConfig, config, namingConventions).build(); @@ -124,49 +107,32 @@ void runAppOnce(TestConfig config, DistroConfig distroConfig) throws Exception { long runDuration = System.currentTimeMillis() - testStart; runDurations.put(distroConfig.getName(), runDuration); - vehicleInventoryService.stop(); - imageService.stop(); - postgres.stop(); + int counter = 0; + while(counter++ < 120) { + if (!"true".equals(System.getenv("PROFILE"))) { + break; + } else if (application.getLogs().contains("Wrote flamegraph data")) { + logger.info("Flamegraph done."); + break; + } else { + logger.info("Waiting for flamegraph."); + sleep(1); + } + } + + application.stop(); + dependency.stop(); } - private void startRecording( - DistroConfig distroConfig, GenericContainer vehicleInventoryService) throws Exception { + private void startRecording(DistroConfig distroConfig, GenericContainer service) + throws Exception { String[] command = { "sh", "executeProfiler.sh", namingConventions.container.performanceMetricsFileWithoutPath(distroConfig), namingConventions.container.root() }; - vehicleInventoryService.execInContainer(command); - } - - private void doWarmupPhase(int seconds) { - System.out.println("Performing warm up phase for " + seconds + " seconds."); - GenericContainer k6 = - new GenericContainer<>(DockerImageName.parse("loadimpact/k6")) - .withNetwork(NETWORK) - .withCopyFileToContainer(MountableFile.forHostPath("./k6"), "/app") - .withCommand("run", "-u", "5", "-d", seconds + "s", "/app/performanceTest.js") - .withStartupCheckStrategy( - new OneShotStartupCheckStrategy().withTimeout(Duration.ofSeconds(seconds * 2L))); - k6.start(); - sleep(seconds); - System.out.println("Awaiting warmup phase end."); - while (k6.isRunning()) { - System.out.println("Warmup still running."); - sleep(1); - } - System.out.println("Warmup complete."); - } - - private void populateDatabase() { - GenericContainer k6 = - new GenericContainer<>(DockerImageName.parse("loadimpact/k6")) - .withNetwork(NETWORK) - .withCopyFileToContainer(MountableFile.forHostPath("./k6"), "/app") - .withCommand("run", "/app/setUp.js") - .withStartupCheckStrategy(new OneShotStartupCheckStrategy()); - k6.start(); + service.execInContainer(command); } private void writeStartupTimeFile(DistroConfig distroConfig, long start) throws IOException { diff --git a/performance-tests/src/test/java/io/opentelemetry/config/Configs.java b/performance-tests/src/test/java/io/opentelemetry/config/Configs.java index b5e5d2be4..10a675716 100644 --- a/performance-tests/src/test/java/io/opentelemetry/config/Configs.java +++ b/performance-tests/src/test/java/io/opentelemetry/config/Configs.java @@ -12,26 +12,7 @@ /** Defines all test configurations */ public enum Configs { - ALL_100_TPS( - TestConfig.builder() - .name("all-100-tps") - .description("Compares all DistroConfigs (100TPS test)") - .withDistroConfigs(DistroConfig.values()) - .warmupSeconds(10) - .maxRequestRate(100) - .duration(System.getenv("DURATION")) - .concurrentConnections(System.getenv("CONCURRENCY")) - .build()), - ALL_800_TPS( - TestConfig.builder() - .name("all-800-tps") - .description("Compares all DistroConfigs (800TPS test)") - .withDistroConfigs(DistroConfig.values()) - .warmupSeconds(10) - .maxRequestRate(800) - .duration(System.getenv("DURATION")) - .concurrentConnections(System.getenv("CONCURRENCY")) - .build()); + ALL_TPS(buildConfig(System.getenv("TPS"))); public final TestConfig config; @@ -39,6 +20,18 @@ public static Stream all() { return Arrays.stream(Configs.values()).map(x -> x.config); } + private static TestConfig buildConfig(String tps) { + return TestConfig.builder() + .name(String.format("%s-tps", tps)) + .description(String.format("Compares all DistroConfigs (%sTPS test)", tps)) + .withDistroConfigs(DistroConfig.values()) + .warmupSeconds(10) + .maxRequestRate(tps) + .duration(System.getenv("DURATION") + "s") + .concurrentConnections(System.getenv("CONCURRENCY")) + .build(); + } + Configs(TestConfig config) { this.config = config; } diff --git a/performance-tests/src/test/java/io/opentelemetry/config/TestConfig.java b/performance-tests/src/test/java/io/opentelemetry/config/TestConfig.java index 9b4636d77..7bb2f6b37 100644 --- a/performance-tests/src/test/java/io/opentelemetry/config/TestConfig.java +++ b/performance-tests/src/test/java/io/opentelemetry/config/TestConfig.java @@ -93,8 +93,9 @@ Builder withDistroConfigs(DistroConfig... distroConfigs) { return this; } - Builder maxRequestRate(int maxRequestRate) { - this.maxRequestRate = maxRequestRate; + Builder maxRequestRate(String maxRequestRate) { + if (maxRequestRate != null && !maxRequestRate.isEmpty()) + this.maxRequestRate = Integer.parseInt(maxRequestRate); return this; } diff --git a/performance-tests/src/test/java/io/opentelemetry/containers/CollectorContainer.java b/performance-tests/src/test/java/io/opentelemetry/containers/CollectorContainer.java index 485f82055..f2eb48a29 100644 --- a/performance-tests/src/test/java/io/opentelemetry/containers/CollectorContainer.java +++ b/performance-tests/src/test/java/io/opentelemetry/containers/CollectorContainer.java @@ -18,7 +18,7 @@ public class CollectorContainer { - static final int COLLECTOR_PORT = 4317; + static final int COLLECTOR_PORT = 4318; static final int COLLECTOR_HEALTH_CHECK_PORT = 13133; private static final Logger logger = LoggerFactory.getLogger(CollectorContainer.class); diff --git a/performance-tests/src/test/java/io/opentelemetry/containers/K6Container.java b/performance-tests/src/test/java/io/opentelemetry/containers/K6Container.java index 29790f766..ed8336eca 100644 --- a/performance-tests/src/test/java/io/opentelemetry/containers/K6Container.java +++ b/performance-tests/src/test/java/io/opentelemetry/containers/K6Container.java @@ -59,11 +59,12 @@ public GenericContainer build() { "--summary-export", k6OutputFile.toString(), "--summary-trend-stats", - "avg,p(0),p(50),p(90),p(99),p(100),count", + "avg,p(0),p(50),p(90),p(91),p(92),p(93),p(94),p(95),p(96),p(97),p(98),p(99),p(99.9),p(100),count", "/app/performanceTest.js") .withCreateContainerCmdModifier( cmd -> cmd.getHostConfig().withCpusetCpus(RuntimeUtil.getNonApplicationCores())) .withStartupCheckStrategy( - new OneShotStartupCheckStrategy().withTimeout(Duration.ofMinutes(15))); + new OneShotStartupCheckStrategy() + .withTimeout(Duration.ofMinutes(90))); // set 90 mins (>1 hour) as timeout } } diff --git a/performance-tests/src/test/java/io/opentelemetry/containers/SimpleRequestsServiceContainer.java b/performance-tests/src/test/java/io/opentelemetry/containers/SimpleRequestsServiceContainer.java new file mode 100644 index 000000000..6a95a0fd0 --- /dev/null +++ b/performance-tests/src/test/java/io/opentelemetry/containers/SimpleRequestsServiceContainer.java @@ -0,0 +1,86 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + * Modifications Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + */ + +package io.opentelemetry.containers; + +import com.github.dockerjava.api.model.Capability; +import io.opentelemetry.distros.DistroConfig; +import io.opentelemetry.util.NamingConventions; +import io.opentelemetry.util.RuntimeUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.Network; +import org.testcontainers.containers.output.Slf4jLogConsumer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.lifecycle.Startable; +import org.testcontainers.utility.DockerImageName; +import org.testcontainers.utility.MountableFile; + +public class SimpleRequestsServiceContainer { + + private static final Logger logger = + LoggerFactory.getLogger(SimpleRequestsServiceContainer.class); + private static final int PORT = 8080; + + private final Network network; + private final Startable collector; + private final DistroConfig distroConfig; + private final NamingConventions namingConventions; + + public SimpleRequestsServiceContainer( + Network network, + Startable collector, + DistroConfig distroConfig, + NamingConventions namingConventions) { + this.network = network; + this.collector = collector; + this.distroConfig = distroConfig; + this.namingConventions = namingConventions; + } + + public GenericContainer build() { + GenericContainer container = + new GenericContainer<>(DockerImageName.parse(distroConfig.getImageName())) + .withNetwork(network) + .withNetworkAliases("requests-service", "backend") + .withLogConsumer(new Slf4jLogConsumer(logger)) + .withExposedPorts(PORT) + .waitingFor(Wait.forHttp("/health-check").forPort(PORT)) + .withFileSystemBind( + namingConventions.localResults(), namingConventions.containerResults()) + .withCopyFileToContainer( + MountableFile.forClasspathResource("run.sh"), "requests/run.sh") + .withCopyFileToContainer( + MountableFile.forClasspathResource("profiler.py"), "requests/profiler.py") + .withCopyFileToContainer( + MountableFile.forClasspathResource("executeProfiler.sh"), + "requests/executeProfiler.sh") + .withEnv(distroConfig.getAdditionalEnvVars()) + .withEnv("TEST_NAME", distroConfig.getName()) + .withEnv("PROFILE", System.getenv("PROFILE")) + .withEnv("DURATION", System.getenv("DURATION")) + .dependsOn(collector) + .withCreateContainerCmdModifier( + cmd -> cmd.getHostConfig() + .withCpusetCpus(RuntimeUtil.getApplicationCores()) + .withCapAdd(Capability.SYS_PTRACE)) + .withCommand("bash run.sh"); + + if (distroConfig.doInstrument()) { + container + .withEnv("DO_INSTRUMENT", "true") + .withEnv("OTEL_TRACES_EXPORTER", "otlp") + .withEnv("OTEL_EXPORTER_OTLP_PROTOCOL", "http/protobuf") + .withEnv("OTEL_METRICS_EXPORTER", "none") + .withEnv("OTEL_METRIC_EXPORT_INTERVAL", "60000") + .withEnv("OTEL_EXPORTER_OTLP_INSECURE", "true") + .withEnv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://collector:4318") + .withEnv("OTEL_RESOURCE_ATTRIBUTES", "service.name=requests_server"); + } + return container; + } +} diff --git a/performance-tests/src/test/java/io/opentelemetry/containers/SimpleServiceContainer.java b/performance-tests/src/test/java/io/opentelemetry/containers/SimpleServiceContainer.java new file mode 100644 index 000000000..ca1787788 --- /dev/null +++ b/performance-tests/src/test/java/io/opentelemetry/containers/SimpleServiceContainer.java @@ -0,0 +1,93 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + * Modifications Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + */ + +package io.opentelemetry.containers; + +import com.github.dockerjava.api.model.Capability; +import io.opentelemetry.distros.DistroConfig; +import io.opentelemetry.util.NamingConventions; +import io.opentelemetry.util.RuntimeUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.Network; +import org.testcontainers.containers.output.Slf4jLogConsumer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.lifecycle.Startable; +import org.testcontainers.utility.DockerImageName; +import org.testcontainers.utility.MountableFile; + +public class SimpleServiceContainer { + + private static final Logger logger = + LoggerFactory.getLogger(SimpleServiceContainer.class); + + private final Network network; + private final Startable collector; + private final DistroConfig distroConfig; + private final NamingConventions namingConventions; + private final int port; + private final boolean isApplication; + + public SimpleServiceContainer( + Network network, + Startable collector, + DistroConfig distroConfig, + NamingConventions namingConventions, + int port, + boolean isApplication) { + this.network = network; + this.collector = collector; + this.distroConfig = distroConfig; + this.namingConventions = namingConventions; + this.port = port; + this.isApplication = isApplication; + } + + public GenericContainer build() { + String cores = isApplication ? RuntimeUtil.getApplicationCores() : RuntimeUtil.getNonApplicationCores(); + GenericContainer container = + new GenericContainer<>(DockerImageName.parse(distroConfig.getImageName())) + .withNetwork(network) + .withNetworkAliases("simple-service", "backend") + .withLogConsumer(new Slf4jLogConsumer(logger)) + .withExposedPorts(port) + .waitingFor(Wait.forHttp("/health-check").forPort(port)) + .withFileSystemBind( + namingConventions.localResults(), namingConventions.containerResults()) + .withCopyFileToContainer( + MountableFile.forClasspathResource("run.sh"), "app/run.sh") + .withCopyFileToContainer( + MountableFile.forClasspathResource("profiler.py"), "app/profiler.py") + .withCopyFileToContainer( + MountableFile.forClasspathResource("executeProfiler.sh"), "app/executeProfiler.sh") + .withEnv(distroConfig.getAdditionalEnvVars()) + .withEnv("TEST_NAME", distroConfig.getName()) + .withEnv("PROFILE", System.getenv("PROFILE")) + .withEnv("DURATION", System.getenv("DURATION")) + .withEnv("LISTEN_ADDRESS", "0.0.0.0:" + port) + .dependsOn(collector) + .withCreateContainerCmdModifier( + cmd -> cmd.getHostConfig() + .withCpusetCpus(cores) + .withCapAdd(Capability.SYS_PTRACE)) + .withCommand("bash run.sh"); + + if (distroConfig.doInstrument()) { + container + .withEnv("DO_INSTRUMENT", "true") + .withEnv("OTEL_TRACES_EXPORTER", "otlp") + .withEnv("OTEL_EXPORTER_OTLP_PROTOCOL", "http/protobuf") + .withEnv("OTEL_METRICS_EXPORTER", "none") + .withEnv("OTEL_METRIC_EXPORT_INTERVAL", "60000") + .withEnv("OTEL_EXPORTER_OTLP_INSECURE", "true") + .withEnv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://collector:4318") + .withEnv("OTEL_PYTHON_DISABLED_INSTRUMENTATIONS", "flask") + .withEnv("OTEL_RESOURCE_ATTRIBUTES", "service.name=flask_server"); + } + return container; + } +} diff --git a/performance-tests/src/test/java/io/opentelemetry/distros/DistroConfig.java b/performance-tests/src/test/java/io/opentelemetry/distros/DistroConfig.java index d1daf38e5..792e77019 100644 --- a/performance-tests/src/test/java/io/opentelemetry/distros/DistroConfig.java +++ b/performance-tests/src/test/java/io/opentelemetry/distros/DistroConfig.java @@ -10,53 +10,65 @@ import java.util.Map; public enum DistroConfig { - NONE("none", "no distro at all", false, Collections.EMPTY_MAP), - APPLICATION_SIGNALS_DISABLED( - "application_signals_disabled", - "ADOT distro with Application Signals disabled", + NONE( + "no distro at all", + false, + Collections.EMPTY_MAP, + "performance-test/simple-service-adot"), + OTEL_100( + "OTEL distro with 100% sampling", true, - Map.of("OTEL_AWS_APPLICATION_SIGNALS_ENABLED", "false", "OTEL_TRACES_SAMPLER", "xray")), - APPLICATION_SIGNALS_NO_TRACES( - "application_signals_no_traces", - "ADOT distro with Application Signals enabled and no tracing", + Map.of("OTEL_TRACES_SAMPLER", "traceidratio", "OTEL_TRACES_SAMPLER_ARG", "1"), + "performance-test/simple-service-otel"), + ADOT_100( + "ADOT distro with Application Signals disabled, 100% sampling", true, Map.of( - "OTEL_AWS_APPLICATION_SIGNALS_ENABLED", - "true", - "OTEL_AWS_APPLICATION_SIGNALS_EXPORTER_ENDPOINT", - "http://collector:4317", "OTEL_TRACES_SAMPLER", - "always_off")), - APPLICATION_SIGNALS_TRACES( - "application_signals_traces", - "ADOT distro with Application Signals enabled and tracing", + "traceidratio", + "OTEL_TRACES_SAMPLER_ARG", + "1", + "OTEL_PYTHON_DISTRO", + "aws_distro", + "OTEL_PYTHON_CONFIGURATOR", + "aws_configurator"), + "performance-test/simple-service-adot"), + AS_100( + "ADOT distro with Application Signals enabled, 100% sampling", true, Map.of( + "OTEL_TRACES_SAMPLER", + "traceidratio", + "OTEL_TRACES_SAMPLER_ARG", + "1", + "OTEL_PYTHON_DISTRO", + "aws_distro", + "OTEL_PYTHON_CONFIGURATOR", + "aws_configurator", "OTEL_AWS_APPLICATION_SIGNALS_ENABLED", "true", "OTEL_AWS_APPLICATION_SIGNALS_EXPORTER_ENDPOINT", - "http://collector:4317", - "OTEL_TRACES_SAMPLER", - "xray")); + "http://collector:4318/v1/metrics"), + "performance-test/simple-service-adot"); - private final String name; private final String description; private final boolean doInstrument; private final Map additionalEnvVars; + private final String imageName; DistroConfig( - String name, String description, boolean doInstrument, - Map additionalEnvVars) { - this.name = name; + Map additionalEnvVars, + String imageName) { this.description = description; this.doInstrument = doInstrument; this.additionalEnvVars = additionalEnvVars; + this.imageName = imageName; } public String getName() { - return name; + return this.name(); } public String getDescription() { @@ -70,4 +82,8 @@ public boolean doInstrument() { public Map getAdditionalEnvVars() { return Collections.unmodifiableMap(additionalEnvVars); } + + public String getImageName() { + return imageName; + } } diff --git a/performance-tests/src/test/java/io/opentelemetry/results/AppPerfResults.java b/performance-tests/src/test/java/io/opentelemetry/results/AppPerfResults.java index 02c570633..6fefceed9 100644 --- a/performance-tests/src/test/java/io/opentelemetry/results/AppPerfResults.java +++ b/performance-tests/src/test/java/io/opentelemetry/results/AppPerfResults.java @@ -19,7 +19,16 @@ public class AppPerfResults { final double requestLatencyP0; final double requestLatencyP50; final double requestLatencyP90; + final double requestLatencyP91; + final double requestLatencyP92; + final double requestLatencyP93; + final double requestLatencyP94; + final double requestLatencyP95; + final double requestLatencyP96; + final double requestLatencyP97; + final double requestLatencyP98; final double requestLatencyP99; + final double requestLatencyP999; final double requestLatencyP100; final long networkBytesSentAvg; final long networkBytesSentP0; @@ -75,7 +84,16 @@ private AppPerfResults(Builder builder) { this.requestLatencyP0 = builder.requestLatencyP0; this.requestLatencyP50 = builder.requestLatencyP50; this.requestLatencyP90 = builder.requestLatencyP90; + this.requestLatencyP91 = builder.requestLatencyP91; + this.requestLatencyP92 = builder.requestLatencyP92; + this.requestLatencyP93 = builder.requestLatencyP93; + this.requestLatencyP94 = builder.requestLatencyP94; + this.requestLatencyP95 = builder.requestLatencyP95; + this.requestLatencyP96 = builder.requestLatencyP96; + this.requestLatencyP97 = builder.requestLatencyP97; + this.requestLatencyP98 = builder.requestLatencyP98; this.requestLatencyP99 = builder.requestLatencyP99; + this.requestLatencyP999 = builder.requestLatencyP999; this.requestLatencyP100 = builder.requestLatencyP100; this.networkBytesSentAvg = builder.networkBytesSentAvg; this.networkBytesSentP0 = builder.networkBytesSentP0; @@ -156,7 +174,16 @@ static class Builder { public double requestLatencyP0; public double requestLatencyP50; public double requestLatencyP90; + public double requestLatencyP91; + public double requestLatencyP92; + public double requestLatencyP93; + public double requestLatencyP94; + public double requestLatencyP95; + public double requestLatencyP96; + public double requestLatencyP97; + public double requestLatencyP98; public double requestLatencyP99; + public double requestLatencyP999; public double requestLatencyP100; public long networkBytesSentAvg; public long networkBytesSentP0; diff --git a/performance-tests/src/test/java/io/opentelemetry/results/CsvPersister.java b/performance-tests/src/test/java/io/opentelemetry/results/CsvPersister.java index abf8e5deb..66b9b0c97 100644 --- a/performance-tests/src/test/java/io/opentelemetry/results/CsvPersister.java +++ b/performance-tests/src/test/java/io/opentelemetry/results/CsvPersister.java @@ -27,7 +27,16 @@ class CsvPersister implements ResultsPersister { FieldSpec.of("requestLatencyP0", r -> r.requestLatencyP0), FieldSpec.of("requestLatencyP50", r -> r.requestLatencyP50), FieldSpec.of("requestLatencyP90", r -> r.requestLatencyP90), + FieldSpec.of("requestLatencyP91", r -> r.requestLatencyP91), + FieldSpec.of("requestLatencyP92", r -> r.requestLatencyP92), + FieldSpec.of("requestLatencyP93", r -> r.requestLatencyP93), + FieldSpec.of("requestLatencyP94", r -> r.requestLatencyP94), + FieldSpec.of("requestLatencyP95", r -> r.requestLatencyP95), + FieldSpec.of("requestLatencyP96", r -> r.requestLatencyP96), + FieldSpec.of("requestLatencyP97", r -> r.requestLatencyP97), + FieldSpec.of("requestLatencyP98", r -> r.requestLatencyP98), FieldSpec.of("requestLatencyP99", r -> r.requestLatencyP99), + FieldSpec.of("requestLatencyP99.9", r -> r.requestLatencyP999), FieldSpec.of("requestLatencyP100", r -> r.requestLatencyP100), FieldSpec.of("networkBytesSentAvg", r -> r.networkBytesSentAvg), FieldSpec.of("networkBytesSentP0", r -> r.networkBytesSentP0), diff --git a/performance-tests/src/test/java/io/opentelemetry/results/PrintStreamPersister.java b/performance-tests/src/test/java/io/opentelemetry/results/PrintStreamPersister.java index fed264943..f91112a3c 100644 --- a/performance-tests/src/test/java/io/opentelemetry/results/PrintStreamPersister.java +++ b/performance-tests/src/test/java/io/opentelemetry/results/PrintStreamPersister.java @@ -43,12 +43,21 @@ public void write(List results) { display(results, "Startup time (ms)", res -> String.valueOf(res.startupDurationMs)); display(results, "Req. Count", res -> format(res.requestCount)); display(results, "Req. Rate", res -> format(res.requestRate)); - display(results, "Req. Lat. mean (ms)", res -> format(res.requestLatencyAvg)); - display(results, "Req. Lat. p0 (ms)", res -> format(res.requestLatencyP0)); - display(results, "Req. Lat. p50 (ms)", res -> format(res.requestLatencyP50)); - display(results, "Req. Lat. p90 (ms)", res -> format(res.requestLatencyP90)); - display(results, "Req. Lat. p99 (ms)", res -> format(res.requestLatencyP99)); - display(results, "Req. Lat. p100 (ms)", res -> format(res.requestLatencyP100)); + display(results, "Req. Lat. mean (ms)", res -> format(res.requestLatencyAvg)); + display(results, "Req. Lat. p0 (ms)", res -> format(res.requestLatencyP0)); + display(results, "Req. Lat. p50 (ms)", res -> format(res.requestLatencyP50)); + display(results, "Req. Lat. p90 (ms)", res -> format(res.requestLatencyP90)); + display(results, "Req. Lat. p91 (ms)", res -> format(res.requestLatencyP91)); + display(results, "Req. Lat. p92 (ms)", res -> format(res.requestLatencyP92)); + display(results, "Req. Lat. p93 (ms)", res -> format(res.requestLatencyP93)); + display(results, "Req. Lat. p94 (ms)", res -> format(res.requestLatencyP94)); + display(results, "Req. Lat. p95 (ms)", res -> format(res.requestLatencyP95)); + display(results, "Req. Lat. p96 (ms)", res -> format(res.requestLatencyP96)); + display(results, "Req. Lat. p97 (ms)", res -> format(res.requestLatencyP97)); + display(results, "Req. Lat. p98 (ms)", res -> format(res.requestLatencyP98)); + display(results, "Req. Lat. p99 (ms)", res -> format(res.requestLatencyP99)); + display(results, "Req. Lat. p99.9 (ms)", res -> format(res.requestLatencyP999)); + display(results, "Req. Lat. p100 (ms)", res -> format(res.requestLatencyP100)); display(results, "Net Sent mean (B)", res -> format(res.networkBytesSentAvg)); display(results, "Net Sent p0 (B)", res -> format(res.networkBytesSentP0)); display(results, "Net Sent p50 (B)", res -> format(res.networkBytesSentP50)); diff --git a/performance-tests/src/test/java/io/opentelemetry/results/ResultsCollector.java b/performance-tests/src/test/java/io/opentelemetry/results/ResultsCollector.java index 6eb81a320..dc01cdbb0 100644 --- a/performance-tests/src/test/java/io/opentelemetry/results/ResultsCollector.java +++ b/performance-tests/src/test/java/io/opentelemetry/results/ResultsCollector.java @@ -71,7 +71,16 @@ private AppPerfResults.Builder addK6Results( double requestLatencyP0 = read(json, "$.metrics.http_req_duration['p(0)']"); double requestLatencyP50 = read(json, "$.metrics.http_req_duration['p(50)']"); double requestLatencyP90 = read(json, "$.metrics.http_req_duration['p(90)']"); + double requestLatencyP91 = read(json, "$.metrics.http_req_duration['p(91)']"); + double requestLatencyP92 = read(json, "$.metrics.http_req_duration['p(92)']"); + double requestLatencyP93 = read(json, "$.metrics.http_req_duration['p(93)']"); + double requestLatencyP94 = read(json, "$.metrics.http_req_duration['p(94)']"); + double requestLatencyP95 = read(json, "$.metrics.http_req_duration['p(95)']"); + double requestLatencyP96 = read(json, "$.metrics.http_req_duration['p(96)']"); + double requestLatencyP97 = read(json, "$.metrics.http_req_duration['p(97)']"); + double requestLatencyP98 = read(json, "$.metrics.http_req_duration['p(98)']"); double requestLatencyP99 = read(json, "$.metrics.http_req_duration['p(99)']"); + double requestLatencyP999 = read(json, "$.metrics.http_req_duration['p(99.9)']"); double requestLatencyP100 = read(json, "$.metrics.http_req_duration['p(100)']"); builder.requestCount = requestCount; builder.requestRate = requestRate; @@ -79,7 +88,16 @@ private AppPerfResults.Builder addK6Results( builder.requestLatencyP0 = requestLatencyP0; builder.requestLatencyP50 = requestLatencyP50; builder.requestLatencyP90 = requestLatencyP90; + builder.requestLatencyP91 = requestLatencyP91; + builder.requestLatencyP92 = requestLatencyP92; + builder.requestLatencyP93 = requestLatencyP93; + builder.requestLatencyP94 = requestLatencyP94; + builder.requestLatencyP95 = requestLatencyP95; + builder.requestLatencyP96 = requestLatencyP96; + builder.requestLatencyP97 = requestLatencyP97; + builder.requestLatencyP98 = requestLatencyP98; builder.requestLatencyP99 = requestLatencyP99; + builder.requestLatencyP999 = requestLatencyP999; builder.requestLatencyP100 = requestLatencyP100; return builder; } diff --git a/performance-tests/src/test/resources/collector.yaml b/performance-tests/src/test/resources/collector.yaml index 8111a2406..e0b2ab1e7 100644 --- a/performance-tests/src/test/resources/collector.yaml +++ b/performance-tests/src/test/resources/collector.yaml @@ -4,7 +4,7 @@ extensions: receivers: otlp: protocols: - grpc: + http: processors: batch: diff --git a/performance-tests/src/test/resources/run.sh b/performance-tests/src/test/resources/run.sh new file mode 100755 index 000000000..2b29b2bb4 --- /dev/null +++ b/performance-tests/src/test/resources/run.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Fail fast +set -e + +# If a distro is not provided, run service normally. If it is, run the service with instrumentation. +if [[ "${DO_INSTRUMENT}" == "true" ]]; then + opentelemetry-instrument gunicorn --config gunicorn_config.py app:app & + #opentelemetry-instrument python3 -u ./requests_server.py & +else + gunicorn --config gunicorn_config.py app:app & + #python3 -u ./requests_server.py & +fi + +if [[ "${PROFILE}" == "true" ]]; then + PID=$! + sleep 3 + py-spy record -d $DURATION -r 33 -o /results/profile-$TEST_NAME.svg --pid $PID +fi + +wait \ No newline at end of file diff --git a/performance-tests/src/test/resources/runVehicleInventory.sh b/performance-tests/src/test/resources/runVehicleInventory.sh deleted file mode 100755 index 37b222a40..000000000 --- a/performance-tests/src/test/resources/runVehicleInventory.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash -# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -# SPDX-License-Identifier: Apache-2.0 - -# Fail fast -set -e - -# Set up service -python3 manage.py migrate --noinput -python3 manage.py collectstatic --noinput - -# If a distro is not provided, run service normally. If it is, run the service with instrumentation. -if [[ -z "${DO_INSTRUMENT}" ]]; then - python3 manage.py runserver 0.0.0.0:$PORT --noreload -else - opentelemetry-instrument python3 manage.py runserver 0.0.0.0:$PORT --noreload -fi diff --git a/sample-applications/simple-flask-service/Dockerfile-ADOT b/sample-applications/simple-flask-service/Dockerfile-ADOT new file mode 100644 index 000000000..5c8a66075 --- /dev/null +++ b/sample-applications/simple-flask-service/Dockerfile-ADOT @@ -0,0 +1,18 @@ +# Meant to be run from aws-otel-python-instrumentation +# Assumes existence of dist/aws_opentelemetry_distro--py3-none-any.whl. +# Assumes filename of aws_opentelemetry_distro--py3-none-any.whl is passed in as "DISTRO" arg. +FROM python:3.10 +WORKDIR /app +COPY ./dist/$DISTRO /app +COPY ./sample-applications/simple-flask-service /app + +RUN rm -rf venv +ENV PIP_ROOT_USER_ACTION=ignore +ENV PYTHONDONTWRITEBYTECODE 1 +ENV PYTHONUNBUFFERED 1 + +EXPOSE 8080 8081 + +ARG DISTRO +RUN pip install --upgrade pip && pip install -r requirements.txt && pip install psutil py-spy && pip install ${DISTRO} --force-reinstall +RUN opentelemetry-bootstrap -a install \ No newline at end of file diff --git a/sample-applications/simple-flask-service/Dockerfile-OTEL b/sample-applications/simple-flask-service/Dockerfile-OTEL new file mode 100644 index 000000000..a60b8acc1 --- /dev/null +++ b/sample-applications/simple-flask-service/Dockerfile-OTEL @@ -0,0 +1,18 @@ +# Meant to be run from aws-otel-python-instrumentation +# Assumes existence of dist/aws_opentelemetry_distro--py3-none-any.whl. +# Assumes filename of aws_opentelemetry_distro--py3-none-any.whl is passed in as "DISTRO" arg. +FROM python:3.10 +WORKDIR /app +COPY ./sample-applications/simple-flask-service /app + +RUN rm -rf venv +ENV PIP_ROOT_USER_ACTION=ignore +ENV PYTHONDONTWRITEBYTECODE 1 +ENV PYTHONUNBUFFERED 1 + +EXPOSE 8080 8081 + +RUN pip install --upgrade pip && pip install -r requirements.txt && pip install psutil py-spy opentelemetry-distro --force-reinstall +RUN pip install --force-reinstall -v "opentelemetry-exporter-otlp==1.22.0" +RUN pip install --force-reinstall -v "opentelemetry-distro==0.43b0" +RUN opentelemetry-bootstrap -a install \ No newline at end of file diff --git a/sample-applications/simple-flask-service/app.py b/sample-applications/simple-flask-service/app.py new file mode 100644 index 000000000..67db684f9 --- /dev/null +++ b/sample-applications/simple-flask-service/app.py @@ -0,0 +1,32 @@ +import logging +import os +from flask import Flask +from requests import sessions + +application = app = Flask(__name__) + +# tying the logger with gunicorn logging +gunicorn_logger = logging.getLogger('gunicorn.error') +app.logger.handlers = gunicorn_logger.handlers +logging.getLogger().setLevel(logging.INFO) +session = sessions.Session() + +@app.route('/health-check') +def call_health_check(): + return 200 + + +@app.route('/dep') +def call_dep(): + session.request("GET", "http://simple-service:8081/health-check") + return 200 + + +if __name__ == "__main__": + address = os.environ.get('LISTEN_ADDRESS') + if address is None: + host = '127.0.0.1' + port = '5000' + else: + host, port = address.split(":") + app.run(host=host, port=int(port), debug=False) diff --git a/sample-applications/simple-flask-service/gunicorn_config.py b/sample-applications/simple-flask-service/gunicorn_config.py new file mode 100644 index 000000000..646b52c03 --- /dev/null +++ b/sample-applications/simple-flask-service/gunicorn_config.py @@ -0,0 +1,15 @@ +import os + +workers = int(os.environ.get('GUNICORN_PROCESSES', '4')) + +threads = int(os.environ.get('GUNICORN_THREADS', '6')) + +# timeout = int(os.environ.get('GUNICORN_TIMEOUT', '120')) + +address = os.environ.get('LISTEN_ADDRESS') + +bind = os.environ.get('GUNICORN_BIND', address) + +forwarded_allow_ips = '*' + +secure_scheme_headers = { 'X-Forwarded-Proto': 'https' } \ No newline at end of file diff --git a/sample-applications/simple-flask-service/requirements.txt b/sample-applications/simple-flask-service/requirements.txt new file mode 100644 index 000000000..f26def7e7 --- /dev/null +++ b/sample-applications/simple-flask-service/requirements.txt @@ -0,0 +1,3 @@ +Flask==2.3.3 +gunicorn==22.0.0 +requests==2.31.0 \ No newline at end of file diff --git a/sample-applications/simple-requests-service/Dockerfile-ADOT b/sample-applications/simple-requests-service/Dockerfile-ADOT new file mode 100644 index 000000000..fe58b910c --- /dev/null +++ b/sample-applications/simple-requests-service/Dockerfile-ADOT @@ -0,0 +1,14 @@ +# Meant to be run from aws-otel-python-instrumentation +# Assumes existence of dist/aws_opentelemetry_distro--py3-none-any.whl. +# Assumes filename of aws_opentelemetry_distro--py3-none-any.whl is passed in as "DISTRO" arg. +FROM python:3.10 +WORKDIR /requests +COPY ./dist/$DISTRO /requests +COPY ./sample-applications/simple-requests-service /requests + +ENV PIP_ROOT_USER_ACTION=ignore +ENV PYTHONDONTWRITEBYTECODE 1 +ENV PYTHONUNBUFFERED 1 +ARG DISTRO +RUN pip install --upgrade pip && pip install -r requirements.txt && pip install psutil py-spy && pip install ${DISTRO} --force-reinstall +RUN opentelemetry-bootstrap -a install \ No newline at end of file diff --git a/sample-applications/simple-requests-service/Dockerfile-OTEL b/sample-applications/simple-requests-service/Dockerfile-OTEL new file mode 100644 index 000000000..9fb66fd58 --- /dev/null +++ b/sample-applications/simple-requests-service/Dockerfile-OTEL @@ -0,0 +1,12 @@ +# Meant to be run from aws-otel-python-instrumentation +# Assumes existence of dist/aws_opentelemetry_distro--py3-none-any.whl. +# Assumes filename of aws_opentelemetry_distro--py3-none-any.whl is passed in as "DISTRO" arg. +FROM python:3.10 +WORKDIR /requests +COPY ./sample-applications/simple-requests-service /requests + +ENV PIP_ROOT_USER_ACTION=ignore +ENV PYTHONDONTWRITEBYTECODE 1 +ENV PYTHONUNBUFFERED 1 +RUN pip install --upgrade pip && pip install -r requirements.txt && pip install psutil py-spy opentelemetry-distro opentelemetry-exporter-otlp --force-reinstall +RUN opentelemetry-bootstrap -a install \ No newline at end of file diff --git a/sample-applications/simple-requests-service/pyproject.toml b/sample-applications/simple-requests-service/pyproject.toml new file mode 100644 index 000000000..7e010fb84 --- /dev/null +++ b/sample-applications/simple-requests-service/pyproject.toml @@ -0,0 +1,6 @@ +[project] +name = "requests-server" +description = "Simple server that relies on requests library" +version = "1.0.0" +license = "Apache-2.0" +requires-python = ">=3.8" \ No newline at end of file diff --git a/sample-applications/simple-requests-service/requests_server.py b/sample-applications/simple-requests-service/requests_server.py new file mode 100644 index 000000000..70b3bb3df --- /dev/null +++ b/sample-applications/simple-requests-service/requests_server.py @@ -0,0 +1,68 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +import atexit +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from threading import Thread + +from requests import Response, request +from typing_extensions import Tuple, override + +_PORT: int = 8080 +_NETWORK_ALIAS: str = "backend" +_SUCCESS: str = "success" +_ERROR: str = "error" +_FAULT: str = "fault" + + +class RequestHandler(BaseHTTPRequestHandler): + @override + # pylint: disable=invalid-name + def do_GET(self): + self.handle_request("GET") + + @override + # pylint: disable=invalid-name + def do_DELETE(self): + self.send_response_only(200) + self.end_headers() + + def handle_request(self, method: str): + status_code: int + if self.in_path("health-check"): + status_code = 200 + elif self.in_path(_NETWORK_ALIAS): + if self.in_path(_SUCCESS): + status_code = 200 + elif self.in_path(_ERROR): + status_code = 400 + elif self.in_path(_FAULT): + status_code = 500 + else: + status_code = 404 + else: + url: str = f"http://{_NETWORK_ALIAS}:{_PORT}/{_NETWORK_ALIAS}{self.path}" + try: + response: Response = request(method, url, timeout=0.01) + status_code = response.status_code + except: + status_code = 500 + self.send_response_only(status_code) + self.end_headers() + + def in_path(self, sub_path: str): + return sub_path in self.path + + +def main() -> None: + server_address: Tuple[str, int] = ("0.0.0.0", _PORT) + request_handler_class: type = RequestHandler + requests_server: ThreadingHTTPServer = ThreadingHTTPServer(server_address, request_handler_class) + atexit.register(requests_server.shutdown) + server_thread: Thread = Thread(target=requests_server.serve_forever) + server_thread.start() + print("Ready") + server_thread.join() + + +if __name__ == "__main__": + main() diff --git a/sample-applications/simple-requests-service/requirements.txt b/sample-applications/simple-requests-service/requirements.txt new file mode 100644 index 000000000..a3635989e --- /dev/null +++ b/sample-applications/simple-requests-service/requirements.txt @@ -0,0 +1,2 @@ +typing-extensions==4.9.0 +requests==2.31.0 \ No newline at end of file diff --git a/scripts/set-up-performance-tests.sh b/scripts/set-up-performance-tests.sh index 4df38fb14..e7024f4bc 100755 --- a/scripts/set-up-performance-tests.sh +++ b/scripts/set-up-performance-tests.sh @@ -13,14 +13,6 @@ if [ "$current_dir" != "aws-otel-python-instrumentation" ]; then exit fi -# Check for expected env variables -for var in AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN S3_BUCKET; do - if [[ -z "${!var}" ]]; then - echo "Variable $var not set, please ensure all of AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN, and S3_BUCKET are set." - exit 1 - fi -done - # Find and store aws_opentelemetry_distro whl file cd dist DISTRO=(aws_opentelemetry_distro-*-py3-none-any.whl) @@ -32,14 +24,14 @@ fi # Create application images # TODO: The vehicle sample app doesn't exist anymore so this needs to be cleaned up cd .. -docker build . -t performance-test/vehicle-inventory-service -f performance-tests/Dockerfile-VehicleInventoryService-base --build-arg="DISTRO=${DISTRO}" +docker build . -t performance-test/simple-service-adot -f sample-applications/simple-flask-service/Dockerfile-ADOT --build-arg="DISTRO=${DISTRO}" if [ $? = 1 ]; then - echo "Docker build for VehicleInventoryService failed" + echo "Docker build for simple-service-adot failed" exit 1 fi -docker build . -t performance-test/image-service -f performance-tests/Dockerfile-ImageService-base +docker build . -t performance-test/simple-service-otel -f sample-applications/simple-flask-service/Dockerfile-OTEL if [ $? = 1 ]; then - echo "Docker build for ImageService failed" + echo "Docker build for simple-service-otel failed" exit 1 fi \ No newline at end of file