Merge branch 'dev' into release-13.x (#877)

wavefrontHQ · Oct 18, 2023 · 231254d · 231254d
2 parents f7b70e7 + 3e17868
commit 231254d
Show file tree

Hide file tree

Showing 7 changed files with 119 additions and 13 deletions.
diff --git a/docker/Dockerfile b/docker/Dockerfile
@@ -25,6 +25,9 @@ RUN chown -R wavefront:wavefront /var/spool/wavefront-proxy
 RUN mkdir -p /var/log/wavefront
 RUN chown -R wavefront:wavefront /var/log/wavefront
 
+# Temp fix for "MONIT-41551"
+RUN apt-get remove -y wget
+
 # Run the agent
 EXPOSE 3878
 EXPOSE 2878

diff --git a/proxy/pom.xml b/proxy/pom.xml
@@ -284,7 +284,7 @@
       <dependency>
         <groupId>io.netty</groupId>
         <artifactId>netty-bom</artifactId>
-        <version>4.1.92.Final</version>
+        <version>4.1.100.Final</version>
         <type>pom</type>
         <scope>import</scope>
       </dependency>
@@ -369,7 +369,7 @@
       <dependency>
         <groupId>org.apache.avro</groupId>
         <artifactId>avro</artifactId>
-        <version>1.11.0</version>
+        <version>1.11.3</version>
       </dependency>
       <dependency>
         <groupId>io.opentelemetry</groupId>

diff --git a/proxy/src/main/java/com/wavefront/agent/listeners/DataDogPortUnificationHandler.java b/proxy/src/main/java/com/wavefront/agent/listeners/DataDogPortUnificationHandler.java
@@ -49,10 +49,10 @@
 import io.netty.channel.ChannelHandlerContext;
 import io.netty.handler.codec.http.FullHttpRequest;
 import io.netty.handler.codec.http.HttpResponseStatus;
-import io.netty.util.CharsetUtil;
 import java.io.IOException;
 import java.net.URI;
 import java.net.URISyntaxException;
+import java.nio.charset.StandardCharsets;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.concurrent.ScheduledThreadPoolExecutor;
@@ -67,7 +67,7 @@
 import org.apache.http.HttpResponse;
 import org.apache.http.client.HttpClient;
 import org.apache.http.client.methods.HttpPost;
-import org.apache.http.entity.StringEntity;
+import org.apache.http.entity.ByteArrayEntity;
 import org.apache.http.util.EntityUtils;
 import wavefront.report.ReportPoint;
 
@@ -216,7 +216,8 @@ protected void handleHttpMessage(final ChannelHandlerContext ctx, final FullHttp
     AtomicInteger pointsPerRequest = new AtomicInteger();
     URI uri = new URI(request.uri());
     HttpResponseStatus status = HttpResponseStatus.ACCEPTED;
-    String requestBody = request.content().toString(CharsetUtil.UTF_8);
+    byte[] bodyBytes = new byte[request.content().readableBytes()];
+    request.content().readBytes(bodyBytes);
 
     if (requestRelayClient != null && requestRelayTarget != null && request.method() == POST) {
       Histogram requestRelayDuration =
@@ -226,10 +227,16 @@ protected void handleHttpMessage(final ChannelHandlerContext ctx, final FullHttp
       try {
         String outgoingUrl = requestRelayTarget.replaceFirst("/*$", "") + request.uri();
         HttpPost outgoingRequest = new HttpPost(outgoingUrl);
-        if (request.headers().contains("Content-Type")) {
-          outgoingRequest.addHeader("Content-Type", request.headers().get("Content-Type"));
-        }
-        outgoingRequest.setEntity(new StringEntity(requestBody));
+
+        request
+            .headers()
+            .forEach(
+                header -> {
+                  if (!header.getKey().equalsIgnoreCase("Content-Length"))
+                    outgoingRequest.addHeader(header.getKey(), header.getValue());
+                });
+
+        outgoingRequest.setEntity(new ByteArrayEntity(bodyBytes));
         if (synchronousMode) {
           if (logger.isLoggable(Level.FINE)) {
             logger.fine("Relaying incoming HTTP request to " + outgoingUrl);
@@ -260,8 +267,10 @@ protected void handleHttpMessage(final ChannelHandlerContext ctx, final FullHttp
                   httpStatusCounterCache.get(httpStatusCode).inc();
                   EntityUtils.consumeQuietly(response.getEntity());
                 } catch (IOException e) {
-                  logger.warning(
-                      "Unable to relay request to " + requestRelayTarget + ": " + e.getMessage());
+                  logger.log(
+                      Level.WARNING,
+                      "Unable to relay request to " + requestRelayTarget + ": " + e.getMessage(),
+                      e);
                   Metrics.newCounter(
                           new TaggedMetricName("listeners", "http-relay.failed", "port", handle))
                       .inc();
@@ -287,8 +296,6 @@ protected void handleHttpMessage(final ChannelHandlerContext ctx, final FullHttp
     switch (path) {
       case "/api/v2/series/": // Check doc's on the beginning of this file
         try {
-          byte[] bodyBytes = new byte[request.content().readableBytes()];
-          request.content().readBytes(bodyBytes);
           AgentPayload.MetricPayload obj = AgentPayload.MetricPayload.parseFrom(bodyBytes);
           reportMetrics(obj, pointsPerRequest, output::append);
         } catch (IOException e) {
@@ -299,6 +306,7 @@ protected void handleHttpMessage(final ChannelHandlerContext ctx, final FullHttp
         break;
       case "/api/v1/series/":
         try {
+          String requestBody = new String(bodyBytes, StandardCharsets.UTF_8);
           status =
               reportMetrics(jsonParser.readTree(requestBody), pointsPerRequest, output::append);
         } catch (Exception e) {
@@ -319,6 +327,7 @@ protected void handleHttpMessage(final ChannelHandlerContext ctx, final FullHttp
           return;
         }
         try {
+          String requestBody = new String(bodyBytes, StandardCharsets.UTF_8);
           reportChecks(jsonParser.readTree(requestBody), pointsPerRequest, output::append);
         } catch (Exception e) {
           status = HttpResponseStatus.BAD_REQUEST;
@@ -334,6 +343,7 @@ protected void handleHttpMessage(final ChannelHandlerContext ctx, final FullHttp
 
       case "/intake/":
         try {
+          String requestBody = new String(bodyBytes, StandardCharsets.UTF_8);
           status =
               processMetadataAndSystemMetrics(
                   jsonParser.readTree(requestBody),

diff --git a/tests/datadog/Makefile b/tests/datadog/Makefile
@@ -0,0 +1,15 @@
+all: testDD
+
+.check-env:
+ifndef WF_SERVER
+	$(error WF_SERVER is undefined)
+endif
+ifndef WF_TOKEN
+	$(error WF_TOKEN is undefined)
+endif
+ifndef DD_API_KEY
+	$(error DD_API_KEY is undefined)
+endif
+
+testDD: .check-env
+	WF_SERVER=${WF_SERVER} WF_TOKEN=${WF_TOKEN} DD_API_KEY=${DD_API_KEY} docker compose up --build --attach wf-proxy
diff --git a/tests/datadog/README.md b/tests/datadog/README.md
@@ -0,0 +1,25 @@
+# "DataDog Agent -> WFProxy -> DataDog" Tests
+
+## Build Proxy
+
+On Proxy repo home run:
+
+```
+MVN_ARGS="-DskipTests" make build-jar docker
+```
+
+## Run test
+
+On `tests/datadog/` run:
+
+```
+WF_SERVER=nimba \
+WF_TOKEN=XXXXX \
+DD_API_KEY=XXXX \
+make
+```
+
+## Test if is working
+
+1. Go to you WF server, and serach for a metric `docker.cpu.usage`, you shoul get some series with a `dd_agent_version=7` tag, and other with a `dd_agent_version=6` tag.
+2. Do the same on your Datadog acount (the `dd_agent_version` will not be available)
diff --git a/tests/datadog/compose.yaml b/tests/datadog/compose.yaml
@@ -0,0 +1,42 @@
+services:
+  wf-proxy:
+    hostname: wf-proxy
+    build: ../../docker
+    environment:
+      WAVEFRONT_URL: https://${WF_SERVER}.wavefront.com/api/
+      WAVEFRONT_TOKEN: ${WF_TOKEN}
+      WAVEFRONT_PROXY_ARGS: >
+        --dataDogJsonPorts 2879,2880 
+        --dataDogProcessSystemMetrics true 
+        --dataDogProcessServiceChecks true 
+        --dataDogRequestRelayTarget https://api.datadoghq.com
+        --preprocessorConfigFile /tmp/preprocessor_rules.yaml
+    volumes:
+      - ${PWD}/preprocessor_rules.yaml:/tmp/preprocessor_rules.yaml
+
+    ports:
+      - "2878:2878"
+      - "2879:2879"
+      - "2880:2880"
+
+  dd-agent-7:
+    hostname: dd-agent-7
+    image: gcr.io/datadoghq/agent:7
+    environment:
+      DD_DD_URL: http://host.docker.internal:2879
+      DD_API_KEY: ${DD_API_KEY}
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+      - /proc/:/host/proc/:ro
+      - /sys/fs/cgroup/:/host/sys/fs/cgroup:ro
+
+  dd-agent-6:
+    hostname: dd-agent-6
+    image: gcr.io/datadoghq/agent:6
+    environment:
+      DD_DD_URL: http://host.docker.internal:2880
+      DD_API_KEY: ${DD_API_KEY}
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+      - /proc/:/host/proc/:ro
+      - /sys/fs/cgroup/:/host/sys/fs/cgroup:ro
diff --git a/tests/datadog/preprocessor_rules.yaml b/tests/datadog/preprocessor_rules.yaml
@@ -0,0 +1,11 @@
+'2879': 
+  - rule    : ddv7
+    action  : addTag
+    tag     : dd_agent_version
+    value   : "7"
+
+'2880': 
+  - rule    : ddv6
+    action  : addTag
+    tag     : dd_agent_version
+    value   : "6"