Fix metric_value_benchmark integration tests by using latest Kafka ve…

…rsion in JMXKafka test (#436)

test/metric_value_benchmark/jmx_kafka_test.go

@@ -6,7 +6,9 @@
 package metric_value_benchmark
 
 import (
+	"fmt"
 	"log"
+	"strings"
 	"time"
 
 	"github.com/aws/amazon-cloudwatch-agent-test/test/metric"
@@ -53,26 +55,35 @@ func (t *JMXKafkaTestRunner) SetupBeforeAgentRun() error {
 		return err
 	}
 
+	log.Println("get latest kafka version")
+	version, err := common.RunCommand("curl https://dlcdn.apache.org/kafka/ | grep -oP \"\\d\\.\\d\\.\\d\" | tail -1")
+	if err != nil {
+		return fmt.Errorf("error getting latest kafka version: %v", err)
+	}
+	version = strings.TrimSpace(version)
+	kafkaPackageName := fmt.Sprintf("kafka_2.13-%s", version)
+
 	log.Println("set up zookeeper and kafka")
 	startJMXCommands := []string{
-		"curl https://dlcdn.apache.org/kafka/3.6.2/kafka_2.13-3.6.2.tgz -o kafka_2.13-3.6.2.tgz",
-		"tar -xzf kafka_2.13-3.6.2.tgz",
-		"echo 'export JMX_PORT=2000'|cat - kafka_2.13-3.6.2/bin/kafka-server-start.sh > /tmp/kafka-server-start.sh && mv /tmp/kafka-server-start.sh kafka_2.13-3.6.2/bin/kafka-server-start.sh",
-		"echo 'export JMX_PORT=2010'|cat - kafka_2.13-3.6.2/bin/kafka-console-consumer.sh > /tmp/kafka-console-consumer.sh && mv /tmp/kafka-console-consumer.sh kafka_2.13-3.6.2/bin/kafka-console-consumer.sh",
-		"echo 'export JMX_PORT=2020'|cat - kafka_2.13-3.6.2/bin/kafka-console-producer.sh > /tmp/kafka-console-producer.sh && mv /tmp/kafka-console-producer.sh kafka_2.13-3.6.2/bin/kafka-console-producer.sh",
-		"sudo chmod +x kafka_2.13-3.6.2/bin/kafka-run-class.sh",
-		"sudo chmod +x kafka_2.13-3.6.2/bin/kafka-server-start.sh",
-		"sudo chmod +x kafka_2.13-3.6.2/bin/kafka-console-consumer.sh",
-		"sudo chmod +x kafka_2.13-3.6.2/bin/kafka-console-producer.sh",
-		"(yes | nohup kafka_2.13-3.6.2/bin/kafka-console-producer.sh --topic quickstart-events --bootstrap-server localhost:9092) > /tmp/kafka-console-producer-logs.txt 2>&1 &",
-		"kafka_2.13-3.6.2/bin/kafka-console-consumer.sh --topic quickstart-events --from-beginning --bootstrap-server localhost:9092 > /tmp/kafka-console-consumer-logs.txt 2>&1 &",
+		fmt.Sprintf("curl https://dlcdn.apache.org/kafka/%s/%s.tgz -o %s.tgz", version, kafkaPackageName, kafkaPackageName),
+		fmt.Sprintf("tar -xzf %s.tgz", kafkaPackageName),
+		fmt.Sprintf("sudo mv %s kafka_2.13-latest", kafkaPackageName),
+		"echo 'export JMX_PORT=2000'|cat - kafka_2.13-latest/bin/kafka-server-start.sh > /tmp/kafka-server-start.sh && mv /tmp/kafka-server-start.sh kafka_2.13-latest/bin/kafka-server-start.sh",
+		"echo 'export JMX_PORT=2010'|cat - kafka_2.13-latest/bin/kafka-console-consumer.sh > /tmp/kafka-console-consumer.sh && mv /tmp/kafka-console-consumer.sh kafka_2.13-latest/bin/kafka-console-consumer.sh",
+		"echo 'export JMX_PORT=2020'|cat - kafka_2.13-latest/bin/kafka-console-producer.sh > /tmp/kafka-console-producer.sh && mv /tmp/kafka-console-producer.sh kafka_2.13-latest/bin/kafka-console-producer.sh",
+		"sudo chmod +x kafka_2.13-latest/bin/kafka-run-class.sh",
+		"sudo chmod +x kafka_2.13-latest/bin/kafka-server-start.sh",
+		"sudo chmod +x kafka_2.13-latest/bin/kafka-console-consumer.sh",
+		"sudo chmod +x kafka_2.13-latest/bin/kafka-console-producer.sh",
+		"(yes | nohup kafka_2.13-latest/bin/kafka-console-producer.sh --topic quickstart-events --bootstrap-server localhost:9092) > /tmp/kafka-console-producer-logs.txt 2>&1 &",
+		"kafka_2.13-latest/bin/kafka-console-consumer.sh --topic quickstart-events --from-beginning --bootstrap-server localhost:9092 > /tmp/kafka-console-consumer-logs.txt 2>&1 &",
 		"curl https://dlcdn.apache.org/zookeeper/zookeeper-3.8.4/apache-zookeeper-3.8.4-bin.tar.gz -o apache-zookeeper-3.8.4-bin.tar.gz",
 		"tar -xzf apache-zookeeper-3.8.4-bin.tar.gz",
 		"mkdir apache-zookeeper-3.8.4-bin/data",
 		"touch apache-zookeeper-3.8.4-bin/conf/zoo.cfg",
 		"echo -e 'tickTime = 2000\ndataDir = ../data\nclientPort = 2181\ninitLimit = 5\nsyncLimit = 2\n' >> apache-zookeeper-3.8.4-bin/conf/zoo.cfg",
 		"sudo apache-zookeeper-3.8.4-bin/bin/zkServer.sh start",
-		"sudo kafka_2.13-3.6.2/bin/kafka-server-start.sh kafka_2.13-3.6.2/config/server.properties > /tmp/kafka-server-start-logs.txt 2>&1 &",
+		"sudo kafka_2.13-latest/bin/kafka-server-start.sh kafka_2.13-latest/config/server.properties > /tmp/kafka-server-start-logs.txt 2>&1 &",
 	}
 
 	err = common.RunCommands(startJMXCommands)