Merge pull request #137 from jekka001/edge-older-version-bbt

Add support of edge older versions in BBT

docker-edge/docker-compose.postgres.yml

@@ -23,7 +23,7 @@ services:
     ports:
     - "5432"
     environment:
-      POSTGRES_MULTIPLE_DATABASES: '"thingsboard","tb_edge_1","tb_edge_2"'
+      POSTGRES_MULTIPLE_DATABASES: '"thingsboard","tb_edge","tb_edge_kafka","tb_edge_38","tb_edge_37"'
       POSTGRES_PASSWORD: postgres
     volumes:
       - ./tb-node/postgres:/var/lib/postgresql/data
@@ -34,12 +34,22 @@ services:
       - tb-node.env
     depends_on:
       - postgres
-  tb-edge-1:
+  tb-edge:
     env_file:
       - tb-edge.env
     depends_on:
       - postgres
-  tb-edge-2:
+  tb-edge-kafka:
+    env_file:
+      - tb-edge.env
+    depends_on:
+      - postgres
+  tb-edge-38:
+    env_file:
+      - tb-edge.env
+    depends_on:
+      - postgres
+  tb-edge-37:
     env_file:
       - tb-edge.env
     depends_on:

docker-edge/docker-compose.volumes.yml

@@ -23,30 +23,50 @@ services:
   tb-monolith:
     volumes:
       - tb-log-volume:/var/log/thingsboard
-  tb-edge-1:
+  tb-edge:
     volumes:
-      - tb-edge-log-volume-1:/var/log/tb-edge
-      - tb-edge-data-volume-1:/data
-  tb-edge-2:
+      - tb-edge-log-volume:/var/log/tb-edge
+      - tb-edge-data-volume:/data
+  tb-edge-kafka:
     volumes:
-      - tb-edge-log-volume-2:/var/log/tb-edge
-      - tb-edge-data-volume-2:/data
+      - tb-edge-log-volume-kafka:/var/log/tb-edge
+      - tb-edge-data-volume-kafka:/data
+  tb-edge-38:
+    volumes:
+      - tb-edge-log-volume-38:/var/log/tb-edge
+      - tb-edge-data-volume-38:/data
+  tb-edge-37:
+    volumes:
+      - tb-edge-log-volume-37:/var/log/tb-edge
+      - tb-edge-data-volume-37:/data
 volumes:
   postgres-db-volume:
     external:
       name: ${POSTGRES_DATA_VOLUME}
   tb-log-volume:
     external:
       name: ${TB_LOG_VOLUME}
-  tb-edge-log-volume-1:
+  tb-edge-log-volume:
+    external:
+      name: ${TB_EDGE_LOG_VOLUME}
+  tb-edge-data-volume:
+    external:
+      name: ${TB_EDGE_DATA_VOLUME}
+  tb-edge-log-volume-kafka:
+    external:
+      name: ${TB_EDGE_LOG_VOLUME_KAFKA}
+  tb-edge-data-volume-kafka:
+    external:
+      name: ${TB_EDGE_DATA_VOLUME_KAFKA}
+  tb-edge-log-volume-38:
     external:
-      name: ${TB_EDGE_LOG_VOLUME_1}
-  tb-edge-data-volume-1:
+      name: ${TB_EDGE_LOG_VOLUME_38}
+  tb-edge-data-volume-38:
     external:
-      name: ${TB_EDGE_DATA_VOLUME_1}
-  tb-edge-log-volume-2:
+      name: ${TB_EDGE_DATA_VOLUME_38}
+  tb-edge-log-volume-37:
     external:
-      name: ${TB_EDGE_LOG_VOLUME_2}
-  tb-edge-data-volume-2:
+      name: ${TB_EDGE_LOG_VOLUME_37}
+  tb-edge-data-volume-37:
     external:
-      name: ${TB_EDGE_DATA_VOLUME_2}
+      name: ${TB_EDGE_DATA_VOLUME_37}

docker-edge/docker-compose.yml

@@ -18,50 +18,92 @@
 version: '3.0'
 
 services:
-  tb-edge-1:
+  tb-edge:
     restart: always
     image: "${DOCKER_REPO}/${TB_EDGE_DOCKER_NAME}:${TB_EDGE_VERSION}"
     ports:
       - "8082"
       - "1883"
     environment:
-      CLOUD_ROUTING_KEY: "${CLOUD_ROUTING_KEY_1}"
-      CLOUD_ROUTING_SECRET: "${CLOUD_ROUTING_SECRET_1}"
-      SPRING_DATASOURCE_URL: "${SPRING_DATASOURCE_URL_1}"
+      CLOUD_ROUTING_KEY: "${CLOUD_ROUTING_KEY}"
+      CLOUD_ROUTING_SECRET: "${CLOUD_ROUTING_SECRET}"
+      SPRING_DATASOURCE_URL: "${SPRING_DATASOURCE_URL}"
       CLOUD_RPC_HOST: "${CLOUD_RPC_HOST}"
       HTTP_BIND_PORT: "8082"
       CONF_FOLDER: "/config"
     env_file:
       - tb-edge.env
     volumes:
       - ./tb-edge/conf:/config
-  tb-edge-2:
+    depends_on:
+      - tb-monolith
+  tb-edge-kafka:
     restart: always
     image: "${DOCKER_REPO}/${TB_EDGE_DOCKER_NAME}:${TB_EDGE_VERSION}"
     ports:
       - "8083"
       - "1884"
     environment:
-      CLOUD_ROUTING_KEY: "${CLOUD_ROUTING_KEY_2}"
-      CLOUD_ROUTING_SECRET: "${CLOUD_ROUTING_SECRET_2}"
-      SPRING_DATASOURCE_URL: "${SPRING_DATASOURCE_URL_2}"
+      CLOUD_ROUTING_KEY: "${CLOUD_ROUTING_KEY_KAFKA}"
+      CLOUD_ROUTING_SECRET: "${CLOUD_ROUTING_SECRET_KAFKA}"
+      SPRING_DATASOURCE_URL: "${SPRING_DATASOURCE_URL_KAFKA}"
       CLOUD_RPC_HOST: "${CLOUD_RPC_HOST}"
       HTTP_BIND_PORT: "8083"
       CONF_FOLDER: "/config"
       TB_QUEUE_TYPE: "kafka"
-      TB_KAFKA_SERVERS: "kafka-edge-2:9092"
+      TB_KAFKA_SERVERS: "kafka-edge:9092"
+    env_file:
+      - tb-edge.env
+    volumes:
+      - ./tb-edge/conf:/config
+    depends_on:
+      - tb-monolith
+  tb-edge-38:
+    restart: always
+    image: "${DOCKER_REPO}/${TB_EDGE_DOCKER_NAME}:3.8.0EDGE"
+    ports:
+      - "8084"
+      - "1885"
+    environment:
+      CLOUD_ROUTING_KEY: "${CLOUD_ROUTING_KEY_38}"
+      CLOUD_ROUTING_SECRET: "${CLOUD_ROUTING_SECRET_38}"
+      SPRING_DATASOURCE_URL: "${SPRING_DATASOURCE_URL_38}"
+      CLOUD_RPC_HOST: "${CLOUD_RPC_HOST}"
+      HTTP_BIND_PORT: "8084"
+      CONF_FOLDER: "/config"
     env_file:
       - tb-edge.env
     volumes:
       - ./tb-edge/conf:/config
-  kafka-edge-2:
+    depends_on:
+      - tb-monolith
+  tb-edge-37:
+    restart: always
+    image: "${DOCKER_REPO}/${TB_EDGE_DOCKER_NAME}:3.7.0EDGE"
+    ports:
+      - "8085"
+      - "1886"
+    environment:
+      CLOUD_ROUTING_KEY: "${CLOUD_ROUTING_KEY_37}"
+      CLOUD_ROUTING_SECRET: "${CLOUD_ROUTING_SECRET_37}"
+      SPRING_DATASOURCE_URL: "${SPRING_DATASOURCE_URL_37}"
+      CLOUD_RPC_HOST: "${CLOUD_RPC_HOST}"
+      HTTP_BIND_PORT: "8085"
+      CONF_FOLDER: "/config"
+    env_file:
+      - tb-edge.env
+    volumes:
+      - ./tb-edge/conf:/config
+    depends_on:
+      - tb-monolith
+  kafka-edge:
     restart: always
     image: "bitnami/kafka:3.8.1"
     ports:
       - "9092"
     environment:
-      KAFKA_CFG_ADVERTISED_LISTENERS: "INSIDE://:9094,OUTSIDE://kafka-edge-2:9092"
-      KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: "0@kafka-edge-2:9093"
+      KAFKA_CFG_ADVERTISED_LISTENERS: "INSIDE://:9094,OUTSIDE://kafka-edge:9092"
+      KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: "0@kafka-edge:9093"
     env_file:
       - kafka.env
   tb-monolith:

msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/AbstractContainerTest.java

@@ -111,8 +111,12 @@ public abstract class AbstractContainerTest {
 
     public static final List<TestEdgeConfiguration> edgeConfigurations =
             Arrays.asList(
-                    new TestEdgeConfiguration("280629c7-f853-ee3d-01c0-fffbb6f2ef38", "g9ta4soeylw6smqkky8g", 8082, 1, "Edge-in-memory"),
-                    new TestEdgeConfiguration("e29dadb1-c487-3b9e-1b5a-02193191c90e", "dmb17p71vz9svfl7tgnz", 8083, 2, "Edge-kafka"));
+                    new TestEdgeConfiguration("280629c7-f853-ee3d-01c0-fffbb6f2ef38", "g9ta4soeylw6smqkky8g", 8082, ""),
+                    new TestEdgeConfiguration("e29dadb1-c487-3b9e-1b5a-02193191c90e", "dmb17p71vz9svfl7tgnz", 8083, "kafka"),
+                    new TestEdgeConfiguration("2cc28012-a2f3-8bff-7b1a-5e686c972e1e", "z2d2z90fqjylht011ram", 8084, "38"),
+                    new TestEdgeConfiguration("774e5e4e-8ec7-9945-1c6a-4d6ba08cb5fc", "om3zzzadzlugth03nibn", 8085, "37")
+            );
+
 
     protected static List<TestEdgeRuntimeParameters> testParameters = new ArrayList<>();
 
@@ -156,10 +160,10 @@ public static void before() throws Exception {
             RuleChainId edgeRuleChainId = updateEdgeRootRuleChain();
 
             for (TestEdgeConfiguration config : edgeConfigurations) {
-                String edgeHost = ContainerTestSuite.testContainer.getServiceHost(TB_EDGE_SERVICE_NAME + "-" + config.getIdx(), config.getPort());
-                Integer edgePort = ContainerTestSuite.testContainer.getServicePort(TB_EDGE_SERVICE_NAME + "-" + config.getIdx(), config.getPort());
+                String edgeHost = ContainerTestSuite.testContainer.getServiceHost(TB_EDGE_SERVICE_NAME + config.getTagWithDash(), config.getPort());
+                Integer edgePort = ContainerTestSuite.testContainer.getServicePort(TB_EDGE_SERVICE_NAME + config.getTagWithDash(), config.getPort());
                 String edgeUrl = "http://" + edgeHost + ":" + edgePort;
-                Edge edge = createEdge(config.getName(), config.getRoutingKey(), config.getSecret());
+                Edge edge = createEdge("edge" + config.getTagWithDash(), config.getRoutingKey(), config.getSecret());
                 testParameters.add(new TestEdgeRuntimeParameters(new RestClient(edgeUrl), edge, edgeUrl));
             }
 

msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java

@@ -22,9 +22,11 @@
 import org.junit.extensions.cpsuite.ClasspathSuite;
 import org.junit.runner.RunWith;
 import org.testcontainers.containers.DockerComposeContainer;
+import org.testcontainers.containers.wait.strategy.Wait;
 
 import java.io.File;
 import java.io.IOException;
+import java.time.Duration;
 import java.util.HashMap;
 import java.util.UUID;
 
@@ -49,8 +51,8 @@ public static DockerComposeContainer getTestContainer() {
         HashMap<String, String> env = new HashMap<>();
         env.put("CLOUD_RPC_HOST", TB_MONOLITH_SERVICE_NAME);
         for (TestEdgeConfiguration config : edgeConfigurations) {
-            env.put("CLOUD_ROUTING_KEY_" + config.getIdx(), config.getRoutingKey());
-            env.put("CLOUD_ROUTING_SECRET_" + config.getIdx(), config.getSecret());
+            env.put("CLOUD_ROUTING_KEY" + config.getTagWithUnderscore().toUpperCase(), config.getRoutingKey());
+            env.put("CLOUD_ROUTING_SECRET" + config.getTagWithUnderscore().toUpperCase(), config.getSecret());
         }
 
         if (testContainer == null) {
@@ -72,6 +74,7 @@ public void stop() {
                         super.stop();
                         tryDeleteDir(targetDir);
                     }
+
                 }
 
                 testContainer = new DockerComposeContainerImpl<>(
@@ -84,12 +87,12 @@ public void stop() {
                         .withTailChildContainers(true)
                         .withEnv(installTb.getEnv())
                         .withEnv(env)
-                        .withExposedService(TB_MONOLITH_SERVICE_NAME, 8080)
-                        .withExposedService("kafka", 9092);
+                        .withExposedService(TB_MONOLITH_SERVICE_NAME, 8080, Wait.defaultWaitStrategy().withStartupTimeout(Duration.ofMinutes(3)))
+                        .withExposedService("kafka", 9092, Wait.defaultWaitStrategy().withStartupTimeout(Duration.ofMinutes(3)));
                 for (TestEdgeConfiguration edgeConfiguration : edgeConfigurations) {
-                    testContainer.withExposedService(TB_EDGE_SERVICE_NAME + "-" + edgeConfiguration.getIdx(), edgeConfiguration.getPort());
-                    if (edgeConfiguration.getName().contains("kafka")) {
-                        testContainer.withExposedService("kafka-edge-" + edgeConfiguration.getIdx(), 9092);
+                    testContainer.withExposedService(TB_EDGE_SERVICE_NAME + edgeConfiguration.getTagWithDash(), edgeConfiguration.getPort(), Wait.defaultWaitStrategy().withStartupTimeout(Duration.ofMinutes(3)));
+                    if (edgeConfiguration.getTag().equals("kafka")) {
+                        testContainer.withExposedService("kafka-edge", 9092, Wait.defaultWaitStrategy().withStartupTimeout(Duration.ofMinutes(3)));
                     }
                 }
 
@@ -109,4 +112,5 @@ private static void tryDeleteDir(String targetDir) {
             log.error("Can't delete temp directory " + targetDir, e);
         }
     }
+
 }

msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/TestEdgeConfiguration.java

@@ -24,6 +24,13 @@ public class TestEdgeConfiguration {
     private String routingKey;
     private String secret;
     private Integer port;
-    private Integer idx;
-    private String name;
+    private String tag;
+
+    public String getTagWithDash(){
+        return tag.isEmpty() ? "" : "-" + tag;
+    }
+
+    public String getTagWithUnderscore(){
+        return tag.isEmpty() ? "" : "_" + tag;
+    }
 }

msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java

@@ -29,6 +29,8 @@
 import java.util.List;
 import java.util.Map;
 
+import static org.thingsboard.server.msa.AbstractContainerTest.edgeConfigurations;
+
 @Slf4j
 public class ThingsBoardDbInstaller extends ExternalResource {
 
@@ -71,10 +73,13 @@ public ThingsBoardDbInstaller() {
             }
             env.put("POSTGRES_DATA_VOLUME", postgresDataVolume);
             env.put("TB_LOG_VOLUME", tbLogVolume);
-            for (int edgeEnv = 1; edgeEnv <= 2; edgeEnv++) {
-                env.put("SPRING_DATASOURCE_URL_" + edgeEnv, "jdbc:postgresql://postgres:5432/tb_edge_" + edgeEnv);
-                env.put("TB_EDGE_LOG_VOLUME_" + edgeEnv, tbEdgeLogVolume + "-" + edgeEnv);
-                env.put("TB_EDGE_DATA_VOLUME_" + edgeEnv, tbEdgeDataVolume + "-" + edgeEnv);
+
+            for (TestEdgeConfiguration config : edgeConfigurations) {
+                env.put("SPRING_DATASOURCE_URL" + config.getTagWithUnderscore().toUpperCase(), "jdbc:postgresql://postgres:5432/tb_edge" + config.getTagWithUnderscore());
+                env.put("TB_EDGE_LOG_VOLUME" + config.getTagWithUnderscore().toUpperCase(), tbEdgeLogVolume + config.getTagWithDash());
+                env.put("TB_EDGE_DATA_VOLUME" + config.getTagWithUnderscore().toUpperCase(), tbEdgeDataVolume + config.getTagWithDash());
+                env.put("CLOUD_ROUTING_KEY" + config.getTagWithUnderscore().toUpperCase(), config.getRoutingKey());
+                env.put("CLOUD_ROUTING_SECRET" + config.getTagWithUnderscore().toUpperCase(), config.getSecret());
             }
 
             dockerCompose.withEnv(env);
@@ -93,11 +98,11 @@ protected void before() throws Throwable {
 
             dockerCompose.withCommand("volume create " + tbLogVolume);
             dockerCompose.invokeDocker();
-            for (int edgeEnv = 1; edgeEnv <= 2; edgeEnv++) {
-                dockerCompose.withCommand("volume create " + tbEdgeLogVolume + "-" + edgeEnv);
+            for (TestEdgeConfiguration config : edgeConfigurations) {
+                dockerCompose.withCommand("volume create " + tbEdgeLogVolume + config.getTagWithDash());
                 dockerCompose.invokeDocker();
 
-                dockerCompose.withCommand("volume create " + tbEdgeDataVolume + "-" + edgeEnv);
+                dockerCompose.withCommand("volume create " + tbEdgeDataVolume  + config.getTagWithDash());
                 dockerCompose.invokeDocker();
             }
 
@@ -107,14 +112,14 @@ protected void before() throws Throwable {
             dockerCompose.withCommand("run --no-deps --rm -e INSTALL_TB=true -e LOAD_DEMO=true tb-monolith");
             dockerCompose.invokeCompose();
 
-            for (int edgeEnv = 1; edgeEnv <= 2; edgeEnv++) {
-                dockerCompose.withCommand("run --no-deps --rm -e INSTALL_TB_EDGE=true -e LOAD_DEMO=true tb-edge" + "-" + edgeEnv);
+            for (TestEdgeConfiguration config : edgeConfigurations) {
+                dockerCompose.withCommand("run --no-deps --rm -e INSTALL_TB_EDGE=true -e LOAD_DEMO=true tb-edge" + config.getTagWithDash());
                 dockerCompose.invokeCompose();
             }
             dockerCompose.withCommand("exec -T postgres psql -U postgres -d thingsboard -f /custom-sql/thingsboard.sql");
             dockerCompose.invokeCompose();
-            for (int edgeEnv = 1; edgeEnv <= 2; edgeEnv++) {
-                dockerCompose.withCommand("exec -T postgres psql -U postgres -d tb_edge" + "_" + edgeEnv + " -f /custom-sql/tb_edge.sql");
+            for (TestEdgeConfiguration config : edgeConfigurations) {
+                dockerCompose.withCommand("exec -T postgres psql -U postgres -d tb_edge" + config.getTagWithUnderscore() + " -f /custom-sql/tb_edge.sql");
                 dockerCompose.invokeCompose();
             }
         } finally {
@@ -130,11 +135,11 @@ protected void before() throws Throwable {
     @Override
     protected void after() {
         try {
-            for (int edgeEnv = 1; edgeEnv <= 2; edgeEnv++) {
+            for (TestEdgeConfiguration config : edgeConfigurations) {
                 copyLogs(tbLogVolume, "./target/tb-logs/");
-                copyLogs(tbEdgeLogVolume + "-" + edgeEnv, "./target/tb-edge-logs/");
+                copyLogs(tbEdgeLogVolume + config.getTagWithDash(), "./target/tb-edge-logs/");
 
-                dockerCompose.withCommand("volume rm -f " + postgresDataVolume + " " + tbLogVolume + " " + tbEdgeLogVolume + "-" + edgeEnv);
+                dockerCompose.withCommand("volume rm -f " + postgresDataVolume + " " + tbLogVolume + " " + tbEdgeLogVolume + config.getTagWithDash());
                 dockerCompose.invokeDocker();
             }
         } catch (Exception e) {