chore: upgrade nwaku to 0.34.0 and update tests suite for compatibility

.github/workflows/ci.yml

@@ -122,13 +122,13 @@ jobs:
     uses: ./.github/workflows/test-node.yml
     secrets: inherit
     with:
-      nim_wakunode_image: ${{ inputs.nim_wakunode_image || 'wakuorg/nwaku:v0.31.0' }}
+      nim_wakunode_image: ${{ inputs.nim_wakunode_image || 'wakuorg/nwaku:v0.34.0' }}
       test_type: node
 
   node_optional:
     uses: ./.github/workflows/test-node.yml
     with:
-      nim_wakunode_image: ${{ inputs.nim_wakunode_image || 'wakuorg/nwaku:v0.31.0' }}
+      nim_wakunode_image: ${{ inputs.nim_wakunode_image || 'wakuorg/nwaku:v0.34.0' }}
       test_type: node-optional
 
   node_with_nwaku_master:

.github/workflows/test-node.yml

@@ -28,6 +28,7 @@ env:
 jobs:
   node:
     runs-on: ubuntu-latest
+    timeout-minutes: 60  # Add a 1-hour timeout to fail faster
     env:
       WAKUNODE_IMAGE: ${{ inputs.nim_wakunode_image }}
     permissions:
@@ -47,6 +48,9 @@ jobs:
 
       - run: npm run build:esm
 
+      - name: Create reports directory
+        run: mkdir -p packages/tests/reports
+
       - run: ${{ (inputs.test_type == 'node-optional') && 'npm run test:optional --workspace=@waku/tests' || 'npm run test:node' }}
 
       - name: Test Report

package.json

@@ -59,6 +59,7 @@
     "karma-webkit-launcher": "^2.4.0",
     "karma-webpack": "github:codymikol/karma-webpack#2337a82beb078c0d8e25ae8333a06249b8e72828",
     "lint-staged": "^15.4.3",
+    "mocha-multi-reporters": "^1.5.1",
     "playwright": "^1.40.1",
     "size-limit": "^11.0.1",
     "ts-loader": "^9.5.1",

packages/tests/.mocharc.cjs

@@ -7,18 +7,27 @@ const config = {
     'loader=ts-node/esm'
   ],
   exit: true,
-  retries: 4
+  retries: 2,
+  timeout: 150_000 
 };
 
 if (process.env.CI) {
   console.log("Running tests in parallel");
   config.parallel = true;
-  config.jobs = 6;
-  console.log("Using JSON reporter for test results");
-  config.reporter = 'json';
+  config.jobs = 4;
+  config.timeout = 1800000;
+  console.log("Using multi reporters for test results");
+  config.reporter = 'mocha-multi-reporters';
   config.reporterOptions = {
-    output: 'reports/mocha-results.json'
+    configFile: 'reporter-config.json'
   };
+  // Ensure reports directory exists
+  const fs = require('fs');
+  const path = require('path');
+  const reportsDir = path.join(__dirname, 'reports');
+  if (!fs.existsSync(reportsDir)){
+    fs.mkdirSync(reportsDir, { recursive: true });
+  }
 } else {
   console.log("Running tests serially. To enable parallel execution update mocha config");
 }

packages/tests/reporter-config.json

@@ -0,0 +1,6 @@
+{
+  "reporterEnabled": "spec, json",
+  "jsonReporter": {
+    "output": "reports/mocha-results.json"
+  }
+} 

packages/tests/src/lib/dockerode.ts

@@ -18,7 +18,7 @@ export default class Dockerode {
   public containerId?: string;
 
   private static network: Docker.Network;
-  private containerIp: string;
+  public readonly containerIp: string;
 
   private constructor(imageName: string, containerIp: string) {
     this.docker = new Docker();
@@ -107,6 +107,7 @@ export default class Dockerode {
     const container = await this.docker.createContainer({
       Image: this.IMAGE_NAME,
       HostConfig: {
+        NetworkMode: NETWORK_NAME,
         AutoRemove: true,
         PortBindings: {
           [`${restPort}/tcp`]: [{ HostPort: restPort.toString() }],
@@ -116,6 +117,8 @@ export default class Dockerode {
             [`${discv5UdpPort}/udp`]: [{ HostPort: discv5UdpPort.toString() }]
           })
         },
+        Dns: ["8.8.8.8"],
+        Links: [],
         Mounts: args.rlnRelayEthClientAddress
           ? [
               {
@@ -135,18 +138,19 @@ export default class Dockerode {
           [`${discv5UdpPort}/udp`]: {}
         })
       },
-      Cmd: argsArrayWithIP
-    });
-    await container.start();
-
-    await Dockerode.network.connect({
-      Container: container.id,
-      EndpointConfig: {
-        IPAMConfig: {
-          IPv4Address: this.containerIp
+      Cmd: argsArrayWithIP,
+      NetworkingConfig: {
+        EndpointsConfig: {
+          [NETWORK_NAME]: {
+            IPAMConfig: {
+              IPv4Address: this.containerIp
+            }
+          }
         }
       }
     });
+    await container.start();
+
     const logStream = fs.createWriteStream(logPath);
 
     container.logs(

packages/tests/src/lib/index.ts

@@ -29,24 +29,28 @@ export class ServiceNodesFleet {
     _args?: Args,
     withoutFilter = false
   ): Promise<ServiceNodesFleet> {
-    const serviceNodePromises = Array.from(
-      { length: nodesToCreate },
-      async () => {
-        const node = new ServiceNode(
-          makeLogFileName(mochaContext) +
-            Math.random().toString(36).substring(7)
-        );
+    const nodes: ServiceNode[] = [];
+
+    for (let index = 0; index < nodesToCreate; index++) {
+      const node = new ServiceNode(
+        makeLogFileName(mochaContext) + Math.random().toString(36).substring(7)
+      );
 
-        const args = getArgs(networkConfig, _args);
-        await node.start(args, {
-          retries: 3
-        });
+      const args = getArgs(networkConfig, _args);
 
-        return node;
+      // If this is not the first node and previous node had a nodekey, use its multiaddr as static node
+      if (index > 0) {
+        const prevNode = nodes[index - 1];
+        const multiaddr = await prevNode.getExternalWebsocketMultiaddr();
+        args.staticnode = multiaddr;
       }
-    );
 
-    const nodes = await Promise.all(serviceNodePromises);
+      await node.start(args, {
+        retries: 3
+      });
+
+      nodes.push(node);
+    }
     return new ServiceNodesFleet(nodes, withoutFilter, strictChecking);
   }
 
@@ -107,7 +111,24 @@ export class ServiceNodesFleet {
     return relayMessages.every((message) => message);
   }
 
-  public async confirmMessageLength(numMessages: number): Promise<void> {
+  public async confirmMessageLength(
+    numMessages: number,
+    { encryptedPayload }: { encryptedPayload?: boolean } = {
+      encryptedPayload: false
+    }
+  ): Promise<void> {
+    if (encryptedPayload) {
+      const filteredMessageList = Array.from(
+        new Set(
+          this.messageCollector.messageList
+            .filter((msg) => msg.payload?.toString)
+            .map((msg) => msg.payload.toString())
+        )
+      );
+      expect(filteredMessageList.length).to.equal(numMessages);
+      return;
+    }
+
     if (this.strictChecking) {
       await Promise.all(
         this.nodes.map(async (node) =>
@@ -132,7 +153,7 @@ export class ServiceNodesFleet {
 
 class MultipleNodesMessageCollector {
   public callback: (msg: DecodedMessage) => void = () => {};
-  protected messageList: Array<DecodedMessage> = [];
+  public readonly messageList: Array<DecodedMessage> = [];
   public constructor(
     private messageCollectors: MessageCollector[],
     private relayNodes?: ServiceNode[],
@@ -182,21 +203,21 @@ class MultipleNodesMessageCollector {
     }
   ): boolean {
     if (this.strictChecking) {
-      return this.messageCollectors.every((collector) => {
+      return this.messageCollectors.every((collector, _i) => {
         try {
           collector.verifyReceivedMessage(index, options);
-          return true; // Verification successful
+          return true;
         } catch (error) {
-          return false; // Verification failed, continue with the next collector
+          return false;
         }
       });
     } else {
-      return this.messageCollectors.some((collector) => {
+      return this.messageCollectors.some((collector, _i) => {
         try {
           collector.verifyReceivedMessage(index, options);
-          return true; // Verification successful
+          return true;
         } catch (error) {
-          return false; // Verification failed, continue with the next collector
+          return false;
         }
       });
     }
@@ -216,6 +237,7 @@ class MultipleNodesMessageCollector {
     const startTime = Date.now();
     const pubsubTopic = options?.pubsubTopic || DefaultTestPubsubTopic;
     const timeoutDuration = options?.timeoutDuration || 400;
+    const maxTimeout = Math.min(timeoutDuration * numMessages, 30000);
     const exact = options?.exact || false;
 
     while (this.messageList.length < numMessages) {
@@ -227,23 +249,102 @@ class MultipleNodesMessageCollector {
               return msgs.length >= numMessages;
             })
           );
-          return results.every((result) => result);
+          if (results.every((result) => result)) {
+            return true;
+          }
         } else {
           const results = await Promise.all(
             this.relayNodes.map(async (node) => {
               const msgs = await node.messages(pubsubTopic);
               return msgs.length >= numMessages;
             })
           );
-          return results.some((result) => result);
+          if (results.some((result) => result)) {
+            return true;
+          }
+        }
+
+        const elapsed = Date.now() - startTime;
+        if (elapsed > maxTimeout) {
+          log.warn(`Timeout waiting for messages after ${elapsed}ms`);
+          return false;
         }
+
+        await delay(10);
       }
+    }
 
-      if (Date.now() - startTime > timeoutDuration * numMessages) {
+    if (exact) {
+      if (this.messageList.length == numMessages) {
+        return true;
+      } else {
+        log.warn(
+          `Was expecting exactly ${numMessages} messages. Received: ${this.messageList.length}`
+        );
         return false;
       }
+    } else {
+      return true;
+    }
+  }
+
+  /**
+   * Waits for a total number of messages across all nodes using autosharding.
+   */
+  public async waitForMessagesAutosharding(
+    numMessages: number,
+    options?: {
+      contentTopic: string;
+      timeoutDuration?: number;
+      exact?: boolean;
+    }
+  ): Promise<boolean> {
+    const startTime = Date.now();
+    const timeoutDuration = options?.timeoutDuration || 400;
+    const exact = options?.exact || false;
 
-      await delay(10);
+    while (this.messageList.length < numMessages) {
+      if (this.relayNodes) {
+        if (this.strictChecking) {
+          // In strict mode, all nodes must have the messages
+          const results = await Promise.all(
+            this.messageCollectors.map(async (collector) => {
+              return collector.waitForMessagesAutosharding(
+                numMessages,
+                options
+              );
+            })
+          );
+          if (results.every((result) => result)) {
+            return true;
+          }
+        } else {
+          // In non-strict mode, at least one node must have the messages
+          const results = await Promise.all(
+            this.messageCollectors.map(async (collector) => {
+              return collector.waitForMessagesAutosharding(
+                numMessages,
+                options
+              );
+            })
+          );
+          if (results.some((result) => result)) {
+            return true;
+          }
+        }
+
+        if (Date.now() - startTime > timeoutDuration * numMessages) {
+          return false;
+        }
+
+        await delay(10);
+      } else {
+        // If no relay nodes, just wait for messages in the list
+        if (Date.now() - startTime > timeoutDuration * numMessages) {
+          return false;
+        }
+        await delay(10);
+      }
     }
 
     if (exact) {
@@ -253,7 +354,6 @@ class MultipleNodesMessageCollector {
         log.warn(
           `Was expecting exactly ${numMessages} messages. Received: ${this.messageList.length}`
         );
-
         return false;
       }
     } else {