Merge branch 'main' into dependabot/github_actions/dependabot/fetch-m…

…etadata-2

.github/workflows/gh-pages.yml

@@ -12,7 +12,7 @@ jobs:
       - uses: actions/checkout@v4
 
       - name: Setup mdBook
-        uses: peaceiris/actions-mdbook@v1
+        uses: peaceiris/actions-mdbook@v2
         with:
           mdbook-version: "latest"
 

.gitlab-ci.yml

@@ -26,6 +26,7 @@ variables:
   # Improve gitlab cache performance
   CACHE_COMPRESSION_LEVEL: "fastest"
   FF_USE_FASTZIP: "true"
+  PUSHGATEWAY_URL: "http://zombienet-prometheus-pushgateway.managed-monitoring:9091/metrics/job/zombie-metrics"
 
 cache:
   - key:

admissioncontroller/keys/server.crt

@@ -0,0 +1,33 @@
+-----BEGIN CERTIFICATE-----
+MIIFqDCCA5CgAwIBAgIUY8xAGirtPjY9LJgXL2Z20fqq2sEwDQYJKoZIhvcNAQEL
+BQAwQzFBMD8GA1UEAww4bGFyZ2UtdGVzdG5ldC1ob29rLmxhcmdlLXRlc3RuZXQt
+YWRtaXNzaW9uY29udHJvbGxlci5zdmMwHhcNMjMxMDE3MTc1OTIwWhcNMjQxMDE2
+MTc1OTIwWjBDMUEwPwYDVQQDDDhsYXJnZS10ZXN0bmV0LWhvb2subGFyZ2UtdGVz
+dG5ldC1hZG1pc3Npb25jb250cm9sbGVyLnN2YzCCAiIwDQYJKoZIhvcNAQEBBQAD
+ggIPADCCAgoCggIBALegwGmYWCODgd616xKc7zprID9Z7YieYm4lzy4IvQYSpyP5
+ibMM+LJ6K5fZFYkLoTbUyp0CCUXmSJqbmKKIrvB84uuem1qhge75HHBWPeBPvTVY
+trkKheTnSZJQemCF3MgTD31j7RJ15glgHp944uus9Q0tLf9csrVBw56TL3a53wyb
+hvKphRYgkLOK+2F3cc19iekaYzmyXc9rXxHxYeYvO0D8roGjztNL6Ml88V9ggYCb
+0ITa7Da8wfMEtyZZxuWvUSRymSTcxrN0gMdOtJlcrooMxv26lais46qYB3ybzqC0
+E1XKXNj98SQbLRHVn8DxxocdHNyoTZ3bl4WmEdIn3tuq2AexlWDy8Ti8u+uGSYvP
+Aw4TDTZkJUooRzkyrmW+l2sYbhMi9pxUQ24avQfDQ8djx0UgwL28P2Ea+yKysfQa
+8qi1ppLVS5gSV3DPoixWVACtNoO0dZGdBJ+91W2w1Kh1bPjy+ZsiIgFHRaZY0CFx
+yMDPQSNgs4SYR4rA890aRRquMsY2GmVQXPnDygaeUScyik1lWY98amn79SAdJlo+
+Y89CVIQIPHlTPlwNWlxnaTzZPQpUJY06kZYMymhD+dqWp0IcWllLCNgLapu1aPnj
+kxekN3rWbjCUE7y0oJt8z1U39zlCyKrbcvPsBzQ7GaWhQqiOKZuRfa7I3Ed3AgMB
+AAGjgZMwgZAwgY0GA1UdEQSBhTCBgoI4bGFyZ2UtdGVzdG5ldC1ob29rLmxhcmdl
+LXRlc3RuZXQtYWRtaXNzaW9uY29udHJvbGxlci5zdmOCRmxhcmdlLXRlc3RuZXQt
+aG9vay5sYXJnZS10ZXN0bmV0LWFkbWlzc2lvbmNvbnRyb2xsZXIuc3ZjLmNsdXN0
+ZXIubG9jYWwwDQYJKoZIhvcNAQELBQADggIBABqWe9400pKXSM/tyCXomyKAyCq1
+mhDimTwX8PiB2VV/cfezeuLB3TMRcBSFz2N8+bNh0YqWAcXjqptzu8b4IqMXDyKR
+OSgSQkBL+rT6albpE5htGjecDl0FzWGyE/lZ/BHQnjPlfjXQihqBR3vZB7L9+0XJ
+0GBecxgsnYIsPh0yqmiz6KpYGq+VrL7owIWixGjRo7i6LwwdIIavtnDXRAReYEca
+ebjSlaVIwrKniZMVnD/vE6EvM2cZJaOBYVH8UcYHU2b03FvM6hhQq8MLcF6git8r
+AkVZ6rnqzdvqIIhbuM8jBJhJ2VDdjUdTA39pz7Njvljxo1W+FAhUiBtc9YsYF5oi
+iGGm9A+hSXMP4InX3NkntMSDBIsBvcWz9r3K53mQInUdiOxONwFdLpJ6NEajx+DN
+kfGMMBu4k/RLREqmA6pfT6i3qJDBYCRk1Jd6LU+R8xKmpugVsSbEXo8zpbM1T/Ww
+0Yh65gX1wwryHTO7tvnT/jAHnzWQAwbC8au3nWK/nu+Ksn1fWf6p3XBT5Gii9sO4
+AGRklkgtMZZIcoQFN7iOROV8oFkNwaN0Viq19GiV2/yldq/Sgo5mQLKjOVllrkfE
+wlF7V53QE/8C4F82ANhUt84wVXKoP620rOce33o7EQLpadG+kmxhCxOw9SFsmQGO
+/eqbmMMT7BSW9hsH
+-----END CERTIFICATE-----

admissioncontroller/keys/server.key

@@ -0,0 +1,52 @@
+-----BEGIN PRIVATE KEY-----
+MIIJRAIBADANBgkqhkiG9w0BAQEFAASCCS4wggkqAgEAAoICAQC3oMBpmFgjg4He
+tesSnO86ayA/We2InmJuJc8uCL0GEqcj+YmzDPiyeiuX2RWJC6E21MqdAglF5kia
+m5iiiK7wfOLrnptaoYHu+RxwVj3gT701WLa5CoXk50mSUHpghdzIEw99Y+0SdeYJ
+YB6feOLrrPUNLS3/XLK1QcOeky92ud8Mm4byqYUWIJCzivthd3HNfYnpGmM5sl3P
+a18R8WHmLztA/K6Bo87TS+jJfPFfYIGAm9CE2uw2vMHzBLcmWcblr1Ekcpkk3Maz
+dIDHTrSZXK6KDMb9upWorOOqmAd8m86gtBNVylzY/fEkGy0R1Z/A8caHHRzcqE2d
+25eFphHSJ97bqtgHsZVg8vE4vLvrhkmLzwMOEw02ZCVKKEc5Mq5lvpdrGG4TIvac
+VENuGr0Hw0PHY8dFIMC9vD9hGvsisrH0GvKotaaS1UuYEldwz6IsVlQArTaDtHWR
+nQSfvdVtsNSodWz48vmbIiIBR0WmWNAhccjAz0EjYLOEmEeKwPPdGkUarjLGNhpl
+UFz5w8oGnlEnMopNZVmPfGpp+/UgHSZaPmPPQlSECDx5Uz5cDVpcZ2k82T0KVCWN
+OpGWDMpoQ/nalqdCHFpZSwjYC2qbtWj545MXpDd61m4wlBO8tKCbfM9VN/c5Qsiq
+23Lz7Ac0OxmloUKojimbkX2uyNxHdwIDAQABAoICABZNmBE7R7lIh8WdIugn14ao
+7EnFimxFkH8AaKGWvh62tUZRUMBGmw5SeGkBHPSATRxzzAg17xpmVfsxWuCi6Vyc
+TpN/uC7R3w4/g4zBbNKJ3NlV5Rukg47B6NW0oKiImcBTUFZShMDxQPdyjUWhUS5+
+22S2UBfsEmihjDj+d41cFhNZ8JIYtOlzGyAziUTVVSU4Uj/VzKd9WpeO2wU65EEp
+C3FhP3DPjJdtaODQJCEJf7E3O6RnUrlzQ7tYn0YxMU9Ec/6RTLiFeDzjsaCDlrvp
+3ek2lBTNTNAn758QyBgCdJ5gQNjys0RH07om8tpvr++nbg2LI65JYuQBf+3jtdYz
++NgEJMYPilH1BWNp+Tpt9ub4Flp4I4wjylfxTUAZu4O70GzavNEnkC7J4x664Uh7
+w1VmzdWE86T+QjkEw9sYLupQXZ8HmsEy4JXZArAqSx0uDNkZu4h8yRzqTHZneY1C
+m2vTWAiga4sRhJpy6avhRcNl5koj048ON8NLVQtiLiC7SdvZlSb7SsOrZT/HQDys
+W12T6YLCp+coRoQ/IAasYqGORhLzGJi/0jf34dmJQZbl5nZsecOzFaTTa83Y90BS
+bH1NXTk9XNFy+j5LxClv7WQY/811TYnbVqLySsuM6+i9MlaMB1JKHokI0zJjjcu6
+FocpBxk1HNXROCoLsnPBAoIBAQDivArB326626fzR2qZ5GzcNgxLFHCr2jFsbo00
+oK+EJbjXWOc2NXk7qTKo0WATSomdSwY7sD5mtYV4Y7H2qlygvsRsXfrU7jCEGTOC
+qDOhoVaYNK7t7gtmtlgLZq45MLAEg+VLB6QQ25BTp96+kY8JZqdtHTw8xRIe99GX
+QoRbA1M8ptPV0HPSRPtLPc0bpMFqrpZnkcoaUo9qY2rZtunWB3VIvqJcpJBpPUkO
+z7knZvD4DEzOBhDlXj+FUFv/ls6RWDwwJSXz7oQYfO/RhsA6CUcRvYZeuLm/RQnQ
+C91n6hr+7Mj/oKzFYTD12G8Eazzp49Ynnym+Krf/mqpWxBQFAoIBAQDPVFgVEf97
+eRh5H6DfzEo0Mja8XSpxz9OXBdy88FV8PyxIG4CxzCiOft1hzNtocZzcQ9g+XquO
+yENEZd7FvBERXuixq5jemsUw+onRBJ7J0DmISMUOorTBeXIP5nnA2ZbyRoV04oCb
+fJyGcovHmyxQIYsjfXZaeF9qKkilKFCFiYfDCNIvDq+DbHmEMRUZWeNI/RF1lLp5
+xI/jRqiBC1cRkCqnsweNlNAaJD6bmAOJveYtU0dtPB1MAGyMU3QLfm3nvCDI97wb
+XLDtN7vHk2qosaS4fqd9vpacKXbAgFoIzoE6YH9ohOoYhXosvGCq4jp5BF8OJh/+
+4n5C9vsPJuJLAoIBAQCEKIc8qjeowxAKRb5lJTguw3I28RsbkHrucSHtYxvrMvGi
+1Dcw6cpYdC8LJ7+QfwnnnSts7zXwOC4sFjBNTs/fAlMLt1vVcWDmHqpfphsyQxBu
+d4bfOLl8jRyUyK0UuMRgXWDxcuq2yMQ/fsHe4ok1OZitiTTrVxNQ6bJJMIRlaHLE
+D5m6L17TWVIAcW10h3wV6t3QULI01ZqysHA/ptZ14rMCS9vnlCR4szg6YenShbY/
+XywFipOenmj4E/cL66NMJNowVfbhUE2+2FQRFTJGodHDDW2dGJ9sVZl6ou/S+iny
+HaKrHcSW6CfZpOZQ9RD0Y9EYFzYFmnDVlBllqbThAoIBAQCKSpYeJ80jzNg4IpKp
+9JgiTMQZzIL9O0TFXZGpLR24LGMbDc0/yw/aTGDaVUycU/vvKAnbQEZqjMXhn5mC
+hzlTPt0zdWQYR1/0lYjLtKs9rfhSIUffxUOQMbOVS4Q20GxQ2h02o8Zo8CtvuoPl
+9mkOYfFvEMaa/eOW6vFdyJA9Ikk+HGomHUORBA/hzuNl74EdNEArD7N6eVQIox7O
+RIQlIOZyyYXmODa+2SIxoqnpfJNF9GCWShy4jyfYSeoDqKcihWfQ5P3BuogBEsXc
+D0Z7k1JuBX0u8KuBKl8dc/+aFDQ6fhc93jpPdOjTYiuQkCm0gufshh2dY/IYNKmt
+p6AVAoIBAQCn7K42KDNrZLm5oXcP2QalcOmlxBvryq2VPBmKaFf8sTJVcKTqsPRv
+xfvMGJLbspjYmsWb5p7zPRAV9kU+patTeVCdV79kzlPjHHbcgEgRBs3QhzVT+j3T
+fOtT9AD/z/1o6WWMcsfVyFtYDWNswZirqBTW8bitaGz6XWx5pE1dOhOBmroz6nK4
+REf3TOHV2timWlZ9W+dgiR/vXeyxyn0thWwAS7V5RB8Zdf/nzoNh4IGfmqpOfuCW
+JO2PCzZGMM3szgtchhWNjru8TDtRQ/vs8XEAyOS1DFW49+29tTVGB7ROShOK1OWh
+NhYkB6YpGjwOqaxpE2b8eiqwFyLG9DrG
+-----END PRIVATE KEY-----

admissioncontroller/main.py

@@ -17,10 +17,16 @@ def deployment_webhook_mutate():
        if labels["x-infra-instance"]=="ondemand":
           return admission_response_patch(True, "Adding allow label", json_patch = jsonpatch.JsonPatch([{"op": "add", "path": "/spec/tolerations", "value": [{"effect":"NoExecute", "key":"workload-type", 
            "operator":"Equal", "value":"large-testnet"}]}, {"op":"add", "path":"/spec/nodeSelector", "value": {"nodetype":"large-network"}}]))
-       else:
+       elif labels["x-infra-instance"]=="ondemand-iops":
+          return admission_response_patch(True, "Adding allow label", json_patch = jsonpatch.JsonPatch([{"op": "add", "path": "/spec/tolerations", "value": [{"effect":"NoExecute", "key":"workload-type", 
+           "operator":"Equal", "value":"large-testnet-iops"}]}, {"op":"add", "path":"/spec/nodeSelector", "value": {"nodetype":"large-network-iops"}}]))
+       elif labels["x-infra-instance"]=="spot-iops":
+          return admission_response_patch(True, "Adding allow label", json_patch = jsonpatch.JsonPatch([{"op": "add", "path": "/spec/tolerations", "value": [{"effect":"NoExecute", "key":"workload-type", 
+           "operator":"Equal", "value":"xlarge-testnet-iops"}]}, {"op":"add", "path":"/spec/nodeSelector", "value": {"nodetype":"xlarge-network-iops"}}]))
+       else:    
           return admission_response_patch(True, "Adding allow label", json_patch = jsonpatch.JsonPatch([{"op": "add", "path": "/spec/tolerations", "value": [{"effect":"NoExecute", "key":"workload-type", 
            "operator":"Equal", "value":"xlarge-testnet"}]}, {"op":"add", "path":"/spec/nodeSelector", "value": {"nodetype":"xlarge-network"}}]))
-           
+
 #          return admission_response_patch(True, "Adding allow label", json_patch = jsonpatch.JsonPatch([{"op": "add", "path": "/spec/tolerations", "value": [{"operator":"Exists"}]}, {"op":"add", "path":"/spec/affinity", "value": {"nodeAffinity":{"preferredDuringSchedulingIgnoredDuringExecution":[{"weight":100,"preference":{"matchExpressions":[{"key":"nodetype","operator":"In","values":["xlarge-network"]}]}},{"weight":50,"preference":{"matchExpressions":[{"key":"nodetype","operator":"In","values":["large-network"]}]}}]}} }]))
 
     elif namespace.startswith("zombie-") and not hasNodeSelector:

javascript/packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@zombienet/cli",
-  "version": "1.3.95",
+  "version": "1.3.100",
   "description": "ZombieNet aim to be a testing framework for substrate based blockchains, providing a simple cli tool that allow users to spawn and test ephemeral Substrate based networks",
   "main": "dist/index.js",
   "scripts": {
@@ -54,8 +54,8 @@
   },
   "dependencies": {
     "@zombienet/dsl-parser-wrapper": "^0.1.10",
-    "@zombienet/orchestrator": "^0.0.77",
-    "@zombienet/utils": "^0.0.24",
+    "@zombienet/orchestrator": "^0.0.82",
+    "@zombienet/utils": "^0.0.25",
     "cli-progress": "^3.12.0",
     "commander": "^11.1.0",
     "debug": "^4.3.4",

javascript/packages/orchestrator/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@zombienet/orchestrator",
-  "version": "0.0.77",
+  "version": "0.0.82",
   "description": "ZombieNet aim to be a testing framework for substrate based blockchains, providing a simple cli tool that allow users to spawn and test ephemeral Substrate based networks",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -40,7 +40,7 @@
     "@polkadot/api": "^10.12.4",
     "@polkadot/keyring": "^12.6.2",
     "@polkadot/util-crypto": "^12.6.1",
-    "@zombienet/utils": "^0.0.24",
+    "@zombienet/utils": "^0.0.25",
     "chai": "^4.3.10",
     "debug": "^4.3.4",
     "execa": "^5.1.1",

javascript/packages/orchestrator/src/orchestrator.ts

@@ -14,6 +14,7 @@ import {
   series,
   setLogType,
   sleep,
+  registerSpawnElapsedTimeSecs,
 } from "@zombienet/utils";
 import fs from "fs";
 import tmp from "tmp-promise";
@@ -49,6 +50,7 @@ import { spawnNode } from "./spawner";
 import { setSubstrateCliArgsVersion } from "./substrateCliArgsHelper";
 import { ComputedNetwork, LaunchConfig } from "./configTypes";
 import { Node, Parachain } from "./sharedTypes";
+import { performance } from "perf_hooks";
 
 const debug = require("debug")("zombie");
 
@@ -74,7 +76,8 @@ export async function start(
   launchConfig: LaunchConfig,
   options?: OrcOptionsInterface,
 ) {
-  console.time("zombie_spawn");
+  const spawnStart = performance.now();
+
   const opts = {
     ...{
       monitor: false,
@@ -535,7 +538,11 @@ export async function start(
     debug(
       `\t 🚀 LAUNCH COMPLETE under namespace ${decorators.green(namespace)} 🚀`,
     );
-    console.timeEnd("zombie_spawn");
+
+    const spawnEnd = performance.now();
+    const spawnElapsedSecs = Math.round((spawnEnd - spawnStart) / 1000);
+    debug(`\t 🕰 [Spawn] elapsed time: ${spawnElapsedSecs} secs`);
+    if (options?.inCI) await registerSpawnElapsedTimeSecs(spawnElapsedSecs);
 
     // clean cache before dump the info.
     network.cleanMetricsCache();

javascript/packages/orchestrator/src/providers/k8s/dynResourceDefinition.ts

@@ -22,9 +22,10 @@ export async function genBootnodeDef(
 export async function genNodeDef(
   namespace: string,
   nodeSetup: Node,
+  inCI: boolean = false,
 ): Promise<any> {
   const nodeResource = new NodeResource(namespace, nodeSetup);
-  return nodeResource.generateSpec();
+  return nodeResource.generateSpec(inCI);
 }
 
 export function genChaosDef(

javascript/packages/orchestrator/src/providers/k8s/kubeClient.ts

@@ -61,6 +61,7 @@ export class KubeClient extends Client {
   localMagicFilepath: string;
   remoteDir: string;
   dataDir: string;
+  inCI: boolean;
 
   constructor(configPath: string, namespace: string, tmpDir: string) {
     super(configPath, namespace, tmpDir, "kubectl", "kubernetes");
@@ -72,6 +73,10 @@ export class KubeClient extends Client {
     this.localMagicFilepath = `${tmpDir}/finished.txt`;
     this.remoteDir = DEFAULT_REMOTE_DIR;
     this.dataDir = DEFAULT_DATA_DIR;
+    // Use the same env vars from spawn/run
+    this.inCI =
+      process.env.RUN_IN_CONTAINER === "1" ||
+      process.env.ZOMBIENET_IMAGE !== undefined;
   }
 
   async validateAccess(): Promise<boolean> {
@@ -729,6 +734,72 @@ export class KubeClient extends Client {
     since: number | undefined = undefined,
     withTimestamp = false,
   ): Promise<string> {
+    if (!this.inCI) {
+      // we can just return the logs from kube
+      const logs = await this.getNodeLogsFromKube(
+        podName,
+        since,
+        withTimestamp,
+      );
+      return logs;
+    }
+
+    // if we are running in CI, could be the case that k8s had rotate the logs,
+    // so the simple `kubectl logs` will retrieve only a part of them.
+    // We should read it from host filesystem to ensure we are reading all the logs.
+
+    // First get the logs files to check if we need to read from disk or not
+    const logFiles = await this.gzippedLogFiles(podName);
+    debug("logFiles", logFiles);
+    let logs = "";
+    if (logFiles.length === 0) {
+      logs = await this.getNodeLogsFromKube(podName, since, withTimestamp);
+    } else {
+      // need to read the files in order and accumulate in logs
+      const promises = logFiles.map((file) =>
+        this.readgzippedLogFile(podName, file),
+      );
+      const results = await Promise.all(promises);
+      for (const r of results) {
+        logs += r;
+      }
+
+      // now read the actual log from kube
+      logs += await this.getNodeLogsFromKube(podName);
+    }
+
+    return logs;
+  }
+
+  async gzippedLogFiles(podName: string): Promise<string[]> {
+    const [podId, podStatus, zombieRole] = await this.getPodInfo(podName);
+    debug("podId", podId);
+    debug("podStatus", podStatus);
+    debug("zombieRole", zombieRole);
+    // we can only get compressed files from `Running` and not temp pods
+    if (podStatus !== "Running" || zombieRole == "temp") return [];
+
+    // log dir in ci /var/log/pods/<nsName>_<podName>_<podId>/<podName>
+    const logsDir = `/var/log/pods/${this.namespace}_${podName}_${podId}/${podName}`;
+    // ls dir sorting asc one file per line (only compressed files)
+    // note: use coreutils here since some images (paras) doesn't have `ls`
+    const args = ["exec", podName, "--", "/cfg/coreutils", "ls", "-1", logsDir];
+    const result = await this.runCommand(args, {
+      scoped: true,
+      allowFail: false,
+    });
+
+    return result.stdout
+      .split("\n")
+      .filter((f) => f !== "0.log")
+      .map((fileName) => `${logsDir}/${fileName}`);
+  }
+
+  async getNodeLogsFromKube(
+    podName: string,
+    since: number | undefined = undefined,
+    withTimestamp = false,
+  ) {
     const args = ["logs"];
     if (since && since > 0) args.push(`--since=${since}s`);
     if (withTimestamp) args.push("--timestamps=true");
@@ -750,6 +821,33 @@ export class KubeClient extends Client {
     }
   }
 
+  async readgzippedLogFile(podName: string, file: string): Promise<string> {
+    const args = ["exec", podName, "--", "zcat", "-f", file];
+    const result = await this.runCommand(args, {
+      scoped: true,
+      allowFail: false,
+    });
+
+    return result.stdout;
+  }
+
+  async getPodInfo(podName: string): Promise<string[]> {
+    //  kubectl get pod <podName>  -n <nsName> -o jsonpath='{.metadata.uid}'
+    const args = [
+      "get",
+      "pod",
+      podName,
+      "-o",
+      'jsonpath={.metadata.uid}{","}{.status.phase}{","}{.metadata.labels.zombie-role}',
+    ];
+    const result = await this.runCommand(args, {
+      scoped: true,
+      allowFail: false,
+    });
+
+    return result.stdout.split(",");
+  }
+
   async dumpLogs(path: string, podName: string) {
     const dstFileName = `${path}/logs/${podName}.log`;
     const logs = await this.getNodeLogs(podName);
@@ -904,6 +1002,7 @@ export class KubeClient extends Client {
     debug(result);
     fileUploadCache[fileHash] = fileName;
   }
+
   getLogsCommand(name: string): string {
     return `kubectl logs -f ${name} -c ${name} -n ${this.namespace}`;
   }