Feat(k8s) read logs from host (in ci) (#1749)

* wip, init infra to access logs through the pod

* impl reading logs from disk

* pods vols readonly

* typo

* typo

* add words

* add words

* only read fs for Running pods

* lint

* use coreutils

* coreutils typo

javascript/packages/orchestrator/src/providers/k8s/dynResourceDefinition.ts

@@ -22,9 +22,10 @@ export async function genBootnodeDef(
 export async function genNodeDef(
   namespace: string,
   nodeSetup: Node,
+  inCI: boolean = false,
 ): Promise<any> {
   const nodeResource = new NodeResource(namespace, nodeSetup);
-  return nodeResource.generateSpec();
+  return nodeResource.generateSpec(inCI);
 }
 
 export function genChaosDef(

javascript/packages/orchestrator/src/providers/k8s/kubeClient.ts

@@ -61,6 +61,7 @@ export class KubeClient extends Client {
   localMagicFilepath: string;
   remoteDir: string;
   dataDir: string;
+  inCI: boolean;
 
   constructor(configPath: string, namespace: string, tmpDir: string) {
     super(configPath, namespace, tmpDir, "kubectl", "kubernetes");
@@ -72,6 +73,10 @@ export class KubeClient extends Client {
     this.localMagicFilepath = `${tmpDir}/finished.txt`;
     this.remoteDir = DEFAULT_REMOTE_DIR;
     this.dataDir = DEFAULT_DATA_DIR;
+    // Use the same env vars from spawn/run
+    this.inCI =
+      process.env.RUN_IN_CONTAINER === "1" ||
+      process.env.ZOMBIENET_IMAGE !== undefined;
   }
 
   async validateAccess(): Promise<boolean> {
@@ -729,6 +734,71 @@ export class KubeClient extends Client {
     since: number | undefined = undefined,
     withTimestamp = false,
   ): Promise<string> {
+    if (!this.inCI) {
+      // we can just return the logs from kube
+      const logs = await this.getNodeLogsFromKube(
+        podName,
+        since,
+        withTimestamp,
+      );
+      return logs;
+    }
+
+    // if we are running in CI, could be the case that k8s had rotate the logs,
+    // so the simple `kubectl logs` will retrieve only a part of them.
+    // We should read it from host filesystem to ensure we are reading all the logs.
+
+    // First get the logs files to check if we need to read from disk or not
+    const logFiles = await this.gzippedLogFiles(podName);
+    debug("logFiles", logFiles);
+    let logs = "";
+    if (logFiles.length === 0) {
+      logs = await this.getNodeLogsFromKube(podName, since, withTimestamp);
+    } else {
+      // need to read the files in order and accumulate in logs
+      const promises = logFiles.map((file) =>
+        this.readgzippedLogFile(podName, file),
+      );
+      const results = await Promise.all(promises);
+      for (const r of results) {
+        logs += r;
+      }
+
+      // now read the actual log from kube
+      logs += await this.getNodeLogsFromKube(podName);
+    }
+
+    return logs;
+  }
+
+  async gzippedLogFiles(podName: string): Promise<string[]> {
+    const [podId, podStatus] = await this.getPodIdAndStatus(podName);
+    debug("podId", podId);
+    debug("podStatus", podStatus);
+    // we can only get compressed files from `Running` pods
+    if (podStatus !== "Running") return [];
+
+    // log dir in ci /var/log/pods/<nsName>_<podName>_<podId>/<podName>
+    const logsDir = `/var/log/pods/${this.namespace}_${podName}_${podId}/${podName}`;
+    // ls dir sorting asc one file per line (only compressed files)
+    // note: use coreutils here since some images (paras) doesn't have `ls`
+    const args = ["exec", podName, "--", "/cfg/coreutils", "ls", "-1", logsDir];
+    const result = await this.runCommand(args, {
+      scoped: true,
+      allowFail: false,
+    });
+
+    return result.stdout
+      .split("\n")
+      .filter((f) => f !== "0.log")
+      .map((fileName) => `${logsDir}/${fileName}`);
+  }
+
+  async getNodeLogsFromKube(
+    podName: string,
+    since: number | undefined = undefined,
+    withTimestamp = false,
+  ) {
     const args = ["logs"];
     if (since && since > 0) args.push(`--since=${since}s`);
     if (withTimestamp) args.push("--timestamps=true");
@@ -750,6 +820,33 @@ export class KubeClient extends Client {
     }
   }
 
+  async readgzippedLogFile(podName: string, file: string): Promise<string> {
+    const args = ["exec", podName, "--", "zcat", file];
+    const result = await this.runCommand(args, {
+      scoped: true,
+      allowFail: false,
+    });
+
+    return result.stdout;
+  }
+
+  async getPodIdAndStatus(podName: string): Promise<string[]> {
+    //  kubectl get pod <podName>  -n <nsName> -o jsonpath='{.metadata.uid}'
+    const args = [
+      "get",
+      "pod",
+      podName,
+      "-o",
+      'jsonpath={.metadata.uid}{","}{.status.phase}',
+    ];
+    const result = await this.runCommand(args, {
+      scoped: true,
+      allowFail: false,
+    });
+
+    return result.stdout.split(",");
+  }
+
   async dumpLogs(path: string, podName: string) {
     const dstFileName = `${path}/logs/${podName}.log`;
     const logs = await this.getNodeLogs(podName);
@@ -904,6 +1001,7 @@ export class KubeClient extends Client {
     debug(result);
     fileUploadCache[fileHash] = fileName;
   }
+
   getLogsCommand(name: string): string {
     return `kubectl logs -f ${name} -c ${name} -n ${this.namespace}`;
   }

javascript/packages/orchestrator/src/providers/k8s/resources/nodeResource.ts

@@ -23,33 +23,51 @@ export class NodeResource {
     protected readonly nodeSetupConfig: Node,
   ) {}
 
-  public async generateSpec() {
-    const volumes = await this.generateVolumes();
-    const volumeMounts = this.generateVolumesMounts();
+  public async generateSpec(inCI: boolean = false) {
+    // DEBUG LOCAL
+    inCI = true;
+    const volumes = await this.generateVolumes(inCI);
+    const volumeMounts = this.generateVolumesMounts(inCI);
     const containersPorts = await this.generateContainersPorts();
     const initContainers = this.generateInitContainers();
     const containers = await this.generateContainers(
       volumeMounts,
       containersPorts,
     );
 
-    return this.generatePodSpec(initContainers, containers, volumes);
+    return this.generatePodSpec(initContainers, containers, volumes, inCI);
   }
 
-  private async generateVolumes(): Promise<Volume[]> {
-    return [
+  private async generateVolumes(inCI: boolean): Promise<Volume[]> {
+    const volumes: Volume[] = [
       { name: "tmp-cfg" },
       { name: "tmp-data" },
       { name: "tmp-relay-data" },
     ];
+
+    if (inCI)
+      volumes.push({
+        name: "pods",
+        hostPath: { path: "/var/log/pods", type: "" },
+      });
+
+    return volumes;
   }
 
-  private generateVolumesMounts() {
-    return [
+  private generateVolumesMounts(inCI: boolean) {
+    const volMount = [
       { name: "tmp-cfg", mountPath: "/cfg", readOnly: false },
       { name: "tmp-data", mountPath: "/data", readOnly: false },
       { name: "tmp-relay-data", mountPath: "/relay-data", readOnly: false },
     ];
+
+    if (inCI)
+      volMount.push({
+        name: "pods",
+        mountPath: "/var/log/pods",
+        readOnly: true /* set to false for debugging */,
+      });
+    return volMount;
   }
 
   private async generateContainersPorts(): Promise<ContainerPort[]> {
@@ -168,6 +186,7 @@ export class NodeResource {
     initContainers: Container[],
     containers: Container[],
     volumes: Volume[],
+    inCI: boolean = false,
   ): PodSpec {
     const { name, zombieRole } = this.nodeSetupConfig;
     const zombieRoleLabel = this.computeZombieRoleLabel();
@@ -197,7 +216,7 @@ export class NodeResource {
         restartPolicy,
         volumes,
         securityContext: {
-          fsGroup: 1000,
+          fsGroup: inCI ? 0 : 1000,
           runAsUser: 1000,
           runAsGroup: 1000,
         },

javascript/packages/orchestrator/src/providers/k8s/resources/types.ts

@@ -15,6 +15,7 @@ export interface VolumeMount {
 
 export interface Volume {
   name: string;
+  hostPath?: any;
 }
 
 export interface ContainerPort {

javascript/packages/orchestrator/src/spawner.ts

@@ -55,7 +55,7 @@ export const spawnNode = async (
   debug(`creating node: ${node.name}`);
   const podDef = await (node.name === "bootnode"
     ? genBootnodeDef(namespace, node)
-    : genNodeDef(namespace, node));
+    : genNodeDef(namespace, node, opts.inCI));
 
   const finalFilesToCopyToNode = [...filesToCopy];
 

javascript/words.txt

@@ -58,6 +58,8 @@ framesystem
 fullpaths
 genshiro
 gurke
+gzipped
+Gzipped
 hasher
 hashers
 hostpath
@@ -138,6 +140,7 @@ propery
 protobuf
 rawdata
 rawmetric
+readgzipped
 relaychain
 resourse
 runtimes
@@ -177,6 +180,7 @@ willbe
 xcmp
 xinfra
 xzvf
+zcat
 zipkin
 zndsl
 zombienet