diff --git a/javascript/packages/orchestrator/src/providers/k8s/dynResourceDefinition.ts b/javascript/packages/orchestrator/src/providers/k8s/dynResourceDefinition.ts index 2b1380e94..04f4ab19b 100644 --- a/javascript/packages/orchestrator/src/providers/k8s/dynResourceDefinition.ts +++ b/javascript/packages/orchestrator/src/providers/k8s/dynResourceDefinition.ts @@ -22,9 +22,10 @@ export async function genBootnodeDef( export async function genNodeDef( namespace: string, nodeSetup: Node, + inCI: boolean = false, ): Promise { const nodeResource = new NodeResource(namespace, nodeSetup); - return nodeResource.generateSpec(); + return nodeResource.generateSpec(inCI); } export function genChaosDef( diff --git a/javascript/packages/orchestrator/src/providers/k8s/kubeClient.ts b/javascript/packages/orchestrator/src/providers/k8s/kubeClient.ts index 62f967a4d..dd9b134c7 100644 --- a/javascript/packages/orchestrator/src/providers/k8s/kubeClient.ts +++ b/javascript/packages/orchestrator/src/providers/k8s/kubeClient.ts @@ -61,6 +61,7 @@ export class KubeClient extends Client { localMagicFilepath: string; remoteDir: string; dataDir: string; + inCI: boolean; constructor(configPath: string, namespace: string, tmpDir: string) { super(configPath, namespace, tmpDir, "kubectl", "kubernetes"); @@ -72,6 +73,10 @@ export class KubeClient extends Client { this.localMagicFilepath = `${tmpDir}/finished.txt`; this.remoteDir = DEFAULT_REMOTE_DIR; this.dataDir = DEFAULT_DATA_DIR; + // Use the same env vars from spawn/run + this.inCI = + process.env.RUN_IN_CONTAINER === "1" || + process.env.ZOMBIENET_IMAGE !== undefined; } async validateAccess(): Promise { @@ -728,6 +733,71 @@ export class KubeClient extends Client { since: number | undefined = undefined, withTimestamp = false, ): Promise { + if (!this.inCI) { + // we can just return the logs from kube + const logs = await this.getNodeLogsFromKube( + podName, + since, + withTimestamp, + ); + return logs; + } + + // if we are running in CI, could be the case that k8s had rotate the logs, + // so the simple `kubectl logs` will retrieve only a part of them. + // We should read it from host filesystem to ensure we are reading all the logs. + + // First get the logs files to check if we need to read from disk or not + const logFiles = await this.gzippedLogFiles(podName); + debug("logFiles", logFiles); + let logs = ""; + if (logFiles.length === 0) { + logs = await this.getNodeLogsFromKube(podName, since, withTimestamp); + } else { + // need to read the files in order and accumulate in logs + const promises = logFiles.map((file) => + this.readgzippedLogFile(podName, file), + ); + const results = await Promise.all(promises); + for (const r of results) { + logs += r; + } + + // now read the actual log from kube + logs += await this.getNodeLogsFromKube(podName); + } + + return logs; + } + + async gzippedLogFiles(podName: string): Promise { + const [podId, podStatus] = await this.getPodIdAndStatus(podName); + debug("podId", podId); + debug("podStatus", podStatus); + // we can only get compressed files from `Running` pods + if (podStatus !== "Running") return []; + + // log dir in ci /var/log/pods/__/ + const logsDir = `/var/log/pods/${this.namespace}_${podName}_${podId}/${podName}`; + // ls dir sorting asc one file per line (only compressed files) + // note: use coreutils here since some images (paras) doesn't have `ls` + const args = ["exec", podName, "--", "/cfg/coreutils", "ls", "-1", logsDir]; + const result = await this.runCommand(args, { + scoped: true, + allowFail: false, + }); + + return result.stdout + .split("\n") + .filter((f) => f !== "0.log") + .map((fileName) => `${logsDir}/${fileName}`); + } + + async getNodeLogsFromKube( + podName: string, + since: number | undefined = undefined, + withTimestamp = false, + ) { const args = ["logs"]; if (since && since > 0) args.push(`--since=${since}s`); if (withTimestamp) args.push("--timestamps=true"); @@ -749,6 +819,33 @@ export class KubeClient extends Client { } } + async readgzippedLogFile(podName: string, file: string): Promise { + const args = ["exec", podName, "--", "zcat", file]; + const result = await this.runCommand(args, { + scoped: true, + allowFail: false, + }); + + return result.stdout; + } + + async getPodIdAndStatus(podName: string): Promise { + // kubectl get pod -n -o jsonpath='{.metadata.uid}' + const args = [ + "get", + "pod", + podName, + "-o", + 'jsonpath={.metadata.uid}{","}{.status.phase}', + ]; + const result = await this.runCommand(args, { + scoped: true, + allowFail: false, + }); + + return result.stdout.split(","); + } + async dumpLogs(path: string, podName: string) { const dstFileName = `${path}/logs/${podName}.log`; const logs = await this.getNodeLogs(podName); @@ -903,6 +1000,7 @@ export class KubeClient extends Client { debug(result); fileUploadCache[fileHash] = fileName; } + getLogsCommand(name: string): string { return `kubectl logs -f ${name} -c ${name} -n ${this.namespace}`; } diff --git a/javascript/packages/orchestrator/src/providers/k8s/resources/nodeResource.ts b/javascript/packages/orchestrator/src/providers/k8s/resources/nodeResource.ts index ea62ca1c0..837775388 100644 --- a/javascript/packages/orchestrator/src/providers/k8s/resources/nodeResource.ts +++ b/javascript/packages/orchestrator/src/providers/k8s/resources/nodeResource.ts @@ -23,9 +23,11 @@ export class NodeResource { protected readonly nodeSetupConfig: Node, ) {} - public async generateSpec() { - const volumes = await this.generateVolumes(); - const volumeMounts = this.generateVolumesMounts(); + public async generateSpec(inCI: boolean = false) { + // DEBUG LOCAL + inCI = true; + const volumes = await this.generateVolumes(inCI); + const volumeMounts = this.generateVolumesMounts(inCI); const containersPorts = await this.generateContainersPorts(); const initContainers = this.generateInitContainers(); const containers = await this.generateContainers( @@ -33,23 +35,39 @@ export class NodeResource { containersPorts, ); - return this.generatePodSpec(initContainers, containers, volumes); + return this.generatePodSpec(initContainers, containers, volumes, inCI); } - private async generateVolumes(): Promise { - return [ + private async generateVolumes(inCI: boolean): Promise { + const volumes: Volume[] = [ { name: "tmp-cfg" }, { name: "tmp-data" }, { name: "tmp-relay-data" }, ]; + + if (inCI) + volumes.push({ + name: "pods", + hostPath: { path: "/var/log/pods", type: "" }, + }); + + return volumes; } - private generateVolumesMounts() { - return [ + private generateVolumesMounts(inCI: boolean) { + const volMount = [ { name: "tmp-cfg", mountPath: "/cfg", readOnly: false }, { name: "tmp-data", mountPath: "/data", readOnly: false }, { name: "tmp-relay-data", mountPath: "/relay-data", readOnly: false }, ]; + + if (inCI) + volMount.push({ + name: "pods", + mountPath: "/var/log/pods", + readOnly: true /* set to false for debugging */, + }); + return volMount; } private async generateContainersPorts(): Promise { @@ -174,6 +192,7 @@ export class NodeResource { initContainers: Container[], containers: Container[], volumes: Volume[], + inCI: boolean = false, ): PodSpec { const { name, zombieRole } = this.nodeSetupConfig; const zombieRoleLabel = this.computeZombieRoleLabel(); @@ -203,7 +222,7 @@ export class NodeResource { restartPolicy, volumes, securityContext: { - fsGroup: 1000, + fsGroup: inCI ? 0 : 1000, runAsUser: 1000, runAsGroup: 1000, }, diff --git a/javascript/packages/orchestrator/src/providers/k8s/resources/types.ts b/javascript/packages/orchestrator/src/providers/k8s/resources/types.ts index 61d190705..327750872 100644 --- a/javascript/packages/orchestrator/src/providers/k8s/resources/types.ts +++ b/javascript/packages/orchestrator/src/providers/k8s/resources/types.ts @@ -15,6 +15,7 @@ export interface VolumeMount { export interface Volume { name: string; + hostPath?: any; } export interface ContainerPort { diff --git a/javascript/packages/orchestrator/src/spawner.ts b/javascript/packages/orchestrator/src/spawner.ts index a6b77c103..f86a5b478 100644 --- a/javascript/packages/orchestrator/src/spawner.ts +++ b/javascript/packages/orchestrator/src/spawner.ts @@ -55,7 +55,7 @@ export const spawnNode = async ( debug(`creating node: ${node.name}`); const podDef = await (node.name === "bootnode" ? genBootnodeDef(namespace, node) - : genNodeDef(namespace, node)); + : genNodeDef(namespace, node, opts.inCI)); const finalFilesToCopyToNode = [...filesToCopy]; diff --git a/javascript/words.txt b/javascript/words.txt index 754a1cc05..ec91f5714 100644 --- a/javascript/words.txt +++ b/javascript/words.txt @@ -58,6 +58,8 @@ framesystem fullpaths genshiro gurke +gzipped +Gzipped hasher hashers hostpath @@ -138,6 +140,7 @@ propery protobuf rawdata rawmetric +readgzipped relaychain resourse runtimes @@ -177,6 +180,7 @@ willbe xcmp xinfra xzvf +zcat zipkin zndsl zombienet