feat(xtask): add ci subcommand

Signed-off-by: Martin Kröning <[email protected]>

xtask/Cargo.toml

@@ -8,4 +8,7 @@ anyhow = "1.0"
 clap = { version = "4", features = ["derive"] }
 goblin = { version = "0.7", default-features = false, features = ["archive", "elf32", "elf64", "std"] }
 llvm-tools = "0.1"
+sysinfo = "0.29"
+ureq = "2"
+wait-timeout = "0.2"
 xshell = "0.2"

xtask/src/arch.rs

@@ -60,6 +60,19 @@ impl Arch {
 		}
 	}
 
+	pub fn ci_cargo_args(&self) -> &'static [&'static str] {
+		match self {
+			Arch::X86_64 => &[
+				"--target=x86_64-unknown-hermit",
+				"-Zbuild-std=std,panic_abort",
+			],
+			Arch::Aarch64 => &[
+				"--target=aarch64-unknown-hermit",
+				"-Zbuild-std=std,panic_abort",
+			],
+		}
+	}
+
 	pub fn rustflags(&self) -> &'static [&'static str] {
 		match self {
 			Self::X86_64 => &[],

xtask/src/artifact.rs

@@ -80,4 +80,16 @@ impl Artifact {
 		.collect::<PathBuf>()
 		.into()
 	}
+
+	pub fn ci_image(&self, package: &str) -> PathBuf {
+		[
+			"..".as_ref(),
+			self.target_dir(),
+			self.arch.hermit_triple().as_ref(),
+			self.profile_path_component().as_ref(),
+			package.as_ref(),
+		]
+		.iter()
+		.collect()
+	}
 }

xtask/src/ci/build.rs

@@ -0,0 +1,34 @@
+use std::path::PathBuf;
+
+use anyhow::Result;
+use clap::Args;
+use xshell::cmd;
+
+use crate::cargo_build::{CargoBuild, CmdExt};
+
+#[derive(Args)]
+pub struct Build {
+	#[command(flatten)]
+	pub cargo_build: CargoBuild,
+
+	#[arg(short, long)]
+	pub package: String,
+}
+
+impl Build {
+	pub fn run(&self) -> Result<()> {
+		let sh = crate::sh()?;
+
+		cmd!(sh, "cargo build --manifest-path ../Cargo.toml")
+			.args(self.cargo_build.artifact.arch.ci_cargo_args())
+			.cargo_build_args(&self.cargo_build)
+			.args(&["--package", self.package.as_str()])
+			.run()?;
+
+		Ok(())
+	}
+
+	pub fn image(&self) -> PathBuf {
+		self.cargo_build.artifact.ci_image(&self.package)
+	}
+}

xtask/src/ci/firecracker.rs

@@ -0,0 +1,41 @@
+use std::path::Path;
+
+use anyhow::Result;
+use clap::Args;
+use xshell::cmd;
+
+use super::build::Build;
+
+#[derive(Args)]
+pub struct Firecracker {
+	#[command(flatten)]
+	build: Build,
+}
+
+impl Firecracker {
+	pub fn run(self) -> Result<()> {
+		self.build.run()?;
+
+		let sh = crate::sh()?;
+
+		let config = format!(
+			include_str!("firecracker_vm_config.json"),
+			kernel_image_path = "../rusty-loader-x86_64-fc",
+			initrd_path = self.build.image().display()
+		);
+		eprintln!("firecracker config");
+		eprintln!("{config}");
+		let config_path = Path::new("firecracker_vm_config.json");
+		sh.write_file(config_path, config)?;
+
+		let log_path = Path::new("firecracker.log");
+		sh.write_file(log_path, "")?;
+		cmd!(sh, "firecracker --no-api --config-file {config_path} --log-path {log_path} --level Info --show-level --show-log-origin").run()?;
+		let log = sh.read_file(log_path)?;
+
+		eprintln!("firecracker log");
+		eprintln!("{log}");
+
+		Ok(())
+	}
+}

xtask/src/ci/firecracker_vm_config.json

@@ -0,0 +1,13 @@
+{{
+  "boot-source": {{
+    "kernel_image_path": "{kernel_image_path}",
+    "initrd_path": "{initrd_path}",
+    "boot_args": ""
+  }},
+  "drives": [],
+  "machine-config": {{
+    "vcpu_count": 1,
+    "mem_size_mib": 256,
+    "smt": false
+  }}
+}}

xtask/src/ci/mod.rs

@@ -0,0 +1,26 @@
+use anyhow::Result;
+use clap::Subcommand;
+
+mod build;
+mod firecracker;
+mod qemu;
+mod uhyve;
+
+#[derive(Subcommand)]
+pub enum Ci {
+	Build(build::Build),
+	Firecracker(firecracker::Firecracker),
+	Qemu(qemu::Qemu),
+	Uhyve(uhyve::Uhyve),
+}
+
+impl Ci {
+	pub fn run(self) -> Result<()> {
+		match self {
+			Self::Build(build) => build.run(),
+			Self::Firecracker(firecracker) => firecracker.run(),
+			Self::Qemu(qemu) => qemu.run(),
+			Self::Uhyve(uhyve) => uhyve.run(),
+		}
+	}
+}

xtask/src/ci/qemu.rs

@@ -0,0 +1,214 @@
+use std::net::UdpSocket;
+use std::path::Path;
+use std::process::{Child, Command, ExitStatus};
+use std::thread;
+use std::time::Duration;
+
+use anyhow::{anyhow, ensure, Result};
+use clap::{Args, ValueEnum};
+use sysinfo::{CpuExt, CpuRefreshKind, System, SystemExt};
+use wait_timeout::ChildExt;
+use xshell::cmd;
+
+use super::build::Build;
+use crate::arch::Arch;
+
+#[derive(Args)]
+pub struct Qemu {
+	#[command(flatten)]
+	build: Build,
+
+	#[arg(long)]
+	accel: bool,
+
+	#[arg(long)]
+	smp: bool,
+
+	#[arg(long)]
+	microvm: bool,
+
+	#[arg(long)]
+	netdev: Option<NetworkDevice>,
+
+	#[arg(long)]
+	virtiofsd: bool,
+}
+
+#[derive(ValueEnum, Clone, Copy)]
+pub enum NetworkDevice {
+	VirtioNetPci,
+	Rtl8139,
+}
+
+trait ExitStatusExt {
+	fn qemu_success(&self) -> bool;
+}
+
+impl ExitStatusExt for ExitStatus {
+	fn qemu_success(&self) -> bool {
+		self.success() || self.code() == Some(3)
+	}
+}
+
+struct KillChildOnDrop(Child);
+
+impl Drop for KillChildOnDrop {
+	fn drop(&mut self) {
+		self.0.kill().ok();
+	}
+}
+
+impl Qemu {
+	pub fn run(self) -> Result<()> {
+		self.build.run()?;
+
+		let sh = crate::sh()?;
+
+		let virtiofsd = self.virtiofsd.then(|| self.spawn_virtiofsd()).transpose()?;
+
+		let arch = self.build.cargo_build.artifact.arch.name();
+
+		let cmd = cmd!(sh, "qemu-system-{arch}")
+			.args(&["-display", "none"])
+			.args(&["-serial", "stdio"])
+			.args(&["-kernel", format!("../rusty-loader-{arch}").as_ref()]);
+
+		let cmd = if self.smp {
+			cmd.args(&["-smp", "4"]).args(&["-m", "4G"])
+		} else {
+			cmd.args(&["-smp", "1"]).args(&["-m", "1G"])
+		};
+
+		let cmd = if self.microvm {
+			let mut sys = System::new();
+			sys.refresh_cpu_specifics(CpuRefreshKind::new().with_frequency());
+			let frequency = sys.cpus().first().unwrap().frequency();
+			assert!(sys.cpus().iter().all(|cpu| cpu.frequency() == frequency));
+
+			cmd.args(&["-M", "microvm,x-option-roms=off,pit=off,pic=off,rtc=on"])
+				.args(&["-global", "virtio-mmio.force-legacy=on"])
+				.arg("-nodefaults")
+				.arg("-no-user-config")
+				.args(&["-append", format!("-freq {frequency}").as_str()])
+		} else {
+			cmd
+		};
+
+		let cmd = match self.netdev {
+			Some(NetworkDevice::VirtioNetPci) => cmd
+				.args(&[
+					"-netdev",
+					"user,id=u1,hostfwd=tcp::9975-:9975,hostfwd=udp::9975-:9975,net=192.168.76.0/24,dhcpstart=192.168.76.9",
+				])
+				.args(&["-device", "virtio-net-pci,netdev=u1,disable-legacy=on"]),
+			Some(NetworkDevice::Rtl8139) => cmd
+				.args(&[
+					"-netdev",
+					"user,id=u1,hostfwd=tcp::9975-:9975,hostfwd=udp::9975-:9975,net=192.168.76.0/24,dhcpstart=192.168.76.9",
+				])
+				.args(&["-device", "rtl8139,netdev=u1"]),
+			None => cmd,
+		};
+
+		let cmd = if self.virtiofsd {
+			cmd.args(&["-chardev", "socket,id=char0,path=./vhostqemu"])
+				.args(&[
+					"-device",
+					"vhost-user-fs-pci,queue-size=1024,chardev=char0,tag=root",
+				])
+				.args(&[
+					"-object",
+					"memory-backend-file,id=mem,size=1G,mem-path=/dev/shm,share=on",
+				])
+				.args(&["-numa", "node,memdev=mem"])
+		} else {
+			cmd
+		};
+
+		let cmd = match self.build.cargo_build.artifact.arch {
+			Arch::X86_64 => {
+				let cpu_args = if self.accel {
+					if cfg!(target_os = "linux") {
+						&["-enable-kvm", "-cpu", "host"][..]
+					} else {
+						todo!()
+					}
+				} else {
+					&["-cpu", "Skylake-Client"][..]
+				};
+				cmd.args(cpu_args)
+					.args(&["-device", "isa-debug-exit,iobase=0xf4,iosize=0x04"])
+					.args::<&[&Path]>(&["-initrd".as_ref(), self.build.image().as_ref()])
+			}
+			Arch::Aarch64 => {
+				if self.accel {
+					todo!()
+				}
+				cmd.arg("-semihosting")
+					.args(&["-machine", "virt,gic-version=3"])
+					.args(&["-cpu", "cortex-a72"])
+					.args::<&[&Path]>(&[
+						"-device".as_ref(),
+						format!(
+							"guest-loader,addr=0x48000000,initrd={}",
+							self.build.image().display()
+						)
+						.as_ref(),
+					])
+			}
+		};
+
+		eprintln!("$ {cmd}");
+		let mut child = KillChildOnDrop(Command::from(cmd).spawn()?);
+
+		thread::sleep(Duration::from_millis(100));
+		if let Some(status) = child.0.try_wait()? {
+			ensure!(status.qemu_success(), "QEMU exit code: {:?}", status.code());
+		}
+
+		if self.build.package == "httpd" {
+			thread::sleep(Duration::from_secs(5));
+			eprintln!("[CI] GET http://127.0.0.1:9975");
+			let body = ureq::get("http://127.0.0.1:9975")
+				.timeout(Duration::from_secs(3))
+				.call()?
+				.into_string()?;
+			eprintln!("[CI] {body}");
+			assert_eq!(body.lines().next(), Some("Hello from Hermit! 🦀"));
+		}
+
+		if self.build.package == "testudp" {
+			thread::sleep(Duration::from_secs(5));
+			let buf = "exit";
+			eprintln!("[CI] send {buf:?} via UDP to 127.0.0.1:9975");
+			let socket = UdpSocket::bind("127.0.0.1:0")?;
+			socket.connect("127.0.0.1:9975")?;
+			socket.send(buf.as_bytes())?;
+		}
+
+		let status = child
+			.0
+			.wait_timeout(Duration::from_secs(60 * 2))?
+			.ok_or(anyhow!("QEMU timeout"))?;
+		ensure!(status.qemu_success(), "QEMU exit code: {:?}", status.code());
+
+		if let Some(mut virtiofsd) = virtiofsd {
+			let status = virtiofsd.wait()?;
+			assert!(status.success());
+		}
+
+		Ok(())
+	}
+
+	fn spawn_virtiofsd(&self) -> Result<Child> {
+		let sh = crate::sh()?;
+
+		sh.create_dir("foo")?;
+
+		let cmd = cmd!(sh, "virtiofsd --socket-path=./vhostqemu --shared-dir ./foo --announce-submounts --sandbox none --seccomp none --inode-file-handles=never");
+
+		eprintln!("$ {cmd}");
+
+		Ok(Command::from(cmd).spawn()?)
+	}
+}