From 1730e1f936fbc0a6022feaa25baebdf190a21c49 Mon Sep 17 00:00:00 2001 From: Simon LUCIDO Date: Sat, 25 Feb 2023 18:27:16 +0100 Subject: [PATCH 01/13] feat: add virtio mmio device manager Signed-off-by: Simon LUCIDO --- Cargo.lock | 5 ++++- src/vmm/Cargo.lock | 5 ++++- src/vmm/Cargo.toml | 2 +- src/vmm/src/cpu/mod.rs | 32 +++++++++++++++++++++++++++++++- src/vmm/src/lib.rs | 12 ++++++++++-- 5 files changed, 50 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index eb4446f..879e472 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -274,7 +274,10 @@ checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" [[package]] name = "vm-device" version = "0.1.0" -source = "git+https://github.com/rust-vmm/vm-device?rev=5847f12#5847f1286492b7191f1400e6647fb220f8941f89" +source = "git+https://github.com/lucido-simon/vm-device?rev=5619cec076594881ddf8a38978c50923b3434494#5619cec076594881ddf8a38978c50923b3434494" +dependencies = [ + "vm-memory", +] [[package]] name = "vm-memory" diff --git a/src/vmm/Cargo.lock b/src/vmm/Cargo.lock index 04178bf..449c89f 100644 --- a/src/vmm/Cargo.lock +++ b/src/vmm/Cargo.lock @@ -56,7 +56,10 @@ dependencies = [ [[package]] name = "vm-device" version = "0.1.0" -source = "git+https://github.com/rust-vmm/vm-device?rev=5847f12#5847f1286492b7191f1400e6647fb220f8941f89" +source = "git+https://github.com/lucido-simon/vm-device?rev=5619cec076594881ddf8a38978c50923b3434494#5619cec076594881ddf8a38978c50923b3434494" +dependencies = [ + "vm-memory", +] [[package]] name = "vm-memory" diff --git a/src/vmm/Cargo.toml b/src/vmm/Cargo.toml index 54c5c37..dca36dc 100644 --- a/src/vmm/Cargo.toml +++ b/src/vmm/Cargo.toml @@ -15,6 +15,6 @@ vmm-sys-util = "0.11.1" # vm-device is not yet published on crates.io. # To make sure that breaking changes to vm-device are not breaking the # vm-vcpu build, we're using a fixed revision. -vm-device = { git = "https://github.com/rust-vmm/vm-device", rev = "5847f12" } +vm-device = { git = "https://github.com/lucido-simon/vm-device", rev = "5619cec076594881ddf8a38978c50923b3434494" } vm-superio = "0.7.0" diff --git a/src/vmm/src/cpu/mod.rs b/src/vmm/src/cpu/mod.rs index d81fdcf..440a6c5 100644 --- a/src/vmm/src/cpu/mod.rs +++ b/src/vmm/src/cpu/mod.rs @@ -8,6 +8,7 @@ use std::{result, u64}; use kvm_bindings::{kvm_fpu, kvm_regs, CpuId}; use kvm_ioctls::{VcpuExit, VcpuFd, VmFd}; +use vm_device::device_manager::{IoManager, MmioManager}; use vm_memory::{Address, Bytes, GuestAddress, GuestMemoryError, GuestMemoryMmap}; use vmm_sys_util::terminal::Terminal; @@ -66,15 +67,22 @@ pub(crate) struct Vcpu { pub vcpu_fd: VcpuFd, serial: Arc>, + virtio_manager: Arc>, } impl Vcpu { /// Create a new vCPU. - pub fn new(vm_fd: &VmFd, index: u64, serial: Arc>) -> Result { + pub fn new( + vm_fd: &VmFd, + index: u64, + serial: Arc>, + virtio_manager: Arc>, + ) -> Result { Ok(Vcpu { index, vcpu_fd: vm_fd.create_vcpu(index).map_err(Error::KvmIoctl)?, serial, + virtio_manager, }) } @@ -266,10 +274,32 @@ impl Vcpu { println!("Unsupported device read at {:x?}", addr); } }, + + // This is a MMIO write, i.e. the guest is trying to write + // something to a memory-mapped I/O region. + VcpuExit::MmioWrite(addr, data) => { + self.virtio_manager + .lock() + .unwrap() + .mmio_write(GuestAddress(addr), data) + .unwrap(); + } + + // This is a MMIO read, i.e. the guest is trying to read + // from a memory-mapped I/O region. + VcpuExit::MmioRead(addr, data) => { + self.virtio_manager + .lock() + .unwrap() + .mmio_read(GuestAddress(addr), data) + .unwrap(); + } + _ => { eprintln!("Unhandled VM-Exit: {:?}", exit_reason); } }, + Err(e) => eprintln!("Emulation error: {}", e), } } diff --git a/src/vmm/src/lib.rs b/src/vmm/src/lib.rs index 48ffcd6..817f947 100644 --- a/src/vmm/src/lib.rs +++ b/src/vmm/src/lib.rs @@ -19,6 +19,7 @@ use std::fs::File; use kvm_bindings::{kvm_userspace_memory_region, KVM_MAX_CPUID_ENTRIES}; use kvm_ioctls::{Kvm, VmFd}; use linux_loader::loader::{self, KernelLoaderResult}; +use vm_device::device_manager::IoManager; use vm_memory::{Address, GuestAddress, GuestMemory, GuestMemoryMmap, GuestMemoryRegion}; use vmm_sys_util::terminal::Terminal; mod cpu; @@ -80,6 +81,7 @@ pub struct VMM { vcpus: Vec, serial: Arc>, + virtio_manager: Arc>, epoll: EpollContext, } @@ -104,6 +106,7 @@ impl VMM { serial: Arc::new(Mutex::new( LumperSerial::new(Box::new(stdout())).map_err(Error::SerialCreation)?, )), + virtio_manager: Arc::new(Mutex::new(IoManager::new())), epoll, }; @@ -195,8 +198,13 @@ impl VMM { .map_err(Error::KvmIoctl)?; for index in 0..num_vcpus { - let vcpu = Vcpu::new(&self.vm_fd, index.into(), Arc::clone(&self.serial)) - .map_err(Error::Vcpu)?; + let vcpu = Vcpu::new( + &self.vm_fd, + index.into(), + Arc::clone(&self.serial), + Arc::clone(&self.virtio_manager), + ) + .map_err(Error::Vcpu)?; // Set CPUID. let mut vcpu_cpuid = base_cpuid.clone(); From 684066acb227e019750e3df3dd132e869abf9ad5 Mon Sep 17 00:00:00 2001 From: Simon LUCIDO Date: Fri, 24 Mar 2023 16:51:06 +0100 Subject: [PATCH 02/13] feat: add interface trait Signed-off-by: Simon LUCIDO --- Cargo.lock | 32 ++++++++++++++++++++++++++++ src/vmm/Cargo.lock | 32 ++++++++++++++++++++++++++++ src/vmm/Cargo.toml | 1 + src/vmm/src/devices/mod.rs | 1 + src/vmm/src/devices/net/interface.rs | 13 +++++++++++ src/vmm/src/devices/net/mod.rs | 23 ++++++++++++++++++++ 6 files changed, 102 insertions(+) create mode 100644 src/vmm/src/devices/net/interface.rs create mode 100644 src/vmm/src/devices/net/mod.rs diff --git a/Cargo.lock b/Cargo.lock index 879e472..734bd13 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -14,6 +14,12 @@ version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + [[package]] name = "clap" version = "4.1.4" @@ -157,6 +163,15 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f051f77a7c8e6957c0696eac88f26b0117e54f52d3fc682ab19397a8812846a4" +[[package]] +name = "log" +version = "0.4.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" +dependencies = [ + "cfg-if", +] + [[package]] name = "lumper" version = "0.1.0" @@ -271,6 +286,22 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +[[package]] +name = "virtio-bindings" +version = "0.2.0" +source = "git+https://github.com/rust-vmm/vm-virtio#467c8ec99375a5f4e08b85b18257cd7e0bac1dc0" + +[[package]] +name = "virtio-queue" +version = "0.7.1" +source = "git+https://github.com/rust-vmm/vm-virtio#467c8ec99375a5f4e08b85b18257cd7e0bac1dc0" +dependencies = [ + "log", + "virtio-bindings", + "vm-memory", + "vmm-sys-util", +] + [[package]] name = "vm-device" version = "0.1.0" @@ -304,6 +335,7 @@ dependencies = [ "kvm-ioctls", "libc", "linux-loader", + "virtio-queue", "vm-device", "vm-memory", "vm-superio", diff --git a/src/vmm/Cargo.lock b/src/vmm/Cargo.lock index 449c89f..09f5a6c 100644 --- a/src/vmm/Cargo.lock +++ b/src/vmm/Cargo.lock @@ -8,6 +8,12 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + [[package]] name = "epoll" version = "4.3.1" @@ -53,6 +59,31 @@ dependencies = [ "vm-memory", ] +[[package]] +name = "log" +version = "0.4.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "virtio-bindings" +version = "0.2.0" +source = "git+https://github.com/rust-vmm/vm-virtio#467c8ec99375a5f4e08b85b18257cd7e0bac1dc0" + +[[package]] +name = "virtio-queue" +version = "0.7.1" +source = "git+https://github.com/rust-vmm/vm-virtio#467c8ec99375a5f4e08b85b18257cd7e0bac1dc0" +dependencies = [ + "log", + "virtio-bindings", + "vm-memory", + "vmm-sys-util", +] + [[package]] name = "vm-device" version = "0.1.0" @@ -86,6 +117,7 @@ dependencies = [ "kvm-ioctls", "libc", "linux-loader", + "virtio-queue", "vm-device", "vm-memory", "vm-superio", diff --git a/src/vmm/Cargo.toml b/src/vmm/Cargo.toml index dca36dc..e6a738b 100644 --- a/src/vmm/Cargo.toml +++ b/src/vmm/Cargo.toml @@ -16,5 +16,6 @@ vmm-sys-util = "0.11.1" # To make sure that breaking changes to vm-device are not breaking the # vm-vcpu build, we're using a fixed revision. vm-device = { git = "https://github.com/lucido-simon/vm-device", rev = "5619cec076594881ddf8a38978c50923b3434494" } +virtio-queue = { git = "https://github.com/rust-vmm/vm-virtio" } vm-superio = "0.7.0" diff --git a/src/vmm/src/devices/mod.rs b/src/vmm/src/devices/mod.rs index 38db994..ea91bcb 100644 --- a/src/vmm/src/devices/mod.rs +++ b/src/vmm/src/devices/mod.rs @@ -1,3 +1,4 @@ // SPDX-License-Identifier: Apache-2.0 +pub(crate) mod net; pub(crate) mod serial; diff --git a/src/vmm/src/devices/net/interface.rs b/src/vmm/src/devices/net/interface.rs new file mode 100644 index 0000000..992767d --- /dev/null +++ b/src/vmm/src/devices/net/interface.rs @@ -0,0 +1,13 @@ +use std::{ + io::{Read, Write}, + os::fd::AsRawFd, +}; + +use super::Result; + +pub trait Interface: Read + Write + AsRawFd + Send + Sync { + fn activate(&self, virtio_flags: u64, virtio_header_size: usize) -> Result<()>; + fn open_named(if_name: &str) -> Result + where + Self: Sized; +} diff --git a/src/vmm/src/devices/net/mod.rs b/src/vmm/src/devices/net/mod.rs new file mode 100644 index 0000000..d068a07 --- /dev/null +++ b/src/vmm/src/devices/net/mod.rs @@ -0,0 +1,23 @@ +use std::{error::Error, fmt::Display}; + +pub mod interface; + +#[derive(Debug)] +#[allow(dead_code)] +pub enum VirtioNetError { + InvalidIfname, + VirtioQueueError(virtio_queue::Error), + IoCtlError(std::io::Error), + IoError(std::io::Error), + MemoryError(vm_memory::GuestMemoryError), + QueueError(virtio_queue::Error), +} +impl Error for VirtioNetError {} +impl Display for VirtioNetError { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "virtio net error") + } +} + +#[allow(dead_code)] +pub type Result = std::result::Result; From cad6213781640ed9cddc91c74c4069b97a5e90bc Mon Sep 17 00:00:00 2001 From: Simon LUCIDO Date: Fri, 24 Mar 2023 17:00:01 +0100 Subject: [PATCH 03/13] feat: add tap Signed-off-by: Simon LUCIDO --- Cargo.lock | 9 +- src/vmm/Cargo.lock | 9 +- src/vmm/Cargo.toml | 1 + src/vmm/src/devices/net/bindings.rs | 265 ++++++++++++++++++++++++++++ src/vmm/src/devices/net/mod.rs | 2 + src/vmm/src/devices/net/tap.rs | 188 ++++++++++++++++++++ 6 files changed, 472 insertions(+), 2 deletions(-) create mode 100644 src/vmm/src/devices/net/bindings.rs create mode 100644 src/vmm/src/devices/net/tap.rs diff --git a/Cargo.lock b/Cargo.lock index 734bd13..73bedb0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -286,6 +286,12 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +[[package]] +name = "virtio-bindings" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b9084faf91b9aa9676ae2cac8f1432df2839d9566e6f19f29dbc13a8b831dff" + [[package]] name = "virtio-bindings" version = "0.2.0" @@ -297,7 +303,7 @@ version = "0.7.1" source = "git+https://github.com/rust-vmm/vm-virtio#467c8ec99375a5f4e08b85b18257cd7e0bac1dc0" dependencies = [ "log", - "virtio-bindings", + "virtio-bindings 0.2.0 (git+https://github.com/rust-vmm/vm-virtio)", "vm-memory", "vmm-sys-util", ] @@ -335,6 +341,7 @@ dependencies = [ "kvm-ioctls", "libc", "linux-loader", + "virtio-bindings 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "virtio-queue", "vm-device", "vm-memory", diff --git a/src/vmm/Cargo.lock b/src/vmm/Cargo.lock index 09f5a6c..b50fd4d 100644 --- a/src/vmm/Cargo.lock +++ b/src/vmm/Cargo.lock @@ -68,6 +68,12 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "virtio-bindings" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b9084faf91b9aa9676ae2cac8f1432df2839d9566e6f19f29dbc13a8b831dff" + [[package]] name = "virtio-bindings" version = "0.2.0" @@ -79,7 +85,7 @@ version = "0.7.1" source = "git+https://github.com/rust-vmm/vm-virtio#467c8ec99375a5f4e08b85b18257cd7e0bac1dc0" dependencies = [ "log", - "virtio-bindings", + "virtio-bindings 0.2.0 (git+https://github.com/rust-vmm/vm-virtio)", "vm-memory", "vmm-sys-util", ] @@ -117,6 +123,7 @@ dependencies = [ "kvm-ioctls", "libc", "linux-loader", + "virtio-bindings 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "virtio-queue", "vm-device", "vm-memory", diff --git a/src/vmm/Cargo.toml b/src/vmm/Cargo.toml index e6a738b..88532d5 100644 --- a/src/vmm/Cargo.toml +++ b/src/vmm/Cargo.toml @@ -11,6 +11,7 @@ libc = "0.2.91" linux-loader = { version = "0.8.1", features = ["bzimage", "elf"] } vm-memory = { version = "0.10.0", features = ["backend-mmap"] } vmm-sys-util = "0.11.1" +virtio-bindings = "0.2.0" # vm-device is not yet published on crates.io. # To make sure that breaking changes to vm-device are not breaking the diff --git a/src/vmm/src/devices/net/bindings.rs b/src/vmm/src/devices/net/bindings.rs new file mode 100644 index 0000000..b4a6ef0 --- /dev/null +++ b/src/vmm/src/devices/net/bindings.rs @@ -0,0 +1,265 @@ +// Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// Portions Copyright 2017 The Chromium OS Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the THIRD-PARTY file. + +// The following are manually copied from crosvm/firecracker. In the latter, they can be found as +// part of the `net_gen` local crate. We should figure out how to proceed going forward (i.e. +// create some bindings of our own, put them in a common crate, etc). + +#![allow(clippy::all)] +#![allow(non_upper_case_globals)] +#![allow(non_camel_case_types)] +#![allow(non_snake_case)] + +pub const TUN_F_CSUM: ::std::os::raw::c_uint = 1; +pub const TUN_F_TSO4: ::std::os::raw::c_uint = 2; +pub const TUN_F_TSO6: ::std::os::raw::c_uint = 4; +pub const TUN_F_UFO: ::std::os::raw::c_uint = 16; + +#[repr(C)] +pub struct __BindgenUnionField(::std::marker::PhantomData); +impl __BindgenUnionField { + #[inline] + pub fn new() -> Self { + __BindgenUnionField(::std::marker::PhantomData) + } + #[inline] + pub unsafe fn as_mut(&mut self) -> &mut T { + ::std::mem::transmute(self) + } +} +impl ::std::default::Default for __BindgenUnionField { + #[inline] + fn default() -> Self { + Self::new() + } +} +impl ::std::clone::Clone for __BindgenUnionField { + #[inline] + fn clone(&self) -> Self { + Self::new() + } +} +impl ::std::marker::Copy for __BindgenUnionField {} +impl ::std::fmt::Debug for __BindgenUnionField { + fn fmt(&self, fmt: &mut ::std::fmt::Formatter) -> ::std::fmt::Result { + fmt.write_str("__BindgenUnionField") + } +} + +#[repr(C)] +#[derive(Debug, Default, Copy)] +pub struct ifreq { + pub ifr_ifrn: ifreq__bindgen_ty_1, + pub ifr_ifru: ifreq__bindgen_ty_2, +} + +impl Clone for ifreq { + fn clone(&self) -> Self { + *self + } +} + +#[repr(C)] +#[derive(Debug, Default, Copy)] +pub struct ifreq__bindgen_ty_1 { + pub ifrn_name: __BindgenUnionField<[::std::os::raw::c_uchar; 16usize]>, + pub bindgen_union_field: [u8; 16usize], +} + +impl Clone for ifreq__bindgen_ty_1 { + fn clone(&self) -> Self { + *self + } +} + +#[repr(C)] +#[derive(Debug, Default, Copy)] +pub struct ifreq__bindgen_ty_2 { + pub ifru_addr: __BindgenUnionField, + pub ifru_dstaddr: __BindgenUnionField, + pub ifru_broadaddr: __BindgenUnionField, + pub ifru_netmask: __BindgenUnionField, + pub ifru_hwaddr: __BindgenUnionField, + pub ifru_flags: __BindgenUnionField<::std::os::raw::c_short>, + pub ifru_ivalue: __BindgenUnionField<::std::os::raw::c_int>, + pub ifru_mtu: __BindgenUnionField<::std::os::raw::c_int>, + pub ifru_map: __BindgenUnionField, + pub ifru_slave: __BindgenUnionField<[::std::os::raw::c_char; 16usize]>, + pub ifru_newname: __BindgenUnionField<[::std::os::raw::c_char; 16usize]>, + pub ifru_data: __BindgenUnionField<*mut ::std::os::raw::c_void>, + pub ifru_settings: __BindgenUnionField, + pub bindgen_union_field: [u64; 3usize], +} + +impl Clone for ifreq__bindgen_ty_2 { + fn clone(&self) -> Self { + *self + } +} + +pub type sa_family_t = ::std::os::raw::c_ushort; + +#[repr(C)] +#[derive(Debug, Default, Copy)] +pub struct sockaddr { + pub sa_family: sa_family_t, + pub sa_data: [::std::os::raw::c_char; 14usize], +} + +impl Clone for sockaddr { + fn clone(&self) -> Self { + *self + } +} + +#[repr(C)] +#[derive(Debug, Default, Copy)] +pub struct if_settings { + pub type_: ::std::os::raw::c_uint, + pub size: ::std::os::raw::c_uint, + pub ifs_ifsu: if_settings__bindgen_ty_1, +} + +impl Clone for if_settings { + fn clone(&self) -> Self { + *self + } +} + +#[repr(C)] +#[derive(Debug, Default, Copy)] +pub struct if_settings__bindgen_ty_1 { + pub raw_hdlc: __BindgenUnionField<*mut raw_hdlc_proto>, + pub cisco: __BindgenUnionField<*mut cisco_proto>, + pub fr: __BindgenUnionField<*mut fr_proto>, + pub fr_pvc: __BindgenUnionField<*mut fr_proto_pvc>, + pub fr_pvc_info: __BindgenUnionField<*mut fr_proto_pvc_info>, + pub sync: __BindgenUnionField<*mut sync_serial_settings>, + pub te1: __BindgenUnionField<*mut te1_settings>, + pub bindgen_union_field: u64, +} + +impl Clone for if_settings__bindgen_ty_1 { + fn clone(&self) -> Self { + *self + } +} + +#[repr(C)] +#[derive(Debug, Default, Copy)] +pub struct ifmap { + pub mem_start: ::std::os::raw::c_ulong, + pub mem_end: ::std::os::raw::c_ulong, + pub base_addr: ::std::os::raw::c_ushort, + pub irq: ::std::os::raw::c_uchar, + pub dma: ::std::os::raw::c_uchar, + pub port: ::std::os::raw::c_uchar, +} + +impl Clone for ifmap { + fn clone(&self) -> Self { + *self + } +} + +#[repr(C)] +#[derive(Debug, Default, Copy)] +pub struct raw_hdlc_proto { + pub encoding: ::std::os::raw::c_ushort, + pub parity: ::std::os::raw::c_ushort, +} + +impl Clone for raw_hdlc_proto { + fn clone(&self) -> Self { + *self + } +} + +#[repr(C)] +#[derive(Debug, Default, Copy)] +pub struct cisco_proto { + pub interval: ::std::os::raw::c_uint, + pub timeout: ::std::os::raw::c_uint, +} + +impl Clone for cisco_proto { + fn clone(&self) -> Self { + *self + } +} + +#[repr(C)] +#[derive(Debug, Default, Copy)] +pub struct fr_proto { + pub t391: ::std::os::raw::c_uint, + pub t392: ::std::os::raw::c_uint, + pub n391: ::std::os::raw::c_uint, + pub n392: ::std::os::raw::c_uint, + pub n393: ::std::os::raw::c_uint, + pub lmi: ::std::os::raw::c_ushort, + pub dce: ::std::os::raw::c_ushort, +} + +impl Clone for fr_proto { + fn clone(&self) -> Self { + *self + } +} + +#[repr(C)] +#[derive(Debug, Default, Copy)] +pub struct fr_proto_pvc { + pub dlci: ::std::os::raw::c_uint, +} + +impl Clone for fr_proto_pvc { + fn clone(&self) -> Self { + *self + } +} + +#[repr(C)] +#[derive(Debug, Default, Copy)] +pub struct fr_proto_pvc_info { + pub dlci: ::std::os::raw::c_uint, + pub master: [::std::os::raw::c_char; 16usize], +} + +impl Clone for fr_proto_pvc_info { + fn clone(&self) -> Self { + *self + } +} + +#[repr(C)] +#[derive(Debug, Default, Copy)] +pub struct sync_serial_settings { + pub clock_rate: ::std::os::raw::c_uint, + pub clock_type: ::std::os::raw::c_uint, + pub loopback: ::std::os::raw::c_ushort, +} + +impl Clone for sync_serial_settings { + fn clone(&self) -> Self { + *self + } +} + +#[repr(C)] +#[derive(Debug, Default, Copy)] +pub struct te1_settings { + pub clock_rate: ::std::os::raw::c_uint, + pub clock_type: ::std::os::raw::c_uint, + pub loopback: ::std::os::raw::c_ushort, + pub slot_map: ::std::os::raw::c_uint, +} + +impl Clone for te1_settings { + fn clone(&self) -> Self { + *self + } +} diff --git a/src/vmm/src/devices/net/mod.rs b/src/vmm/src/devices/net/mod.rs index d068a07..d3bd547 100644 --- a/src/vmm/src/devices/net/mod.rs +++ b/src/vmm/src/devices/net/mod.rs @@ -1,6 +1,8 @@ use std::{error::Error, fmt::Display}; +pub mod bindings; pub mod interface; +pub mod tap; #[derive(Debug)] #[allow(dead_code)] diff --git a/src/vmm/src/devices/net/tap.rs b/src/vmm/src/devices/net/tap.rs new file mode 100644 index 0000000..f67db92 --- /dev/null +++ b/src/vmm/src/devices/net/tap.rs @@ -0,0 +1,188 @@ +// Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// Portions Copyright 2017 The Chromium OS Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the THIRD-PARTY file. + +// We should add a tap abstraction to rust-vmm as well. Using this one, which is copied from +// Firecracker until then. + +use std::fs::File; +use std::io::{Error as IoError, Read, Result as IoResult, Write}; +use std::os::raw::{c_char, c_uint, c_ulong}; +use std::os::unix::io::{AsRawFd, FromRawFd, RawFd}; + +use virtio_bindings::bindings::virtio_net::{VIRTIO_NET_F_CSUM, VIRTIO_NET_F_HOST_UFO}; +use vmm_sys_util::ioctl::{ioctl_with_mut_ref, ioctl_with_ref, ioctl_with_val}; +use vmm_sys_util::{ioctl_ioc_nr, ioctl_iow_nr}; + +use super::bindings::{ifreq, TUN_F_CSUM, TUN_F_TSO4, TUN_F_TSO6, TUN_F_UFO}; +use super::interface::Interface; +use super::VirtioNetError; + +// As defined in the Linux UAPI: +// https://elixir.bootlin.com/linux/v4.17/source/include/uapi/linux/if.h#L33 +const IFACE_NAME_MAX_LEN: usize = 16; + +// Taken from firecracker net_gen/if_tun.rs ... we should see what to do about the net related +// bindings overall for rust-vmm. +const IFF_TAP: ::std::os::raw::c_uint = 2; +const IFF_NO_PI: ::std::os::raw::c_uint = 4096; +const IFF_VNET_HDR: ::std::os::raw::c_uint = 16384; + +const TUNTAP: ::std::os::raw::c_uint = 84; +ioctl_iow_nr!(TUNSETIFF, TUNTAP, 202, ::std::os::raw::c_int); +ioctl_iow_nr!(TUNSETOFFLOAD, TUNTAP, 208, ::std::os::raw::c_uint); +ioctl_iow_nr!(TUNSETVNETHDRSZ, TUNTAP, 216, ::std::os::raw::c_int); + +/// Handle for a network tap interface. +/// +/// For now, this simply wraps the file descriptor for the tap device so methods +/// can run ioctls on the interface. The tap interface fd will be closed when +/// Tap goes out of scope, and the kernel will clean up the interface automatically. +#[derive(Debug)] +pub struct Tap { + tap_file: File, +} + +impl Tap { + fn virtio_flags_to_tuntap_flags(virtio_flags: u64) -> c_uint { + // Check if VIRTIO_NET_F_CSUM is set and set TUN_F_CSUM if so. Do the same for UFO, TSO6 and TSO4. + let mut flags = 0; + if virtio_flags & (1 << VIRTIO_NET_F_CSUM) != 0 { + flags |= TUN_F_CSUM; + } + if virtio_flags & (1 << VIRTIO_NET_F_HOST_UFO) != 0 { + flags |= TUN_F_UFO; + } + if virtio_flags & (1 << VIRTIO_NET_F_HOST_UFO) != 0 { + flags |= TUN_F_TSO4; + } + if virtio_flags & (1 << VIRTIO_NET_F_HOST_UFO) != 0 { + flags |= TUN_F_TSO6; + } + + flags + } +} + +impl Interface for Tap { + fn activate(&self, virtio_flags: u64, virtio_header_size: usize) -> super::Result<()> { + let flags = Tap::virtio_flags_to_tuntap_flags(virtio_flags); + + let ret = unsafe { ioctl_with_val(self, TUNSETOFFLOAD(), flags as c_ulong) }; + if ret < 0 { + return Err(std::io::Error::last_os_error()).map_err(VirtioNetError::IoCtlError); + } + + // Safe because we know that our file is a valid tap device and we verify the result. + let ret = unsafe { ioctl_with_ref(self, TUNSETVNETHDRSZ(), &virtio_header_size) }; + if ret < 0 { + return Err(std::io::Error::last_os_error()).map_err(VirtioNetError::IoCtlError); + } + + Ok(()) + } + + fn open_named(if_name: &str) -> super::Result { + let terminated_if_name = build_terminated_if_name(if_name)?; + + let fd = unsafe { + // Open calls are safe because we give a constant null-terminated + // string and verify the result. + libc::open( + b"/dev/net/tun\0".as_ptr() as *const c_char, + libc::O_RDWR | libc::O_NONBLOCK, + ) + }; + if fd < 0 { + return Err(IoError::last_os_error()).map_err(VirtioNetError::IoError); + } + // We just checked that the fd is valid. + let tuntap = unsafe { File::from_raw_fd(fd) }; + + IfReqBuilder::new() + .if_name(&terminated_if_name) + .flags((IFF_TAP | IFF_NO_PI | IFF_VNET_HDR) as i16) + .execute(&tuntap, TUNSETIFF()) + .unwrap(); + + // Safe since only the name is accessed, and it's cloned out. + Ok(Tap { tap_file: tuntap }) + } +} + +// Returns a byte vector representing the contents of a null terminated C string which +// contains if_name. +fn build_terminated_if_name(if_name: &str) -> super::Result<[u8; IFACE_NAME_MAX_LEN]> { + // Convert the string slice to bytes, and shadow the variable, + // since we no longer need the &str version. + let if_name = if_name.as_bytes(); + + if if_name.len() >= IFACE_NAME_MAX_LEN { + return Err(VirtioNetError::InvalidIfname); + } + + let mut terminated_if_name = [b'\0'; IFACE_NAME_MAX_LEN]; + terminated_if_name[..if_name.len()].copy_from_slice(if_name); + + Ok(terminated_if_name) +} + +pub struct IfReqBuilder(ifreq); + +impl IfReqBuilder { + #[allow(clippy::new_without_default)] + pub fn new() -> Self { + Self(Default::default()) + } + + pub fn if_name(mut self, if_name: &[u8; IFACE_NAME_MAX_LEN]) -> Self { + // Since we don't call as_mut on the same union field more than once, this block is safe. + let ifrn_name = unsafe { self.0.ifr_ifrn.ifrn_name.as_mut() }; + ifrn_name.copy_from_slice(if_name.as_ref()); + + self + } + + pub(crate) fn flags(mut self, flags: i16) -> Self { + // Since we don't call as_mut on the same union field more than once, this block is safe. + let ifru_flags = unsafe { self.0.ifr_ifru.ifru_flags.as_mut() }; + *ifru_flags = flags; + + self + } + + pub(crate) fn execute(mut self, socket: &F, ioctl: u64) -> super::Result { + // ioctl is safe. Called with a valid socket fd, and we check the return. + let ret = unsafe { ioctl_with_mut_ref(socket, ioctl, &mut self.0) }; + if ret < 0 { + return Err(VirtioNetError::IoCtlError(IoError::last_os_error())); + } + + Ok(self.0) + } +} + +impl Read for Tap { + fn read(&mut self, buf: &mut [u8]) -> IoResult { + self.tap_file.read(buf) + } +} + +impl Write for Tap { + fn write(&mut self, buf: &[u8]) -> IoResult { + self.tap_file.write(buf) + } + + fn flush(&mut self) -> IoResult<()> { + Ok(()) + } +} + +impl AsRawFd for Tap { + fn as_raw_fd(&self) -> RawFd { + self.tap_file.as_raw_fd() + } +} From 95157e2277e5769130d182b7cb32272903dfa1e8 Mon Sep 17 00:00:00 2001 From: Nils Ponsard Date: Sun, 26 Mar 2023 21:36:08 +0200 Subject: [PATCH 04/13] feat: add dynamic cmdline management to the kernel Signed-off-by: Nils Ponsard --- src/vmm/src/kernel.rs | 26 ++++++++++++++++++++------ src/vmm/src/lib.rs | 18 +++++++++++++++++- 2 files changed, 37 insertions(+), 7 deletions(-) diff --git a/src/vmm/src/kernel.rs b/src/vmm/src/kernel.rs index 203e67b..100a1a9 100644 --- a/src/vmm/src/kernel.rs +++ b/src/vmm/src/kernel.rs @@ -44,7 +44,7 @@ const HIMEM_START: u64 = 0x0010_0000; // 1 MB /// Address where the kernel command line is written. const CMDLINE_START: u64 = 0x0002_0000; // Default command line -const CMDLINE: &str = "console=ttyS0 i8042.nokbd reboot=k panic=1 pci=off"; +pub const DEFAULT_CMDLINE: &str = "console=ttyS0 i8042.nokbd reboot=k panic=1 pci=off"; fn add_e820_entry( params: &mut boot_params, @@ -110,6 +110,7 @@ pub fn kernel_setup( guest_memory: &GuestMemoryMmap, kernel_path: PathBuf, initramfs_path: Option, + cmdline: &Cmdline, ) -> Result { let mut kernel_image = File::open(kernel_path).map_err(Error::IO)?; let zero_page_addr = GuestAddress(ZEROPG_START); @@ -126,9 +127,25 @@ pub fn kernel_setup( // Generate boot parameters. let mut bootparams = build_bootparams(guest_memory, GuestAddress(HIMEM_START))?; + let cmdline_str = cmdline + .as_cstring() + .map_err(Error::Cmdline)? + .into_string() + .map_err(Error::IntoStringError)?; + + let cmdline_size = cmdline_str.len() as u32; + // Add the kernel command line to the boot parameters. bootparams.hdr.cmd_line_ptr = CMDLINE_START as u32; - bootparams.hdr.cmdline_size = CMDLINE.len() as u32 + 1; + bootparams.hdr.cmdline_size = cmdline_size + 1; + + // Shrink the command line to the actual size. + + let mut shrinked_cmdline = + linux_loader::cmdline::Cmdline::new(cmdline_size as usize + 1).map_err(Error::Cmdline)?; + shrinked_cmdline + .insert_str(&cmdline_str) + .map_err(Error::Cmdline)?; // Add the initramfs to the boot parameters if one was provided. if let Some(initramfs_path) = initramfs_path { @@ -155,14 +172,11 @@ pub fn kernel_setup( } // Load the kernel command line into guest memory. - let mut cmdline = Cmdline::new(CMDLINE.len() + 1).map_err(Error::Cmdline)?; - - cmdline.insert_str(CMDLINE).map_err(Error::Cmdline)?; load_cmdline( guest_memory, GuestAddress(CMDLINE_START), // Safe because the command line is valid. - &cmdline, + cmdline, ) .map_err(Error::KernelLoad)?; diff --git a/src/vmm/src/lib.rs b/src/vmm/src/lib.rs index 817f947..922d2fa 100644 --- a/src/vmm/src/lib.rs +++ b/src/vmm/src/lib.rs @@ -8,13 +8,13 @@ extern crate linux_loader; extern crate vm_memory; extern crate vm_superio; +use std::fs::File; use std::io::stdout; use std::os::unix::io::AsRawFd; use std::os::unix::prelude::RawFd; use std::sync::{Arc, Mutex}; use std::thread; use std::{io, path::PathBuf}; -use std::fs::File; use kvm_bindings::{kvm_userspace_memory_region, KVM_MAX_CPUID_ENTRIES}; use kvm_ioctls::{Kvm, VmFd}; @@ -31,6 +31,8 @@ mod epoll_context; use epoll_context::{EpollContext, EPOLL_EVENTS_LEN}; mod kernel; +const CMDLINE_MAX_SIZE: usize = 4096; + #[derive(Debug)] /// VMM errors. @@ -69,6 +71,8 @@ pub enum Error { TerminalConfigure(kvm_ioctls::Error), /// Console configuration error ConsoleError(io::Error), + /// IntoString error + IntoStringError(std::ffi::IntoStringError), } /// Dedicated [`Result`](https://doc.rust-lang.org/std/result/) type. @@ -83,6 +87,8 @@ pub struct VMM { serial: Arc>, virtio_manager: Arc>, epoll: EpollContext, + + cmdline: linux_loader::cmdline::Cmdline, } impl VMM { @@ -108,6 +114,8 @@ impl VMM { )), virtio_manager: Arc::new(Mutex::new(IoManager::new())), epoll, + cmdline: linux_loader::cmdline::Cmdline::new(CMDLINE_MAX_SIZE) + .map_err(Error::Cmdline)?, }; Ok(vmm) @@ -146,6 +154,12 @@ impl VMM { Ok(()) } + pub fn load_default_cmdline(&mut self) -> Result<()> { + self.cmdline + .insert_str(kernel::DEFAULT_CMDLINE) + .map_err(Error::Cmdline) + } + pub fn configure_io(&mut self) -> Result<()> { // First, create the irqchip. // On `x86_64`, this _must_ be created _before_ the vCPUs. @@ -286,10 +300,12 @@ impl VMM { ) -> Result<()> { self.configure_console(console)?; self.configure_memory(mem_size_mb)?; + self.load_default_cmdline()?; let kernel_load = kernel::kernel_setup( &self.guest_memory, PathBuf::from(kernel_path), initramfs_path, + &self.cmdline, )?; self.configure_io()?; self.configure_vcpus(num_vcpus, kernel_load)?; From ea29c6a13e30038fb52e60e5e843ba6616943efb Mon Sep 17 00:00:00 2001 From: Nils Ponsard Date: Fri, 17 Feb 2023 17:30:17 +0100 Subject: [PATCH 05/13] feat: add dynamic cmdline management to the kernel Signed-off-by: Nils Ponsard --- src/vmm/src/lib.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/vmm/src/lib.rs b/src/vmm/src/lib.rs index 922d2fa..3275866 100644 --- a/src/vmm/src/lib.rs +++ b/src/vmm/src/lib.rs @@ -183,10 +183,7 @@ impl VMM { Ok(()) } - pub fn configure_console( - &mut self, - console_path: Option - ) -> Result<()> { + pub fn configure_console(&mut self, console_path: Option) -> Result<()> { if let Some(console_path) = console_path { // We create the file if it does not exist, else we open let file = File::create(&console_path).map_err(Error::ConsoleError)?; From a9e00ea49a9f3888741f877863442bed996a084f Mon Sep 17 00:00:00 2001 From: Simon LUCIDO Date: Fri, 24 Mar 2023 17:17:58 +0100 Subject: [PATCH 06/13] feat: add virtio_net device Signed-off-by: Simon LUCIDO --- Cargo.lock | 14 +- src/vmm/Cargo.lock | 14 +- src/vmm/Cargo.toml | 3 +- src/vmm/src/devices/net/bindings.rs | 6 + src/vmm/src/devices/net/mod.rs | 309 +++++++++++++++++++++++++++- 5 files changed, 336 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 73bedb0..48f012f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -297,6 +297,17 @@ name = "virtio-bindings" version = "0.2.0" source = "git+https://github.com/rust-vmm/vm-virtio#467c8ec99375a5f4e08b85b18257cd7e0bac1dc0" +[[package]] +name = "virtio-device" +version = "0.1.0" +source = "git+https://github.com/rust-vmm/vm-virtio#467c8ec99375a5f4e08b85b18257cd7e0bac1dc0" +dependencies = [ + "log", + "virtio-bindings 0.2.0 (git+https://github.com/rust-vmm/vm-virtio)", + "virtio-queue", + "vm-memory", +] + [[package]] name = "virtio-queue" version = "0.7.1" @@ -311,7 +322,7 @@ dependencies = [ [[package]] name = "vm-device" version = "0.1.0" -source = "git+https://github.com/lucido-simon/vm-device?rev=5619cec076594881ddf8a38978c50923b3434494#5619cec076594881ddf8a38978c50923b3434494" +source = "git+https://github.com/lucido-simon/vm-device?rev=27537ff920fb6b1f8749294c911648d8e44fd5e5#27537ff920fb6b1f8749294c911648d8e44fd5e5" dependencies = [ "vm-memory", ] @@ -342,6 +353,7 @@ dependencies = [ "libc", "linux-loader", "virtio-bindings 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "virtio-device", "virtio-queue", "vm-device", "vm-memory", diff --git a/src/vmm/Cargo.lock b/src/vmm/Cargo.lock index b50fd4d..a381450 100644 --- a/src/vmm/Cargo.lock +++ b/src/vmm/Cargo.lock @@ -79,6 +79,17 @@ name = "virtio-bindings" version = "0.2.0" source = "git+https://github.com/rust-vmm/vm-virtio#467c8ec99375a5f4e08b85b18257cd7e0bac1dc0" +[[package]] +name = "virtio-device" +version = "0.1.0" +source = "git+https://github.com/rust-vmm/vm-virtio#467c8ec99375a5f4e08b85b18257cd7e0bac1dc0" +dependencies = [ + "log", + "virtio-bindings 0.2.0 (git+https://github.com/rust-vmm/vm-virtio)", + "virtio-queue", + "vm-memory", +] + [[package]] name = "virtio-queue" version = "0.7.1" @@ -93,7 +104,7 @@ dependencies = [ [[package]] name = "vm-device" version = "0.1.0" -source = "git+https://github.com/lucido-simon/vm-device?rev=5619cec076594881ddf8a38978c50923b3434494#5619cec076594881ddf8a38978c50923b3434494" +source = "git+https://github.com/lucido-simon/vm-device?rev=27537ff920fb6b1f8749294c911648d8e44fd5e5#27537ff920fb6b1f8749294c911648d8e44fd5e5" dependencies = [ "vm-memory", ] @@ -124,6 +135,7 @@ dependencies = [ "libc", "linux-loader", "virtio-bindings 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "virtio-device", "virtio-queue", "vm-device", "vm-memory", diff --git a/src/vmm/Cargo.toml b/src/vmm/Cargo.toml index 88532d5..7c23c78 100644 --- a/src/vmm/Cargo.toml +++ b/src/vmm/Cargo.toml @@ -16,7 +16,8 @@ virtio-bindings = "0.2.0" # vm-device is not yet published on crates.io. # To make sure that breaking changes to vm-device are not breaking the # vm-vcpu build, we're using a fixed revision. -vm-device = { git = "https://github.com/lucido-simon/vm-device", rev = "5619cec076594881ddf8a38978c50923b3434494" } +vm-device = { git = "https://github.com/lucido-simon/vm-device", rev = "27537ff920fb6b1f8749294c911648d8e44fd5e5" } +virtio-device = { git = "https://github.com/rust-vmm/vm-virtio" } virtio-queue = { git = "https://github.com/rust-vmm/vm-virtio" } vm-superio = "0.7.0" diff --git a/src/vmm/src/devices/net/bindings.rs b/src/vmm/src/devices/net/bindings.rs index b4a6ef0..a6e5ed6 100644 --- a/src/vmm/src/devices/net/bindings.rs +++ b/src/vmm/src/devices/net/bindings.rs @@ -14,11 +14,17 @@ #![allow(non_camel_case_types)] #![allow(non_snake_case)] +use virtio_bindings::virtio_net::virtio_net_hdr_v1; + pub const TUN_F_CSUM: ::std::os::raw::c_uint = 1; pub const TUN_F_TSO4: ::std::os::raw::c_uint = 2; pub const TUN_F_TSO6: ::std::os::raw::c_uint = 4; pub const TUN_F_UFO: ::std::os::raw::c_uint = 16; +pub const VIRTIO_F_VERSION_1: u64 = 32; +pub const VIRTIO_HDR_LEN: usize = ::core::mem::size_of::(); +pub const VIRTIO_NET_DEVICE_ID: u32 = 1; + #[repr(C)] pub struct __BindgenUnionField(::std::marker::PhantomData); impl __BindgenUnionField { diff --git a/src/vmm/src/devices/net/mod.rs b/src/vmm/src/devices/net/mod.rs index d3bd547..30116a7 100644 --- a/src/vmm/src/devices/net/mod.rs +++ b/src/vmm/src/devices/net/mod.rs @@ -1,11 +1,46 @@ -use std::{error::Error, fmt::Display}; - -pub mod bindings; pub mod interface; -pub mod tap; + +pub(crate) mod bindings; +pub(crate) mod tap; + +use std::{ + borrow::{Borrow, BorrowMut}, + cmp, + error::Error, + fmt::{self, Debug, Display}, + os::fd::{AsRawFd, RawFd}, + sync::atomic::Ordering, +}; + +use virtio_device::{VirtioConfig, VirtioDeviceActions, VirtioDeviceType, VirtioMmioDevice}; + +use virtio_bindings::bindings::virtio_net::{ + self, VIRTIO_NET_F_CSUM, VIRTIO_NET_F_GUEST_CSUM, VIRTIO_NET_F_GUEST_TSO4, + VIRTIO_NET_F_GUEST_TSO6, VIRTIO_NET_F_GUEST_UFO, VIRTIO_NET_F_HOST_TSO4, + VIRTIO_NET_F_HOST_TSO6, VIRTIO_NET_F_HOST_UFO, +}; +use virtio_queue::{Queue, QueueOwnedT, QueueT}; +use vm_device::{MutVirtioMmioDevice, VirtioMmioOffset}; +use vm_memory::{Bytes, GuestAddress, GuestAddressSpace}; +use vmm_sys_util::eventfd::EventFd; + +use interface::Interface; + +// TODO: Make this configurable. +const VIRTIO_FEATURES: u64 = (1 << bindings::VIRTIO_F_VERSION_1) + | (1 << VIRTIO_NET_F_CSUM) + | (1 << VIRTIO_NET_F_GUEST_CSUM) + | (1 << VIRTIO_NET_F_HOST_TSO4) + | (1 << VIRTIO_NET_F_HOST_TSO6) + | (1 << VIRTIO_NET_F_HOST_UFO) + | (1 << VIRTIO_NET_F_GUEST_TSO4) + | (1 << VIRTIO_NET_F_GUEST_TSO6) + | (1 << VIRTIO_NET_F_GUEST_UFO); + +const MAX_BUFFER_SIZE: usize = 65565; #[derive(Debug)] -#[allow(dead_code)] + pub enum VirtioNetError { InvalidIfname, VirtioQueueError(virtio_queue::Error), @@ -16,10 +51,270 @@ pub enum VirtioNetError { } impl Error for VirtioNetError {} impl Display for VirtioNetError { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { write!(f, "virtio net error") } } -#[allow(dead_code)] pub type Result = std::result::Result; + +pub struct VirtioNet { + pub device_config: VirtioConfig, + pub guest_irq_fd: EventFd, + pub address_space: M, + pub interface: I, +} + +impl VirtioNet { + pub fn new(memory: M, irq_fd: EventFd, if_name: &str) -> Result { + Ok(Self { + device_config: VirtioConfig::new( + VIRTIO_FEATURES, + vec![ + Queue::new(256).map_err(VirtioNetError::QueueError)?, + Queue::new(256).map_err(VirtioNetError::QueueError)?, + ], + // Not used in the current implementation. + Self::config_vec(virtio_net::virtio_net_config { + ..Default::default() + }), + ), + address_space: memory, + guest_irq_fd: irq_fd, + interface: I::open_named(if_name)?, + }) + } + + fn config_vec(config: virtio_net::virtio_net_config) -> Vec { + let mut config_vec = Vec::new(); + config_vec.extend_from_slice(&config.mac); + config_vec.extend_from_slice(&config.status.to_le_bytes()); + config_vec.extend_from_slice(&config.max_virtqueue_pairs.to_le_bytes()); + config_vec.extend_from_slice(&config.mtu.to_le_bytes()); + config_vec.extend_from_slice(&config.speed.to_le_bytes()); + config_vec.extend_from_slice(&config.duplex.to_le_bytes()); + config_vec + } + + fn is_reading_register(&self, offset: &VirtioMmioOffset) -> bool { + if let VirtioMmioOffset::DeviceSpecific(offset) = offset { + !(*offset as usize) < self.device_config.config_space.len() * 8 + } else { + true + } + } + + fn write_frame_to_guest( + &mut self, + original_buffer: &mut [u8; MAX_BUFFER_SIZE], + size: usize, + ) -> Result { + let mem = self.address_space.memory(); + let mut chain = match &mut self.device_config.queues[0] + .iter(&*mem) + .map_err(VirtioNetError::QueueError)? + .next() + { + Some(c) => c.to_owned(), + _ => return Ok(false), + }; + + let mut count = 0; + let buffer = &mut original_buffer[..size]; + + while let Some(desc) = chain.next() { + let left = buffer.len() - count; + + if left == 0 { + break; + } + + let len = cmp::min(left, desc.len() as usize); + chain + .memory() + .write_slice(&buffer[count..count + len], desc.addr()) + .map_err(VirtioNetError::MemoryError)?; + + count += len; + } + + if count != buffer.len() { + // The frame was too large for the chain. + println!("rx frame too large"); + } + + self.device_config.queues[0] + .add_used(&*mem, chain.head_index(), count as u32) + .map_err(VirtioNetError::QueueError)?; + + Ok(true) + } + + pub fn process_tap(&mut self) -> Result<()> { + { + let buffer = &mut [0u8; MAX_BUFFER_SIZE]; + + loop { + let read_size = match self.interface.read(buffer) { + Ok(size) => size, + Err(_) => { + break; + } + }; + + let mem = self.address_space.memory().borrow_mut().clone(); + + if !self.write_frame_to_guest(buffer, read_size)? + && !self.device_config.queues[0] + .enable_notification(&*mem.clone()) + .map_err(VirtioNetError::QueueError)? + { + break; + } + } + } + + if self.device_config.queues[0] + .needs_notification(&*self.address_space.memory()) + .map_err(VirtioNetError::QueueError)? + { + // TODO: Figure out why we need to do that + self.device_config + .interrupt_status + .store(1, Ordering::SeqCst); + + // Error should be recoverable as is, so we just log it. + self.guest_irq_fd.write(1).unwrap_or_else(|e| { + println!("Failed to signal irq: {:?}", e); + }); + } + + Ok(()) + } +} + +impl AsRawFd for VirtioNet { + fn as_raw_fd(&self) -> RawFd { + self.interface.as_raw_fd() + } +} + +impl VirtioDeviceType for VirtioNet { + fn device_type(&self) -> u32 { + bindings::VIRTIO_NET_DEVICE_ID + } +} + +impl VirtioMmioDevice for VirtioNet { + // Please note that this method can be improved error handling wise. + // We are limited in how we can handle errors here, as we are not allowed to return a Result. + fn queue_notify(&mut self, val: u32) { + if val == 0 { + return; + } + + let mem = self.address_space.memory().clone(); + let irq = &mut self.guest_irq_fd; + let queue = &mut self.device_config.queues[1]; + + loop { + match queue.disable_notification(&*mem) { + Ok(_) => {} + Err(e) => { + println!("Failed to disable notification: {:?}", e); + break; + } + } + + // Consume entries from the available ring. + // Never fails since we know the memory is valid. + while let Some(chain) = queue.iter(&*mem).unwrap().next() { + let mut data_buffer: Vec = Vec::new(); + chain.clone().for_each(|desc| { + let initial_buffer_len = data_buffer.len(); + + data_buffer.resize(data_buffer.len() + desc.len() as usize, 0); + + // Safe as we just allocated the buffer and mem is valid. + // If it actually fails, it is probably unrecoverable anyway. + mem.read_slice(&mut data_buffer[initial_buffer_len..], desc.addr()) + .unwrap(); + }); + + if (data_buffer.len() as usize) < bindings::VIRTIO_HDR_LEN { + println!("invalid net packet"); + return; + } + + match self.interface.write(&data_buffer) { + Ok(_) => { + queue + .add_used(&*mem, chain.head_index(), 0x100) + // Try continuing even if we failed to add the used buffer. + .unwrap_or_else(|e| { + println!("Failed to add used buffer: {:?}", e); + }); + + if queue.needs_notification(&*mem).unwrap_or_default() { + irq.write(1).unwrap_or_else(|e| { + println!("Failed to signal irq: {:?}", e); + }); + } + } + Err(e) => { + println!("Failed to write to tap: {:?}", e); + } + } + } + + if !queue.enable_notification(&*mem).unwrap_or_default() { + break; + } + } + } +} + +impl Borrow> + for VirtioNet +{ + fn borrow(&self) -> &VirtioConfig { + &self.device_config + } +} + +impl BorrowMut> + for VirtioNet +{ + fn borrow_mut(&mut self) -> &mut VirtioConfig { + &mut self.device_config + } +} + +impl VirtioDeviceActions for VirtioNet { + type E = VirtioNetError; + + fn activate(&mut self) -> Result<()> { + self.interface + .activate(VIRTIO_FEATURES, bindings::VIRTIO_HDR_LEN)?; + + Ok(()) + } + fn reset(&mut self) -> std::result::Result<(), Self::E> { + println!("virtio net reset"); + Ok(()) + } +} + +impl MutVirtioMmioDevice for VirtioNet { + fn virtio_mmio_read(&mut self, _base: GuestAddress, offset: VirtioMmioOffset, data: &mut [u8]) { + if self.is_reading_register(&offset) { + self.read(u64::from(offset), data); + } + } + + fn virtio_mmio_write(&mut self, _base: GuestAddress, offset: VirtioMmioOffset, data: &[u8]) { + if self.is_reading_register(&offset) { + self.write(u64::from(offset), data); + } + } +} From f0543ee5c137c04c8029f021930e60c1fc1111ee Mon Sep 17 00:00:00 2001 From: Simon LUCIDO Date: Fri, 24 Mar 2023 17:21:36 +0100 Subject: [PATCH 07/13] feat: add generic fd in epoll_context Signed-off-by: Simon LUCIDO --- src/vmm/src/epoll_context.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/vmm/src/epoll_context.rs b/src/vmm/src/epoll_context.rs index 47d453b..445a715 100644 --- a/src/vmm/src/epoll_context.rs +++ b/src/vmm/src/epoll_context.rs @@ -26,6 +26,16 @@ impl EpollContext { epoll::Event::new(epoll::Events::EPOLLIN, libc::STDIN_FILENO as u64), )?; + Ok(()) + } + pub fn add_fd(&self, fd: RawFd) -> result::Result<(), io::Error> { + epoll::ctl( + self.raw_fd, + epoll::ControlOptions::EPOLL_CTL_ADD, + fd, + epoll::Event::new(epoll::Events::EPOLLIN, fd as u64), + )?; + Ok(()) } } From 01273fb47cd6b23215cc3429da939886c3ecca07 Mon Sep 17 00:00:00 2001 From: Simon LUCIDO Date: Fri, 24 Mar 2023 17:38:34 +0100 Subject: [PATCH 08/13] feat: integrate virtio_net to lumper Signed-off-by: Simon LUCIDO --- src/main.rs | 5 +++ src/vmm/src/lib.rs | 101 ++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 101 insertions(+), 5 deletions(-) diff --git a/src/main.rs b/src/main.rs index fd4c646..02605ba 100644 --- a/src/main.rs +++ b/src/main.rs @@ -29,6 +29,10 @@ struct VMMOpts { /// Stdout console file path #[clap(long)] console: Option, + + /// Interface name + #[clap(long)] + net: Option, } #[derive(Debug)] @@ -57,6 +61,7 @@ fn main() -> Result<(), Error> { &opts.kernel, opts.console, opts.initramfs, + opts.net, ) .map_err(Error::VmmConfigure)?; diff --git a/src/vmm/src/lib.rs b/src/vmm/src/lib.rs index 3275866..f816442 100644 --- a/src/vmm/src/lib.rs +++ b/src/vmm/src/lib.rs @@ -16,11 +16,15 @@ use std::sync::{Arc, Mutex}; use std::thread; use std::{io, path::PathBuf}; +use devices::net::tap::Tap; +use devices::net::VirtioNet; use kvm_bindings::{kvm_userspace_memory_region, KVM_MAX_CPUID_ENTRIES}; use kvm_ioctls::{Kvm, VmFd}; use linux_loader::loader::{self, KernelLoaderResult}; use vm_device::device_manager::IoManager; +use vm_device::resources::Resource; use vm_memory::{Address, GuestAddress, GuestMemory, GuestMemoryMmap, GuestMemoryRegion}; +use vmm_sys_util::eventfd::EventFd; use vmm_sys_util::terminal::Terminal; mod cpu; use cpu::{cpuid, mptable, Vcpu}; @@ -73,6 +77,12 @@ pub enum Error { ConsoleError(io::Error), /// IntoString error IntoStringError(std::ffi::IntoStringError), + /// Error writing to the guest memory. + GuestMemory(vm_memory::guest_memory::Error), + /// Error related to the virtio-net device. + VirtioNet(devices::net::VirtioNetError), + /// Error related to IOManager. + IoManager(vm_device::device_manager::Error), } /// Dedicated [`Result`](https://doc.rust-lang.org/std/result/) type. @@ -86,6 +96,8 @@ pub struct VMM { serial: Arc>, virtio_manager: Arc>, + virtio_net: Option, Tap>>>>, + epoll: EpollContext, cmdline: linux_loader::cmdline::Cmdline, @@ -112,6 +124,7 @@ impl VMM { serial: Arc::new(Mutex::new( LumperSerial::new(Box::new(stdout())).map_err(Error::SerialCreation)?, )), + virtio_net: None, virtio_manager: Arc::new(Mutex::new(IoManager::new())), epoll, cmdline: linux_loader::cmdline::Cmdline::new(CMDLINE_MAX_SIZE) @@ -159,6 +172,53 @@ impl VMM { .insert_str(kernel::DEFAULT_CMDLINE) .map_err(Error::Cmdline) } + // configure the virtio-net device + pub fn configure_net(&mut self, interface: Option) -> Result<()> { + let if_name = match interface { + Some(if_name) => if_name, + None => return Ok(()), + }; + + // Temporary hardcoded address, see allocator PR + let virtio_address = GuestAddress(0xd0000000); + + let irq_fd = EventFd::new(libc::EFD_NONBLOCK).map_err(Error::IrqRegister)?; + + let virtio_net = VirtioNet::new( + Arc::new(self.guest_memory.clone()), + irq_fd, + if_name.as_str(), + ) + .map_err(Error::VirtioNet)?; + + self.epoll + .add_fd(virtio_net.as_raw_fd()) + .map_err(Error::EpollError)?; + let mut io_manager = self.virtio_manager.lock().unwrap(); + + self.virtio_net = Some(Arc::new(Mutex::new(virtio_net))); + + io_manager + .register_mmio_resources( + // It's safe to unwrap because the virtio-net was just assigned + self.virtio_net.as_ref().unwrap().clone(), + &[ + Resource::GuestAddressRange { + base: virtio_address.raw_value(), + size: 0x1000, + }, + Resource::LegacyIrq(5), + ], + ) + .map_err(Error::IoManager)?; + + // Add the virtio-net device to the cmdline. + self.cmdline + .add_virtio_mmio_device(0x1000, virtio_address, 5, None) + .map_err(Error::Cmdline)?; + + Ok(()) + } pub fn configure_io(&mut self) -> Result<()> { // First, create the irqchip. @@ -180,6 +240,11 @@ impl VMM { ) .map_err(Error::KvmIoctl)?; + if let Some(virtio_net) = self.virtio_net.as_ref() { + self.vm_fd + .register_irqfd(&virtio_net.lock().unwrap().guest_irq_fd, 5) + .map_err(Error::KvmIoctl)?; + } Ok(()) } @@ -213,7 +278,7 @@ impl VMM { &self.vm_fd, index.into(), Arc::clone(&self.serial), - Arc::clone(&self.virtio_manager), + self.virtio_manager.clone(), ) .map_err(Error::Vcpu)?; @@ -262,11 +327,22 @@ impl VMM { .map_err(Error::TerminalConfigure)?; let mut events = vec![epoll::Event::new(epoll::Events::empty(), 0); EPOLL_EVENTS_LEN]; let epoll_fd = self.epoll.as_raw_fd(); - - // Let's start the STDIN polling thread. + let interface_fd = match self.virtio_net.as_ref() { + Some(virtio_net) => Some(virtio_net.lock().unwrap().interface.as_raw_fd()), + None => None, + }; + // Let's start the STDIN/Network interface polling thread. loop { - let num_events = - epoll::wait(epoll_fd, -1, &mut events[..]).map_err(Error::EpollError)?; + let num_events = match epoll::wait(epoll_fd, -1, &mut events[..]) { + Ok(num_events) => num_events, + Err(e) => { + if e.kind() == io::ErrorKind::Interrupted { + continue; + } else { + return Err(Error::EpollError(e)); + } + } + }; for event in events.iter().take(num_events) { let event_data = event.data as RawFd; @@ -283,6 +359,17 @@ impl VMM { .enqueue_raw_bytes(&out[..count]) .map_err(Error::StdinWrite)?; } + + if interface_fd == Some(event_data) { + self.virtio_net + .as_ref() + // Safe because we checked that the virtio_net is Some before the loop. + .unwrap() + .lock() + .unwrap() + .process_tap() + .map_err(Error::VirtioNet)?; + } } } } @@ -294,10 +381,14 @@ impl VMM { kernel_path: &str, console: Option, initramfs_path: Option, + if_name: Option, ) -> Result<()> { self.configure_console(console)?; self.configure_memory(mem_size_mb)?; self.load_default_cmdline()?; + + self.configure_net(if_name)?; + let kernel_load = kernel::kernel_setup( &self.guest_memory, PathBuf::from(kernel_path), From edca17bceaeb13092daef8aa1e8472278c77c95e Mon Sep 17 00:00:00 2001 From: esteban Date: Wed, 22 Mar 2023 10:52:09 +0100 Subject: [PATCH 09/13] feat: add --no-console option It is now possible to run a VM without a console with --no-console option Co-authored-by: Esteban Baron Signed-off-by: Alexis Langlet --- src/main.rs | 7 +++++- src/vmm/src/cpu/mod.rs | 22 ++++++++++++----- src/vmm/src/kernel.rs | 2 +- src/vmm/src/lib.rs | 55 ++++++++++++++++++++++++++++++++++++++---- 4 files changed, 73 insertions(+), 13 deletions(-) diff --git a/src/main.rs b/src/main.rs index 02605ba..b35e37f 100644 --- a/src/main.rs +++ b/src/main.rs @@ -33,6 +33,10 @@ struct VMMOpts { /// Interface name #[clap(long)] net: Option, + + /// no-console + #[clap(long)] + no_console: bool, } #[derive(Debug)] @@ -60,13 +64,14 @@ fn main() -> Result<(), Error> { opts.memory, &opts.kernel, opts.console, + opts.no_console, opts.initramfs, opts.net, ) .map_err(Error::VmmConfigure)?; // Run the VMM - vmm.run().map_err(Error::VmmRun)?; + vmm.run(opts.no_console).map_err(Error::VmmRun)?; Ok(()) } diff --git a/src/vmm/src/cpu/mod.rs b/src/vmm/src/cpu/mod.rs index 440a6c5..c7e0467 100644 --- a/src/vmm/src/cpu/mod.rs +++ b/src/vmm/src/cpu/mod.rs @@ -223,20 +223,30 @@ impl Vcpu { } /// vCPU emulation loop. - pub fn run(&mut self) { + pub fn run(&mut self, no_console: bool, should_stop: Arc>) { // Call into KVM to launch (VMLAUNCH) or resume (VMRESUME) the virtual CPU. // This is a blocking function, it only returns for either an error or a // VM-Exit. In the latter case, we can inspect the exit reason. - match self.vcpu_fd.run() { + println!("Before running vCPU {}...", self.index); + let run = self.vcpu_fd.run(); + println!("After running vCPU {}...", self.index); + + match run { Ok(exit_reason) => match exit_reason { // The VM stopped (Shutdown ot HLT). VcpuExit::Shutdown | VcpuExit::Hlt => { println!("Guest shutdown: {:?}. Bye!", exit_reason); - let stdin = io::stdin(); - let stdin_lock = stdin.lock(); - stdin_lock.set_canon_mode().unwrap(); - unsafe { libc::exit(0) }; + if no_console { + println!("Exiting... {:?}", self.index); + *should_stop.lock().unwrap() = true; + } else { + let stdin = io::stdin(); + let stdin_lock = stdin.lock(); + stdin_lock.set_canon_mode().unwrap(); + + unsafe { libc::exit(0) }; + } } // This is a PIO write, i.e. the guest is trying to write diff --git a/src/vmm/src/kernel.rs b/src/vmm/src/kernel.rs index 100a1a9..4a5f01f 100644 --- a/src/vmm/src/kernel.rs +++ b/src/vmm/src/kernel.rs @@ -44,7 +44,7 @@ const HIMEM_START: u64 = 0x0010_0000; // 1 MB /// Address where the kernel command line is written. const CMDLINE_START: u64 = 0x0002_0000; // Default command line -pub const DEFAULT_CMDLINE: &str = "console=ttyS0 i8042.nokbd reboot=k panic=1 pci=off"; +pub const DEFAULT_CMDLINE: &str = "i8042.nokbd reboot=k panic=1 pci=off"; fn add_e820_entry( params: &mut boot_params, diff --git a/src/vmm/src/lib.rs b/src/vmm/src/lib.rs index f816442..3a8cf8e 100644 --- a/src/vmm/src/lib.rs +++ b/src/vmm/src/lib.rs @@ -8,6 +8,7 @@ extern crate linux_loader; extern crate vm_memory; extern crate vm_superio; +use std::any::Any; use std::fs::File; use std::io::stdout; use std::os::unix::io::AsRawFd; @@ -83,6 +84,10 @@ pub enum Error { VirtioNet(devices::net::VirtioNetError), /// Error related to IOManager. IoManager(vm_device::device_manager::Error), + /// Access thread handler error + AccessThreadHandlerError, + /// Join thread error + JoinThreadError(Box), } /// Dedicated [`Result`](https://doc.rust-lang.org/std/result/) type. @@ -248,7 +253,19 @@ impl VMM { Ok(()) } - pub fn configure_console(&mut self, console_path: Option) -> Result<()> { + pub fn configure_console( + &mut self, + console_path: Option, + disable_console: bool, + ) -> Result<()> { + if disable_console { + return Ok(()); + } + + self.cmdline + .insert_str("console=ttyS0") + .map_err(Error::Cmdline)?; + if let Some(console_path) = console_path { // We create the file if it does not exist, else we open let file = File::create(&console_path).map_err(Error::ConsoleError)?; @@ -312,12 +329,39 @@ impl VMM { } // Run all virtual CPUs. - pub fn run(&mut self) -> Result<()> { + pub fn run(&mut self, no_console: bool) -> Result<()> { + let mut handlers: Vec> = Vec::new(); + let should_stop = Arc::new(Mutex::new(false)); + for mut vcpu in self.vcpus.drain(..) { println!("Starting vCPU {:?}", vcpu.index); - let _ = thread::Builder::new().spawn(move || loop { - vcpu.run(); + + let should_stop_cloned = Arc::clone(&should_stop); + + let handler = thread::Builder::new().spawn(move || loop { + if *should_stop_cloned.lock().unwrap() { + println!("Stopping vCPU {:?}", vcpu.index); + break; + } + + vcpu.run(no_console, Arc::clone(&should_stop_cloned)); }); + + match handler { + Ok(handler) => handlers.push(handler), + Err(_) => { + println!("Failed to start vCPU"); + return Err(Error::AccessThreadHandlerError); + } + } + } + + if no_console { + for handler in handlers { + handler.join().map_err(Error::JoinThreadError)? + } + + return Ok(()); // We don't want to start the console if we are in no_console mode. } let stdin = io::stdin(); @@ -380,10 +424,11 @@ impl VMM { mem_size_mb: u32, kernel_path: &str, console: Option, + no_console: bool, initramfs_path: Option, if_name: Option, ) -> Result<()> { - self.configure_console(console)?; + self.configure_console(console, no_console)?; self.configure_memory(mem_size_mb)?; self.load_default_cmdline()?; From dd82520b4a051655eb21f7a3f24e4e7f97a0991c Mon Sep 17 00:00:00 2001 From: Maxime Date: Thu, 23 Mar 2023 23:51:25 +0100 Subject: [PATCH 10/13] feat : add writer implementation Signed-off-by: Maxime --- src/main.rs | 8 ++++++++ src/vmm/src/devices/mod.rs | 27 +++++++++++++++++++++++++++ src/vmm/src/lib.rs | 16 +++++++++++++++- 3 files changed, 50 insertions(+), 1 deletion(-) diff --git a/src/main.rs b/src/main.rs index b35e37f..db6b85f 100644 --- a/src/main.rs +++ b/src/main.rs @@ -70,6 +70,14 @@ fn main() -> Result<(), Error> { ) .map_err(Error::VmmConfigure)?; + // To use Writer with serial device : + // * Create mpsc channel : + // let (tx, rx) = std::sync::mpsc::channel(); + // * Create a new Writer + // let writer = Writer::new(tx); + // * Add the Writer when configuring the VMM + // * Use the rx receiver to read the data + // Run the VMM vmm.run(opts.no_console).map_err(Error::VmmRun)?; diff --git a/src/vmm/src/devices/mod.rs b/src/vmm/src/devices/mod.rs index ea91bcb..169c642 100644 --- a/src/vmm/src/devices/mod.rs +++ b/src/vmm/src/devices/mod.rs @@ -1,4 +1,31 @@ // SPDX-License-Identifier: Apache-2.0 +use std::io::{Result, Write}; +use std::sync::mpsc; + pub(crate) mod net; pub(crate) mod serial; + +pub struct Writer { + tx: mpsc::Sender, +} + +impl Write for Writer { + fn write(&mut self, buf: &[u8]) -> Result { + let s = String::from_utf8_lossy(buf); + self.tx + .send(s.to_string()) + .map_err(|_| std::io::Error::new(std::io::ErrorKind::Other, "Error sending data"))?; + Ok(buf.len()) + } + + fn flush(&mut self) -> Result<()> { + Ok(()) + } +} + +impl Writer { + pub fn new(tx: mpsc::Sender) -> Self { + Writer { tx } + } +} diff --git a/src/vmm/src/lib.rs b/src/vmm/src/lib.rs index 3a8cf8e..ffeed34 100644 --- a/src/vmm/src/lib.rs +++ b/src/vmm/src/lib.rs @@ -19,6 +19,7 @@ use std::{io, path::PathBuf}; use devices::net::tap::Tap; use devices::net::VirtioNet; +use devices::Writer; use kvm_bindings::{kvm_userspace_memory_region, KVM_MAX_CPUID_ENTRIES}; use kvm_ioctls::{Kvm, VmFd}; use linux_loader::loader::{self, KernelLoaderResult}; @@ -29,7 +30,7 @@ use vmm_sys_util::eventfd::EventFd; use vmm_sys_util::terminal::Terminal; mod cpu; use cpu::{cpuid, mptable, Vcpu}; -mod devices; +pub mod devices; use devices::serial::LumperSerial; mod epoll_context; @@ -88,6 +89,8 @@ pub enum Error { AccessThreadHandlerError, /// Join thread error JoinThreadError(Box), + /// Writer configuration error + WriterError(io::Error), } /// Dedicated [`Result`](https://doc.rust-lang.org/std/result/) type. @@ -253,6 +256,14 @@ impl VMM { Ok(()) } + pub fn configure_writer(&mut self, writer: Option) -> Result<()> { + if let Some(writer) = writer { + let mut serial = self.serial.lock().unwrap(); + *serial = LumperSerial::new(Box::new(writer)).map_err(Error::WriterError)?; + } + Ok(()) + } + pub fn configure_console( &mut self, console_path: Option, @@ -418,6 +429,9 @@ impl VMM { } } + console: Option, + no_console: bool, + pub fn configure( &mut self, num_vcpus: u8, From a97939a995bb77e1edc215ca25e8d6942f210523 Mon Sep 17 00:00:00 2001 From: Maxime Date: Thu, 23 Mar 2023 13:48:04 +0100 Subject: [PATCH 11/13] fix : dont send specials characters We don't want to send new line (10) and carriage return (13) to the receiver. Signed-off-by: Maxime --- src/vmm/src/devices/mod.rs | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/vmm/src/devices/mod.rs b/src/vmm/src/devices/mod.rs index 169c642..9ea0ca0 100644 --- a/src/vmm/src/devices/mod.rs +++ b/src/vmm/src/devices/mod.rs @@ -12,10 +12,15 @@ pub struct Writer { impl Write for Writer { fn write(&mut self, buf: &[u8]) -> Result { - let s = String::from_utf8_lossy(buf); - self.tx - .send(s.to_string()) - .map_err(|_| std::io::Error::new(std::io::ErrorKind::Other, "Error sending data"))?; + if buf.len() > 0 && (buf[0] != 10 && buf[0] != 13) { + let s = String::from_utf8_lossy(buf).to_string(); + self.tx.send(s).map_err(|_| { + std::io::Error::new( + std::io::ErrorKind::Other, + "Error while sending data to channel", + ) + })?; + } Ok(buf.len()) } From ed58c443049cc99d7e533fb1b80d1bc66db55eef Mon Sep 17 00:00:00 2001 From: Maxime Date: Thu, 23 Mar 2023 23:47:14 +0100 Subject: [PATCH 12/13] fix : dependency lumper Signed-off-by: Maxime --- Cargo.lock | 45 +++++++++++++++++++++++++++++++++++++++++++ src/vmm/Cargo.toml | 3 +++ src/vmm/src/kernel.rs | 2 -- 3 files changed, 48 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 48f012f..9c958fc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -122,6 +122,12 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "itoa" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6" + [[package]] name = "kvm-bindings" version = "0.6.0" @@ -248,6 +254,43 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "ryu" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" + +[[package]] +name = "serde" +version = "1.0.156" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "314b5b092c0ade17c00142951e50ced110ec27cea304b1037c6969246c2469a4" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.156" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7e29c4601e36bcec74a223228dce795f4cd3616341a4af93520ca1a837c087d" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d721eca97ac802aa7777b701877c8004d950fc142651367300d21c1cc0194744" +dependencies = [ + "itoa", + "ryu", + "serde", +] + [[package]] name = "strsim" version = "0.10.0" @@ -352,6 +395,8 @@ dependencies = [ "kvm-ioctls", "libc", "linux-loader", + "serde", + "serde_json", "virtio-bindings 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "virtio-device", "virtio-queue", diff --git a/src/vmm/Cargo.toml b/src/vmm/Cargo.toml index 7c23c78..403ddfc 100644 --- a/src/vmm/Cargo.toml +++ b/src/vmm/Cargo.toml @@ -21,3 +21,6 @@ virtio-device = { git = "https://github.com/rust-vmm/vm-virtio" } virtio-queue = { git = "https://github.com/rust-vmm/vm-virtio" } vm-superio = "0.7.0" + +serde_json = "1.0.94" +serde = { version = "1.0.126", features = ["derive"] } \ No newline at end of file diff --git a/src/vmm/src/kernel.rs b/src/vmm/src/kernel.rs index 4a5f01f..b562ea3 100644 --- a/src/vmm/src/kernel.rs +++ b/src/vmm/src/kernel.rs @@ -1,5 +1,3 @@ -// SPDX-License-Identifier: Apache-2.0 OR BSD-3-Clause - #![cfg(target_arch = "x86_64")] use std::fs::File; From dd57431a09d069b9b1083c3726f1c4501dd89850 Mon Sep 17 00:00:00 2001 From: Maxime Date: Fri, 31 Mar 2023 11:00:49 +0200 Subject: [PATCH 13/13] fix : read from unix socket Signed-off-by: Maxime --- Cargo.lock | 4 ++-- src/main.rs | 44 ++++++++++++++++++++++++++++---------- src/vmm/src/cpu/mod.rs | 2 -- src/vmm/src/devices/mod.rs | 20 ++++++----------- src/vmm/src/lib.rs | 28 +++++++++++++++++------- 5 files changed, 62 insertions(+), 36 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9c958fc..a0f81ff 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -310,9 +310,9 @@ dependencies = [ [[package]] name = "termcolor" -version = "1.1.2" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dfed899f0eb03f32ee8c6a0aabdb8a7949659e3466561fc0adf54e26d88c5f4" +checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6" dependencies = [ "winapi-util", ] diff --git a/src/main.rs b/src/main.rs index db6b85f..7b0c8ee 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,7 +1,7 @@ -use std::u32; +use std::{io::Read, os::unix::net::UnixListener, path::Path, thread::sleep, u32}; use clap::Parser; -use vmm::VMM; +use vmm::{devices::Writer, VMM}; #[derive(Parser)] #[clap(version = "0.1", author = "Polytech Montpellier - DevOps")] @@ -37,6 +37,9 @@ struct VMMOpts { /// no-console #[clap(long)] no_console: bool, + + #[clap(long)] + socket: bool, } #[derive(Debug)] @@ -51,6 +54,32 @@ pub enum Error { fn main() -> Result<(), Error> { let opts: VMMOpts = VMMOpts::parse(); + let console = opts.console.unwrap(); + if opts.socket { + let path = Path::new(console.as_str()); + if std::fs::metadata(path).is_ok() { + std::fs::remove_file(path).unwrap(); + } + + println!("Socket path: {}", path.to_str().unwrap()); + + let unix_listener = UnixListener::bind(path).unwrap(); + + std::thread::spawn(move || { + // read from socket + let (mut stream, _) = unix_listener.accept().unwrap(); + let mut buffer = [0; 1024]; + loop { + let n = stream.read(&mut buffer).unwrap(); + if n == 0 { + break; + } + let s = String::from_utf8_lossy(&buffer[0..n]).to_string(); + print!("{}", s); + } + }); + } + // Create a new VMM let mut vmm = VMM::new().map_err(Error::VmmNew)?; @@ -63,21 +92,14 @@ fn main() -> Result<(), Error> { opts.cpus, opts.memory, &opts.kernel, - opts.console, + Some(console), opts.no_console, opts.initramfs, opts.net, + opts.socket, ) .map_err(Error::VmmConfigure)?; - // To use Writer with serial device : - // * Create mpsc channel : - // let (tx, rx) = std::sync::mpsc::channel(); - // * Create a new Writer - // let writer = Writer::new(tx); - // * Add the Writer when configuring the VMM - // * Use the rx receiver to read the data - // Run the VMM vmm.run(opts.no_console).map_err(Error::VmmRun)?; diff --git a/src/vmm/src/cpu/mod.rs b/src/vmm/src/cpu/mod.rs index c7e0467..b72e54c 100644 --- a/src/vmm/src/cpu/mod.rs +++ b/src/vmm/src/cpu/mod.rs @@ -227,9 +227,7 @@ impl Vcpu { // Call into KVM to launch (VMLAUNCH) or resume (VMRESUME) the virtual CPU. // This is a blocking function, it only returns for either an error or a // VM-Exit. In the latter case, we can inspect the exit reason. - println!("Before running vCPU {}...", self.index); let run = self.vcpu_fd.run(); - println!("After running vCPU {}...", self.index); match run { Ok(exit_reason) => match exit_reason { diff --git a/src/vmm/src/devices/mod.rs b/src/vmm/src/devices/mod.rs index 9ea0ca0..d6bb0e4 100644 --- a/src/vmm/src/devices/mod.rs +++ b/src/vmm/src/devices/mod.rs @@ -1,26 +1,20 @@ // SPDX-License-Identifier: Apache-2.0 use std::io::{Result, Write}; -use std::sync::mpsc; +use std::os::unix::net::UnixStream; pub(crate) mod net; pub(crate) mod serial; pub struct Writer { - tx: mpsc::Sender, + unix_stream: UnixStream, } impl Write for Writer { fn write(&mut self, buf: &[u8]) -> Result { - if buf.len() > 0 && (buf[0] != 10 && buf[0] != 13) { - let s = String::from_utf8_lossy(buf).to_string(); - self.tx.send(s).map_err(|_| { - std::io::Error::new( - std::io::ErrorKind::Other, - "Error while sending data to channel", - ) - })?; - } + let s = String::from_utf8_lossy(buf).to_string(); + let _ = &self.unix_stream.write(s.as_bytes()).unwrap(); + Ok(buf.len()) } @@ -30,7 +24,7 @@ impl Write for Writer { } impl Writer { - pub fn new(tx: mpsc::Sender) -> Self { - Writer { tx } + pub fn new(unix_stream: UnixStream) -> Self { + Writer { unix_stream } } } diff --git a/src/vmm/src/lib.rs b/src/vmm/src/lib.rs index ffeed34..be11e00 100644 --- a/src/vmm/src/lib.rs +++ b/src/vmm/src/lib.rs @@ -12,6 +12,7 @@ use std::any::Any; use std::fs::File; use std::io::stdout; use std::os::unix::io::AsRawFd; +use std::os::unix::net::UnixStream; use std::os::unix::prelude::RawFd; use std::sync::{Arc, Mutex}; use std::thread; @@ -268,6 +269,7 @@ impl VMM { &mut self, console_path: Option, disable_console: bool, + is_socket: bool, ) -> Result<()> { if disable_console { return Ok(()); @@ -278,11 +280,23 @@ impl VMM { .map_err(Error::Cmdline)?; if let Some(console_path) = console_path { - // We create the file if it does not exist, else we open - let file = File::create(&console_path).map_err(Error::ConsoleError)?; + if is_socket { + println!("Connecting to socket: {}", console_path); + let unix_stream = UnixStream::connect(console_path).unwrap(); - let mut serial = self.serial.lock().unwrap(); - *serial = LumperSerial::new(Box::new(file)).map_err(Error::SerialCreation)?; + // create writer + + let writer = Writer::new(unix_stream); + let mut serial = self.serial.lock().unwrap(); + + *serial = LumperSerial::new(Box::new(writer)).map_err(Error::SerialCreation)?; + } else { + // We create the file if it does not exist, else we open + let file = File::create(&console_path).map_err(Error::ConsoleError)?; + + let mut serial = self.serial.lock().unwrap(); + *serial = LumperSerial::new(Box::new(file)).map_err(Error::SerialCreation)?; + } } Ok(()) @@ -429,9 +443,6 @@ impl VMM { } } - console: Option, - no_console: bool, - pub fn configure( &mut self, num_vcpus: u8, @@ -441,8 +452,9 @@ impl VMM { no_console: bool, initramfs_path: Option, if_name: Option, + is_socket: bool, ) -> Result<()> { - self.configure_console(console, no_console)?; + self.configure_console(console, no_console, is_socket)?; self.configure_memory(mem_size_mb)?; self.load_default_cmdline()?;