diff --git a/docs/src/SUMMARY.md b/docs/src/SUMMARY.md index 183d4681b..09dfc9501 100644 --- a/docs/src/SUMMARY.md +++ b/docs/src/SUMMARY.md @@ -39,6 +39,7 @@ - [NVIDIA Jetson AGX Orin: UART Passthrough](technologies/nvidia_agx_pt_uart.md) - [NVIDIA Jetson AGX Orin: PCIe Passthrough](technologies/nvidia_agx_pt_pcie.md) - [Generic x86: PCIe Passthrough on crosvm](technologies/x86_pcie_crosvm.md) + - [Platform Bus Virtualization - NVIDIA BPMP](technologies/nvidia_virtualization_bpmp.md) - [Hypervisor Options](technologies/hypervisor_options.md) # Build System and Supply Chain diff --git a/docs/src/technologies/nvidia_virtualization_bpmp.md b/docs/src/technologies/nvidia_virtualization_bpmp.md new file mode 100644 index 000000000..0ed8d1dea --- /dev/null +++ b/docs/src/technologies/nvidia_virtualization_bpmp.md @@ -0,0 +1,105 @@ + + +# NVIDIA Jetson AGX Orin: Boot and Power Management Processor Virtualization + +## Introduction + +Boot and Power Management Processor (BPMP) virtualization on the NVIDIA Jetson AGX Orin involves enabling virtual machines (VMs) to access specific BPMP resources. This capability is crucial for passing through platform devices where control over resets and clocks configurations is required. + +## Architectural Overview + +- **Resource Access**: BPMP virtualization allows VMs to access and manage resources such as device clocks and resets. +- **Foundation for Device Virtualization**: This setup lays the groundwork for future virtualization of more complex devices like GPUs. +- **Module Introduction**: A new `virtualization` module is introduced, divided into `common` and `host` modules, with a plan to add a `guest` module for NixOS-based guests. +- **Device Tree Configurations**: Modifications are made via patching to support virtualization features. +- **Compatibility**: The current implementation supports a Ghaf host with an Ubuntu guest. + +## Use Cases + +The current implementation includes a host configuration for UARTA +passthrough as a test case, demonstrating the practical application of +BPMP virtualization. Current implementation still requires a manuall +built Ubuntu guest. Work continues to integrate `microvm.nix` declared +guest that supports NVIDIA BPMP virtualization with the UARTA +passthrough demo. In general, this is work is important for future +NVIDIA Jetson platform bus GPU passthrough. With this feature, it is +possible to virtualize NVIDIA Jetson integrated GPU connected to +platform bus. + +## Instructions for Using BPMP Virtualization Options on NVIDIA Jetson AGX Orin + +1. Enable NVIDIA BPMP virtualization on Ghaf host for a NVIDIA +Jetson-target using the following configuration options: + + +```nix + hardware.nvidia = { + virtualization.enable = true; + passthroughs.uarta.enable = true; +}; +``` +Please note that these options are integrated to [NVIDIA Jetson Orin + targets](https://github.com/tiiuae/ghaf/blob/main/targets/nvidia-jetson-orin/default.nix) + but disabled by default until the implementation is finished. + +2. Build the target and boot with the image. You can write the image +to an SSD for testing with a recent NVIDIA UEFI FW. + +## Testing + +### Host + +1. Check for `bpmp-host` device. + +``` +[ghaf@ghaf-host:~]$ ls /dev | grep bpmp-host +bpmp-host +``` + +2. Check that `vfio-platform` binding is successful. + +``` +ghaf@ghaf-host:~]$ ls -l /sys/bus/platform/drivers/vfio-platform/3100000.serial +lrwxrwxrwx 1 root root 0 Dec 8 08:26 /sys/bus/platform/drivers/vfio-platform/3100000.serial -> ../../../../devices/platform/3100000.serial +``` + +### Guest for UARTA Test + +1. Build guest kernel according to instructions at [https://github.com/jpruiz84/bpmp-virt](https://github.com/jpruiz84/bpmp-virt) and use the following script to start the vm (IMG is the kernel image and FS the rootfs). + +``` +IMG=$1 +FS=$2 + +qemu-system-aarch64 \ + -nographic \ + -machine virt,accel=kvm \ + -cpu host \ + -m 1G \ + -no-reboot \ + -kernel $IMG \ + -drive file=$FS,if=virtio,format=qcow2 \ + -net user,hostfwd=tcp::2222-:22 -net nic \ + -device vfio-platform,host=3100000.serial \ + -dtb virt.dtb \ + -append "rootwait root=/dev/vda console=ttyAMA0" +``` + +2. With UARTA connected as instructed in [https://github.com/jpruiz84/bpmp-virt](https://github.com/jpruiz84/bpmp-virt), start minicom on the working PC: + +``` +minicom -b 9600 -D /dev/ttyUSB0 +``` + +3. Test UARTA by echoing a string to the correct tty in the vm: + +``` +echo 123 > /dev/ttyTHS0 +``` + +## Related Topics + +- [NVIDIA Jetson AGX Orin: UART Passthrough](https://tiiuae.github.io/ghaf/technologies/nvidia_agx_pt_uart.html#nvidia-jetson-agx-orin-uart-passthrough) diff --git a/hydrajobs.nix b/hydrajobs.nix index 7d05a2fc2..3d76a62ae 100644 --- a/hydrajobs.nix +++ b/hydrajobs.nix @@ -1,6 +1,19 @@ # Copyright 2022-2024 TII (SSRC) and the Ghaf contributors # SPDX-License-Identifier: Apache-2.0 -{self, ...}: { +{self, ...}: let + mkBpmpEnabled = cfg: let + bpmpEnableModule = {lib, ...}: { + ghaf.hardware.nvidia = { + virtualization.enable = lib.mkForce true; + virtualization.host.bpmp.enable = lib.mkForce true; + passthroughs.host.uarta.enable = lib.mkForce true; + }; + }; + newCfg = cfg.extendModules {modules = [bpmpEnableModule];}; + package = newCfg.config.system.build.${newCfg.config.formatAttr}; + in + package; +in { flake.hydraJobs = { generic-x86_64-debug.x86_64-linux = self.packages.x86_64-linux.generic-x86_64-debug; lenovo-x1-carbon-gen11-debug.x86_64-linux = self.packages.x86_64-linux.lenovo-x1-carbon-gen11-debug; @@ -19,5 +32,11 @@ # Build also cross-compiled images without demo apps nvidia-jetson-orin-agx-debug-nodemoapps-from-x86_64.x86_64-linux = self.packages.x86_64-linux.nvidia-jetson-orin-agx-debug-nodemoapps-from-x86_64; nvidia-jetson-orin-nx-debug-nodemoapps-from-x86_64.x86_64-linux = self.packages.x86_64-linux.nvidia-jetson-orin-nx-debug-nodemoapps-from-x86_64; + + # BPMP virt enabled versions + nvidia-jetson-orin-agx-debug-bpmp.aarch64-linux = mkBpmpEnabled self.nixosConfigurations.nvidia-jetson-orin-agx-debug; + nvidia-jetson-orin-nx-debug-bpmp.aarch64-linux = mkBpmpEnabled self.nixosConfigurations.nvidia-jetson-orin-nx-debug; + nvidia-jetson-orin-agx-debug-bpmp-from-x86_64.x86_64-linux = mkBpmpEnabled self.nixosConfigurations.nvidia-jetson-orin-agx-debug-from-x86_64; + nvidia-jetson-orin-nx-debug-bpmp-from-x86_64.x86_64-linux = mkBpmpEnabled self.nixosConfigurations.nvidia-jetson-orin-nx-debug-from-x86_64; }; } diff --git a/modules/hardware/nvidia-jetson-orin/default.nix b/modules/hardware/nvidia-jetson-orin/default.nix index 394b9efcd..28d4032da 100644 --- a/modules/hardware/nvidia-jetson-orin/default.nix +++ b/modules/hardware/nvidia-jetson-orin/default.nix @@ -13,5 +13,6 @@ ./nx-netvm-ethernet-pci-passthrough.nix ./ota-utils-fix.nix + ./virtualization ]; } diff --git a/modules/hardware/nvidia-jetson-orin/virtualization/common/bpmp-virt-common/default.nix b/modules/hardware/nvidia-jetson-orin/virtualization/common/bpmp-virt-common/default.nix new file mode 100644 index 000000000..c2b5968a2 --- /dev/null +++ b/modules/hardware/nvidia-jetson-orin/virtualization/common/bpmp-virt-common/default.nix @@ -0,0 +1,63 @@ +# Copyright 2022-2023 TII (SSRC) and the Ghaf contributors +# SPDX-License-Identifier: Apache-2.0 +{ + lib, + config, + ... +}: let + cfg = config.ghaf.hardware.nvidia.virtualization; +in { + options.ghaf.hardware.nvidia.virtualization.enable = lib.mkOption { + type = lib.types.bool; + default = false; + description = '' + Enable virtualization support for NVIDIA Orin + + This option is an implementation level detail and is toggled automatically + by modules that need it. Manually enabling this option is not recommended in + release builds. + ''; + }; + + config = lib.mkIf cfg.enable { + boot.kernelPatches = [ + { + name = "Added Configurations to Support Vda"; + patch = null; + extraStructuredConfig = with lib.kernel; { + PCI_STUB = lib.mkDefault yes; + VFIO = lib.mkDefault yes; + VIRTIO_PCI = lib.mkDefault yes; + VIRTIO_MMIO = lib.mkDefault yes; + HOTPLUG_PCI = lib.mkDefault yes; + PCI_DEBUG = lib.mkDefault yes; + PCI_HOST_GENERIC = lib.mkDefault yes; + VFIO_IOMMU_TYPE1 = lib.mkDefault yes; + HOTPLUG_PCI_ACPI = lib.mkDefault yes; + PCI_HOST_COMMON = lib.mkDefault yes; + VFIO_PLATFORM = lib.mkDefault yes; + TEGRA_BPMP_GUEST_PROXY = lib.mkDefault no; + TEGRA_BPMP_HOST_PROXY = lib.mkDefault no; + }; + } + { + name = "Vfio_platform Reset Required False"; + patch = ./patches/0002-vfio_platform-reset-required-false.patch; + } + { + name = "Bpmp Support Virtualization"; + patch = ./patches/0003-bpmp-support-bpmp-virt.patch; + } + { + name = "Bpmp Virt Drivers"; + patch = ./patches/0004-bpmp-virt-drivers.patch; + } + { + name = "Bpmp Overlay"; + patch = ./patches/0005-bpmp-overlay.patch; + } + ]; + + boot.kernelParams = ["vfio_iommu_type1.allow_unsafe_interrupts=1"]; + }; +} diff --git a/modules/hardware/nvidia-jetson-orin/virtualization/common/bpmp-virt-common/patches/0002-vfio_platform-reset-required-false.patch b/modules/hardware/nvidia-jetson-orin/virtualization/common/bpmp-virt-common/patches/0002-vfio_platform-reset-required-false.patch new file mode 100644 index 000000000..df060edb7 --- /dev/null +++ b/modules/hardware/nvidia-jetson-orin/virtualization/common/bpmp-virt-common/patches/0002-vfio_platform-reset-required-false.patch @@ -0,0 +1,24 @@ +From 03c7ecfd16e46838ab70c43a18990ef3dd35d08c Mon Sep 17 00:00:00 2001 +From: Juan Pablo Ruiz +Date: Thu, 4 May 2023 12:19:37 +0400 +Subject: [PATCH 2/3] vfio_platform: reset required false + +--- + drivers/vfio/platform/vfio_platform.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/drivers/vfio/platform/vfio_platform.c b/drivers/vfio/platform/vfio_platform.c +index 1e2769010089..3eabe37f400d 100644 +--- a/drivers/vfio/platform/vfio_platform.c ++++ b/drivers/vfio/platform/vfio_platform.c +@@ -15,7 +15,7 @@ + #define DRIVER_AUTHOR "Antonios Motakis " + #define DRIVER_DESC "VFIO for platform devices - User Level meta-driver" + +-static bool reset_required = true; ++static bool reset_required = false; + module_param(reset_required, bool, 0444); + MODULE_PARM_DESC(reset_required, "override reset requirement (default: 1)"); + +-- +2.25.1 diff --git a/modules/hardware/nvidia-jetson-orin/virtualization/common/bpmp-virt-common/patches/0003-bpmp-support-bpmp-virt.patch b/modules/hardware/nvidia-jetson-orin/virtualization/common/bpmp-virt-common/patches/0003-bpmp-support-bpmp-virt.patch new file mode 100644 index 000000000..a2871e572 --- /dev/null +++ b/modules/hardware/nvidia-jetson-orin/virtualization/common/bpmp-virt-common/patches/0003-bpmp-support-bpmp-virt.patch @@ -0,0 +1,153 @@ +diff --git a/drivers/firmware/tegra/bpmp-tegra186.c b/drivers/firmware/tegra/bpmp-tegra186.c +index f1a7ffa1b8dc..4d34a916f4ec 100644 +--- a/drivers/firmware/tegra/bpmp-tegra186.c ++++ b/drivers/firmware/tegra/bpmp-tegra186.c +@@ -30,6 +30,9 @@ struct tegra186_bpmp { + } mbox; + }; + ++extern uint64_t bpmp_vpa; ++int tegra_bpmp_guest_init(void); ++ + static inline struct tegra_bpmp * + mbox_client_to_bpmp(struct mbox_client *client) + { +@@ -177,7 +180,17 @@ static int tegra186_bpmp_init(struct tegra_bpmp *bpmp) + bpmp->priv = priv; + priv->parent = bpmp; + +- priv->tx.pool = of_gen_pool_get(bpmp->dev->of_node, "shmem", 0); ++#ifdef CONFIG_TEGRA_BPMP_GUEST_PROXY ++ // If virtual-pa node is defined, it means that we are using a virtual BPMP ++ // then we have to initialize the bpmp-guest ++ err = of_property_read_u64(bpmp->dev->of_node, "virtual-pa", &bpmp_vpa); ++ if(!err){ ++ printk("BPMP virtual-pa: 0x%llX", bpmp_vpa); ++ return tegra_bpmp_guest_init(); ++ } ++#endif ++ ++ priv->tx.pool = of_gen_pool_get(bpmp->dev->of_node, "shmem", 0); + if (!priv->tx.pool) { + dev_err(bpmp->dev, "TX shmem pool not found\n"); + return -EPROBE_DEFER; +@@ -267,6 +280,11 @@ static void tegra186_bpmp_deinit(struct tegra_bpmp *bpmp) + struct tegra186_bpmp *priv = bpmp->priv; + unsigned int i; + ++ // If using BPMP guest proxy, do no deinit the module ++ if(bpmp_vpa){ ++ return; ++ } ++ + mbox_free_channel(priv->mbox.channel); + + for (i = 0; i < bpmp->threaded.count; i++) +diff --git a/drivers/firmware/tegra/bpmp.c b/drivers/firmware/tegra/bpmp.c +index c7e39db14dac..802c2f0f7cf6 100644 +--- a/drivers/firmware/tegra/bpmp.c ++++ b/drivers/firmware/tegra/bpmp.c +@@ -40,6 +40,18 @@ channel_to_ops(struct tegra_bpmp_channel *channel) + return bpmp->soc->ops; + } + ++struct tegra_bpmp *tegra_bpmp_host_device = NULL; ++EXPORT_SYMBOL_GPL(tegra_bpmp_host_device); ++ ++int (*tegra_bpmp_transfer_redirect)(struct tegra_bpmp *bpmp, ++ struct tegra_bpmp_message *msg) = NULL; ++int tegra_bpmp_outloud = 0; ++uint64_t bpmp_vpa = 0; ++ ++EXPORT_SYMBOL_GPL(tegra_bpmp_transfer_redirect); ++EXPORT_SYMBOL_GPL(tegra_bpmp_outloud); ++EXPORT_SYMBOL_GPL(bpmp_vpa); ++ + struct tegra_bpmp *tegra_bpmp_get(struct device *dev) + { + struct platform_device *pdev; +@@ -65,6 +77,7 @@ struct tegra_bpmp *tegra_bpmp_get(struct device *dev) + + put: + of_node_put(np); ++ tegra_bpmp_host_device = bpmp; + return bpmp; + } + EXPORT_SYMBOL_GPL(tegra_bpmp_get); +@@ -315,6 +328,30 @@ static ssize_t tegra_bpmp_channel_write(struct tegra_bpmp_channel *channel, + return __tegra_bpmp_channel_write(channel, mrq, flags, data, size); + } + ++int _tegra_bpmp_transfer(struct tegra_bpmp *bpmp, ++ struct tegra_bpmp_message *msg) ++{ ++ int err = 0; ++ ++ // vadikas -- redirect request to virtio module ++ // the tegra_bpmp_transfer_redirect code is in bpmp-virt overlay ++ if (tegra_bpmp_outloud){ ++ printk("tegra_bpmp_transfer_redirect tx: %x tx.size= %ld \n", ++ msg->mrq, msg->tx.size); ++ print_hex_dump(KERN_INFO, "tegra_bpmp_transfer_redirect tx:", ++ DUMP_PREFIX_NONE, 16, 1, msg->tx.data, msg->tx.size, false); ++ } ++ err = (*tegra_bpmp_transfer_redirect)(bpmp, msg); ++ ++ if (tegra_bpmp_outloud){ ++ printk("tegra_bpmp_transfer_redirect rx: err=%d\n msg->rx.ret=%d", ++ err, msg->rx.ret); ++ print_hex_dump(KERN_INFO, "tegra_bpmp_transfer_redirect rx:" , ++ DUMP_PREFIX_NONE, 16, 1, msg->rx.data, msg->rx.size, false); ++ } ++ return err; ++} ++ + int tegra_bpmp_transfer_atomic(struct tegra_bpmp *bpmp, + struct tegra_bpmp_message *msg) + { +@@ -331,6 +368,10 @@ int tegra_bpmp_transfer_atomic(struct tegra_bpmp *bpmp, + + spin_lock(&bpmp->atomic_tx_lock); + ++ // vadikas -- redirect request to virtio module ++ if (tegra_bpmp_transfer_redirect) ++ return _tegra_bpmp_transfer(bpmp, msg); ++ + err = tegra_bpmp_channel_write(channel, msg->mrq, MSG_ACK, + msg->tx.data, msg->tx.size); + if (err < 0) { +@@ -366,8 +407,17 @@ int tegra_bpmp_transfer(struct tegra_bpmp *bpmp, + if (!tegra_bpmp_message_valid(msg)) + return -EINVAL; + ++ if (tegra_bpmp_transfer_redirect) ++ return _tegra_bpmp_transfer(bpmp, msg); ++ + channel = tegra_bpmp_write_threaded(bpmp, msg->mrq, msg->tx.data, + msg->tx.size); ++ ++ if (tegra_bpmp_outloud){ ++ printk("tegra_bpmp_transfer tx: %x tx.size= %ld \n", msg->mrq, msg->tx.size); ++ print_hex_dump(KERN_INFO, "tegra_bpmp_transfer tx:" ,DUMP_PREFIX_NONE, 16, 1, msg->tx.data, msg->tx.size, false); ++ } ++ + if (IS_ERR(channel)) + return PTR_ERR(channel); + +@@ -381,8 +431,15 @@ int tegra_bpmp_transfer(struct tegra_bpmp *bpmp, + if (err == 0) + return -ETIMEDOUT; + +- return tegra_bpmp_channel_read(channel, msg->rx.data, msg->rx.size, ++ err = tegra_bpmp_channel_read(channel, msg->rx.data, msg->rx.size, + &msg->rx.ret); ++ ++ if(tegra_bpmp_outloud){ ++ printk("tegra_bpmp_transfer rx: err=%d\n msg->rx.ret=%d", err, msg->rx.ret); ++ print_hex_dump(KERN_INFO,"tegra_bpmp_transfer rx:" ,DUMP_PREFIX_NONE, 16, 1, msg->rx.data, msg->rx.size, false); ++ } ++ ++ return err; + } + EXPORT_SYMBOL_GPL(tegra_bpmp_transfer); + diff --git a/modules/hardware/nvidia-jetson-orin/virtualization/common/bpmp-virt-common/patches/0004-bpmp-virt-drivers.patch b/modules/hardware/nvidia-jetson-orin/virtualization/common/bpmp-virt-common/patches/0004-bpmp-virt-drivers.patch new file mode 100644 index 000000000..5ccbc8124 --- /dev/null +++ b/modules/hardware/nvidia-jetson-orin/virtualization/common/bpmp-virt-common/patches/0004-bpmp-virt-drivers.patch @@ -0,0 +1,1050 @@ +--- + drivers/Kconfig | 8 + + drivers/Makefile | 5 + + drivers/bpmp-guest-proxy/Kconfig | 11 + + drivers/bpmp-guest-proxy/Makefile | 1 + + drivers/bpmp-guest-proxy/bpmp-guest-proxy.c | 444 ++++++++++++++++++ + drivers/bpmp-host-proxy/Kconfig | 10 + + drivers/bpmp-host-proxy/Makefile | 1 + + drivers/bpmp-host-proxy/bpmp-host-proxy.c | 470 ++++++++++++++++++++ + drivers/bpmp-host-proxy/bpmp-host-proxy.h | 17 + + 9 files changed, 967 insertions(+) + create mode 100644 drivers/bpmp-guest-proxy/Kconfig + create mode 100644 drivers/bpmp-guest-proxy/Makefile + create mode 100644 drivers/bpmp-guest-proxy/bpmp-guest-proxy.c + create mode 100644 drivers/bpmp-host-proxy/Kconfig + create mode 100644 drivers/bpmp-host-proxy/Makefile + create mode 100755 drivers/bpmp-host-proxy/bpmp-host-proxy.c + create mode 100644 drivers/bpmp-host-proxy/bpmp-host-proxy.h + +diff --git a/drivers/Kconfig b/drivers/Kconfig +index dcecc9f6e33f..39e2e85ac893 100644 +--- a/drivers/Kconfig ++++ b/drivers/Kconfig +@@ -236,3 +236,11 @@ source "drivers/counter/Kconfig" + + source "drivers/most/Kconfig" + endmenu ++append_menu "Device Drivers" ++ ++if ARCH_TEGRA ++source "drivers/bpmp-host-proxy/Kconfig" ++source "drivers/bpmp-guest-proxy/Kconfig" ++endif ++ ++endmenu +diff --git a/drivers/Makefile b/drivers/Makefile +index d3500440b928..06cb4bc1c69c 100644 +--- a/drivers/Makefile ++++ b/drivers/Makefile +@@ -191,3 +191,8 @@ obj-$(CONFIG_GNSS) += gnss/ + obj-$(CONFIG_INTERCONNECT) += interconnect/ + obj-$(CONFIG_COUNTER) += counter/ + obj-$(CONFIG_MOST) += most/ ++# ++# ++# ++obj-y += bpmp-host-proxy/ ++obj-y += bpmp-guest-proxy/ +\ No newline at end of file +diff --git a/drivers/bpmp-guest-proxy/Kconfig b/drivers/bpmp-guest-proxy/Kconfig +new file mode 100644 +index 000000000000..33877730ad5b +--- /dev/null ++++ b/drivers/bpmp-guest-proxy/Kconfig +@@ -0,0 +1,11 @@ ++config TEGRA_BPMP_GUEST_PROXY ++ bool "Tegra BPMP guest proxy driver" ++ depends on TEGRA_BPMP ++ help ++ The Tegra BPMP guest proxy driver, virtualize the BPMP transfer ++ function, allowing the guest to have access to the host BPMP. ++ ++ Say Y here to enable this driver and to compile this driver as a module, ++ choose M here. If unsure, say N ++ ++ +diff --git a/drivers/bpmp-guest-proxy/Makefile b/drivers/bpmp-guest-proxy/Makefile +new file mode 100644 +index 000000000000..ac22344a1596 +--- /dev/null ++++ b/drivers/bpmp-guest-proxy/Makefile +@@ -0,0 +1 @@ ++obj-$(CONFIG_TEGRA_BPMP_GUEST_PROXY) += bpmp-guest-proxy.o +diff --git a/drivers/bpmp-guest-proxy/bpmp-guest-proxy.c b/drivers/bpmp-guest-proxy/bpmp-guest-proxy.c +new file mode 100644 +index 000000000000..c4177435168a +--- /dev/null ++++ b/drivers/bpmp-guest-proxy/bpmp-guest-proxy.c +@@ -0,0 +1,444 @@ ++/** ++ * ++ * NVIDIA BPMP Guest Proxy Kernel Module ++ * (c) 2023 Unikie, Oy ++ * (c) 2023 Vadim Likholetov vadim.likholetov@unikie.com ++ * ++*/ ++#include // Core header for modules. ++#include // Supports driver model. ++#include // Kernel header for convenient functions. ++#include // File-system support. ++#include // User access copy function support. ++#include ++#include ++#include ++#include ++#include ++ ++ ++#define DEVICE_NAME "bpmp-guest" // Device name. ++#define CLASS_NAME "char" ++ ++MODULE_LICENSE("GPL"); ++MODULE_AUTHOR("Vadim Likholetov"); ++MODULE_DESCRIPTION("NVidia BPMP Guest Proxy Kernel Module"); ++MODULE_VERSION("0.1"); ++ ++ ++#define TX_BUF 0x0000 ++#define RX_BUF 0x0200 ++#define TX_SIZ 0x0400 ++#define RX_SIZ 0x0408 ++#define RET_COD 0x0410 ++#define MRQ 0x0500 ++#define MEM_SIZE 0x0600 ++#define MESSAGE_SIZE 0x0200 ++ ++ ++#define BPMP_GUEST_VERBOSE 0 ++ ++#if BPMP_GUEST_VERBOSE ++#define deb_info(...) printk(KERN_INFO DEVICE_NAME ": "__VA_ARGS__) ++#else ++#define deb_info(...) ++#endif ++ ++#define deb_error(...) printk(KERN_ALERT DEVICE_NAME ": "__VA_ARGS__) ++ ++ ++static volatile void __iomem *mem_iova = NULL; ++ ++extern int tegra_bpmp_transfer(struct tegra_bpmp *, struct tegra_bpmp_message *); ++extern struct tegra_bpmp *tegra_bpmp_host_device; ++int my_tegra_bpmp_transfer(struct tegra_bpmp *, struct tegra_bpmp_message *); ++ ++ ++extern int (*tegra_bpmp_transfer_redirect)(struct tegra_bpmp *, struct tegra_bpmp_message *); ++extern int tegra_bpmp_outloud; ++extern uint64_t bpmp_vpa; ++ ++ ++/** ++ * Important variables that store data and keep track of relevant information. ++ */ ++static int major_number; ++ ++static struct class *bpmp_guest_proxy_class = NULL; ///< The device-driver class struct pointer ++static struct device *bpmp_guest_proxy_device = NULL; ///< The device-driver device struct pointer ++ ++/** ++ * Prototype functions for file operations. ++ */ ++static int open(struct inode *, struct file *); ++static int close(struct inode *, struct file *); ++static ssize_t read(struct file *, char *, size_t, loff_t *); ++static ssize_t write(struct file *, const char *, size_t, loff_t *); ++ ++/** ++ * File operations structure and the functions it points to. ++ */ ++static struct file_operations fops = ++ { ++ .owner = THIS_MODULE, ++ .open = open, ++ .release = close, ++ .read = read, ++ .write = write, ++}; ++ ++ ++#if BPMP_GUEST_VERBOSE ++// Usage: ++// hexDump(desc, addr, len, perLine); ++// desc: if non-NULL, printed as a description before hex dump. ++// addr: the address to start dumping from. ++// len: the number of bytes to dump. ++// perLine: number of bytes on each output line. ++void static hexDump ( ++ const char * desc, ++ const void * addr, ++ const int len ++) { ++ // Silently ignore silly per-line values. ++ ++ int i; ++ unsigned char buff[17]; ++ unsigned char out_buff[200]; ++ unsigned char *p_out_buff = out_buff; ++ const unsigned char * pc = (const unsigned char *)addr; ++ ++ ++ ++ // Output description if given. ++ ++ if (desc != NULL) printk ("%s:\n", desc); ++ ++ // Length checks. ++ ++ if (len == 0) { ++ printk(" ZERO LENGTH\n"); ++ return; ++ } ++ if (len < 0) { ++ printk(" NEGATIVE LENGTH: %d\n", len); ++ return; ++ } ++ ++ if(len > 2046){ ++ printk(" VERY LONG: %d\n", len); ++ return; ++ } ++ ++ // Process every byte in the data. ++ ++ for (i = 0; i < len; i++) { ++ // Multiple of perLine means new or first line (with line offset). ++ ++ if ((i % 16) == 0) { ++ // Only print previous-line ASCII buffer for lines beyond first. ++ ++ if (i != 0) { ++ p_out_buff += sprintf (p_out_buff, " %s\n", buff); ++ printk("%s", out_buff); ++ memset(out_buff, 0, sizeof(out_buff)); ++ p_out_buff = out_buff; ++ } ++ // Output the offset of current line. ++ ++ p_out_buff += sprintf (p_out_buff," %04x ", i); ++ } ++ ++ // Now the hex code for the specific character. ++ ++ p_out_buff += sprintf (p_out_buff, " %02x", pc[i]); ++ ++ // And buffer a printable ASCII character for later. ++ ++ if ((pc[i] < 0x20) || (pc[i] > 0x7e)) // isprint() may be better. ++ buff[i % 16] = '.'; ++ else ++ buff[i % 16] = pc[i]; ++ buff[(i % 16) + 1] = '\0'; ++ } ++ ++ // Pad out last line if not exactly perLine characters. ++ ++ while ((i % 16) != 0) { ++ p_out_buff += sprintf (p_out_buff, " "); ++ i++; ++ } ++ ++ // And print the final ASCII buffer. ++ ++ p_out_buff += sprintf (p_out_buff, " %s\n", buff); ++ ++ printk("%s", out_buff); ++} ++#else ++ #define hexDump(...) ++#endif ++ ++/** ++ * Initializes module at installation ++ */ ++int tegra_bpmp_guest_init(void) ++{ ++ ++ ++ deb_info("%s, installing module.", __func__); ++ ++ deb_info("bpmp_vpa: 0x%llX", bpmp_vpa); ++ ++ if(!bpmp_vpa){ ++ deb_error("Failed, bpmp_vpa not defined\n"); ++ } ++ ++ // Allocate a major number for the device. ++ major_number = register_chrdev(0, DEVICE_NAME, &fops); ++ if (major_number < 0) ++ { ++ deb_error("could not register number.\n"); ++ return major_number; ++ } ++ deb_info("registered correctly with major number %d\n", major_number); ++ ++ // Register the device class ++ bpmp_guest_proxy_class = class_create(THIS_MODULE, CLASS_NAME); ++ if (IS_ERR(bpmp_guest_proxy_class)) ++ { // Check for error and clean up if there is ++ unregister_chrdev(major_number, DEVICE_NAME); ++ deb_error("Failed to register device class\n"); ++ return PTR_ERR(bpmp_guest_proxy_class); // Correct way to return an error on a pointer ++ } ++ deb_info("device class registered correctly\n"); ++ ++ // Register the device driver ++ bpmp_guest_proxy_device = device_create(bpmp_guest_proxy_class, NULL, MKDEV(major_number, 0), NULL, DEVICE_NAME); ++ if (IS_ERR(bpmp_guest_proxy_device)) ++ { // Clean up if there is an error ++ class_destroy(bpmp_guest_proxy_class); ++ unregister_chrdev(major_number, DEVICE_NAME); ++ deb_error("Failed to create the device\n"); ++ return PTR_ERR(bpmp_guest_proxy_device); ++ } ++ deb_info("device class created correctly\n"); // Made it! device was initialized ++ ++ // map iomem ++ mem_iova = ioremap(bpmp_vpa, MEM_SIZE); ++ ++ if (!mem_iova) { ++ deb_error("ioremap failed\n"); ++ return -ENOMEM; ++ } ++ ++ deb_info("bpmp_vpa: 0x%llX, mem_iova: %p\n", bpmp_vpa, mem_iova); ++ ++ tegra_bpmp_transfer_redirect = my_tegra_bpmp_transfer; // Hook func ++ ++ return 0; ++} ++ ++EXPORT_SYMBOL(tegra_bpmp_guest_init); ++ ++ ++ ++/* ++ * Removes module, sends appropriate message to kernel ++ */ ++void tegra_bpmp_guest_cleanup(void) ++{ ++ deb_info("removing module.\n"); ++ ++ // unmap iomem ++ iounmap((void __iomem*)bpmp_vpa); ++ ++ tegra_bpmp_transfer_redirect = NULL; // unhook function ++ device_destroy(bpmp_guest_proxy_class, MKDEV(major_number, 0)); // remove the device ++ class_unregister(bpmp_guest_proxy_class); // unregister the device class ++ class_destroy(bpmp_guest_proxy_class); // remove the device class ++ unregister_chrdev(major_number, DEVICE_NAME); // unregister the major number ++ deb_info("Goodbye from the LKM!\n"); ++ unregister_chrdev(major_number, DEVICE_NAME); ++ return; ++} ++ ++/* ++ * Opens device module, sends appropriate message to kernel ++ */ ++static int open(struct inode *inodep, struct file *filep) ++{ ++ deb_info("device opened.\n"); ++ tegra_bpmp_outloud = 1; ++ return 0; ++} ++ ++/* ++ * Closes device module, sends appropriate message to kernel ++ */ ++static int close(struct inode *inodep, struct file *filep) ++{ ++ deb_info("device closed.\n"); ++ tegra_bpmp_outloud = 0; ++ return 0; ++} ++ ++/* ++ * Reads from device, displays in userspace, and deletes the read data ++ */ ++static ssize_t read(struct file *filep, char *buffer, size_t len, loff_t *offset) ++{ ++ deb_info("read stub"); ++ return 0; ++} ++ ++ ++ ++int my_tegra_bpmp_transfer(struct tegra_bpmp *bpmp, struct tegra_bpmp_message *msg) ++{ ++ ++ unsigned char io_buffer[MEM_SIZE]; ++ deb_info("%s\n", __func__); ++ ++ memset(io_buffer, 0, sizeof(io_buffer)); ++ ++ if (msg->tx.size >= MESSAGE_SIZE) ++ return -EINVAL; ++ ++ // Copy msg, tx data and rx data to a single io_buffer ++ memcpy(&io_buffer[TX_BUF], msg->tx.data, msg->tx.size); ++ memcpy(&io_buffer[TX_SIZ], &msg->tx.size, sizeof(msg->tx.size)); ++ ++ memcpy(&io_buffer[RX_BUF], msg->rx.data, msg->rx.size); ++ memcpy(&io_buffer[RX_SIZ], &msg->rx.size, sizeof(msg->rx.size)); ++ ++ memcpy(&io_buffer[MRQ], &msg->mrq, sizeof(msg->mrq)); ++ ++ ++ hexDump("msg", &msg, sizeof(struct tegra_bpmp_message)); ++ deb_info("msg.tx.data: %p\n", msg->tx.data); ++ hexDump("msg.tx.data", msg->tx.data, msg->tx.size); ++ deb_info("msg->rx.size: %ld\n", msg->rx.size); ++ ++ // Execute the request by coping the io_buffer ++ memcpy_toio(mem_iova, io_buffer, MEM_SIZE); ++ ++ // Read response to io_buffer ++ memcpy_fromio(io_buffer, mem_iova, MEM_SIZE); ++ ++ // Copy from io_buffer to msg, tx data and rx data ++ memcpy(&msg->tx.size, &io_buffer[TX_SIZ], sizeof(msg->tx.size)); ++ memcpy((void *)msg->tx.data, &io_buffer[TX_BUF], msg->tx.size); ++ ++ memcpy(&msg->rx.size, &io_buffer[RX_SIZ], sizeof(msg->rx.size)); ++ memcpy(msg->rx.data, &io_buffer[RX_BUF], msg->rx.size); ++ ++ memcpy(&msg->rx.ret, &io_buffer[RET_COD], sizeof(msg->rx.ret)); ++ ++ deb_info("%s, END ret: %d\n", __func__, msg->rx.ret); ++ ++ return msg->rx.ret; ++} ++ ++/* ++ * Writes to the device ++ */ ++ ++#define BUF_SIZE 1024 ++ ++static ssize_t write(struct file *filep, const char *buffer, size_t len, loff_t *offset) ++{ ++ ++ int ret = len; ++ struct tegra_bpmp_message *kbuf = NULL; ++ void *txbuf = NULL; ++ void *rxbuf = NULL; ++ void *usertxbuf = NULL; ++ void *userrxbuf = NULL; ++ ++ if (len > 65535) { /* paranoia */ ++ deb_error("count %zu exceeds max # of bytes allowed, " ++ "aborting write\n", len); ++ goto out_nomem; ++ } ++ ++ deb_info(" wants to write %zu bytes\n", len); ++ ++ if (len!=sizeof(struct tegra_bpmp_message )) ++ { ++ deb_error("message size %zu != %zu", len, sizeof(struct tegra_bpmp_message)); ++ goto out_notok; ++ } ++ ++ ret = -ENOMEM; ++ kbuf = kmalloc(len, GFP_KERNEL); ++ txbuf = kmalloc(BUF_SIZE, GFP_KERNEL); ++ rxbuf = kmalloc(BUF_SIZE, GFP_KERNEL); ++ ++ if (!kbuf || !txbuf || !rxbuf) ++ goto out_nomem; ++ ++ memset(kbuf, 0, len); ++ memset(txbuf, 0, len); ++ memset(rxbuf, 0, len); ++ ++ ret = -EFAULT; ++ ++ if (copy_from_user(kbuf, buffer, len)) { ++ deb_error("copy_from_user(1) failed\n"); ++ goto out_cfu; ++ } ++ ++ if (copy_from_user(txbuf, kbuf->tx.data, kbuf->tx.size)) { ++ deb_error("copy_from_user(2) failed\n"); ++ goto out_cfu; ++ } ++ ++ if (copy_from_user(rxbuf, kbuf->rx.data, kbuf->rx.size)) { ++ deb_error("copy_from_user(3) failed\n"); ++ goto out_cfu; ++ } ++ ++ usertxbuf = (void*)kbuf->tx.data; //save userspace buffers addresses ++ userrxbuf = kbuf->rx.data; ++ ++ kbuf->tx.data=txbuf; //reassing to kernel space buffers ++ kbuf->rx.data=rxbuf; ++ ++ ++ ret = tegra_bpmp_transfer(tegra_bpmp_host_device, (struct tegra_bpmp_message *)kbuf); ++ ++ ++ ++ if (copy_to_user((void *)usertxbuf, kbuf->tx.data, kbuf->tx.size)) { ++ deb_error("copy_to_user(2) failed\n"); ++ goto out_notok; ++ } ++ ++ if (copy_to_user((void *)userrxbuf, kbuf->rx.data, kbuf->rx.size)) { ++ deb_error("copy_to_user(3) failed\n"); ++ goto out_notok; ++ } ++ ++ kbuf->tx.data=usertxbuf; ++ kbuf->rx.data=userrxbuf; ++ ++ if (copy_to_user((void *)buffer, kbuf, len)) { ++ deb_error("copy_to_user(1) failed\n"); ++ goto out_notok; ++ } ++ ++ ++ ++ kfree(kbuf); ++ return len; ++out_notok: ++out_nomem: ++ deb_error ("memory allocation failed"); ++out_cfu: ++ kfree(kbuf); ++ kfree(txbuf); ++ kfree(rxbuf); ++ return -EINVAL; ++} ++ +diff --git a/drivers/bpmp-host-proxy/Kconfig b/drivers/bpmp-host-proxy/Kconfig +new file mode 100644 +index 000000000000..08eb89367064 +--- /dev/null ++++ b/drivers/bpmp-host-proxy/Kconfig +@@ -0,0 +1,10 @@ ++config TEGRA_BPMP_HOST_PROXY ++ depends on TEGRA_BPMP ++ bool "Tegra BPMP host proxy support" ++ help ++ Exposes the BPMP capabilities to the user level, in order to support the ++ BPMP dependent devices passthrough to the virtual machines. ++ ++ Say Y here to enable this driver and to compile this driver as a module, ++ choose M here. If unsure, say N ++ +diff --git a/drivers/bpmp-host-proxy/Makefile b/drivers/bpmp-host-proxy/Makefile +new file mode 100644 +index 000000000000..ecc7233d8360 +--- /dev/null ++++ b/drivers/bpmp-host-proxy/Makefile +@@ -0,0 +1 @@ ++obj-$(CONFIG_TEGRA_BPMP_HOST_PROXY) += bpmp-host-proxy.o +diff --git a/drivers/bpmp-host-proxy/bpmp-host-proxy.c b/drivers/bpmp-host-proxy/bpmp-host-proxy.c +new file mode 100755 +index 000000000000..e20d50428b11 +--- /dev/null ++++ b/drivers/bpmp-host-proxy/bpmp-host-proxy.c +@@ -0,0 +1,470 @@ ++/** ++ * ++ * NVIDIA BPMP Host Proxy Kernel Module ++ * (c) 2023 Unikie, Oy ++ * (c) 2023 Vadim Likholetov vadim.likholetov@unikie.com ++ * ++*/ ++#include // Core header for modules. ++#include // Supports driver model. ++#include // Kernel header for convenient functions. ++#include // File-system support. ++#include // User access copy function support. ++#include ++#include ++#include ++#include "bpmp-host-proxy.h" ++ ++ ++#define DEVICE_NAME "bpmp-host" // Device name. ++#define CLASS_NAME "chardrv" // < The device class -- this is a character device driver ++ ++MODULE_LICENSE("GPL"); ///< The license type -- this affects available functionality ++MODULE_AUTHOR("Vadim Likholetov"); ///< The author -- visible when you use modinfo ++MODULE_DESCRIPTION("NVidia BPMP Host Proxy Kernel Module"); ///< The description -- see modinfo ++MODULE_VERSION("0.1"); ///< A version number to inform users ++ ++ ++#define BPMP_HOST_VERBOSE 0 ++ ++#if BPMP_HOST_VERBOSE ++#define deb_info(...) printk(KERN_INFO DEVICE_NAME ": "__VA_ARGS__) ++#else ++#define deb_info(...) ++#endif ++ ++#define deb_error(...) printk(KERN_ALERT DEVICE_NAME ": "__VA_ARGS__) ++ ++/** ++ * Important variables that store data and keep track of relevant information. ++ */ ++static int major_number; ++ ++static struct class *bpmp_host_proxy_class = NULL; ///< The device-driver class struct pointer ++static struct device *bpmp_host_proxy_device = NULL; ///< The device-driver device struct pointer ++ ++/** ++ * Prototype functions for file operations. ++ */ ++static int open(struct inode *, struct file *); ++static int close(struct inode *, struct file *); ++static ssize_t read(struct file *, char *, size_t, loff_t *); ++static ssize_t write(struct file *, const char *, size_t, loff_t *); ++ ++/** ++ * File operations structure and the functions it points to. ++ */ ++static struct file_operations fops = ++ { ++ .owner = THIS_MODULE, ++ .open = open, ++ .release = close, ++ .read = read, ++ .write = write, ++}; ++ ++// BPMP allowed resources structure ++static struct bpmp_allowed_res bpmp_ares; ++ ++#if BPMP_HOST_VERBOSE ++// Usage: ++// hexDump(desc, addr, len, perLine); ++// desc: if non-NULL, printed as a description before hex dump. ++// addr: the address to start dumping from. ++// len: the number of bytes to dump. ++// perLine: number of bytes on each output line. ++void static hexDump ( ++ const char * desc, ++ const void * addr, ++ const int len ++) { ++ // Silently ignore silly per-line values. ++ ++ int i; ++ unsigned char buff[17]; ++ unsigned char out_buff[4000]; ++ unsigned char *p_out_buff = out_buff; ++ const unsigned char * pc = (const unsigned char *)addr; ++ ++ ++ ++ // Output description if given. ++ ++ if (desc != NULL) printk ("%s:\n", desc); ++ ++ // Length checks. ++ ++ if (len == 0) { ++ printk(DEVICE_NAME ": ZERO LENGTH\n"); ++ return; ++ } ++ if (len < 0) { ++ printk(DEVICE_NAME ": NEGATIVE LENGTH: %d\n", len); ++ return; ++ } ++ ++ if(len > 400){ ++ printk(DEVICE_NAME ": VERY LONG: %d\n", len); ++ return; ++ } ++ ++ // Process every byte in the data. ++ ++ for (i = 0; i < len; i++) { ++ // Multiple of perLine means new or first line (with line offset). ++ ++ if ((i % 16) == 0) { ++ // Only print previous-line ASCII buffer for lines beyond first. ++ ++ if (i != 0) { ++ p_out_buff += sprintf (p_out_buff, " %s\n", buff); ++ } ++ // Output the offset of current line. ++ ++ p_out_buff += sprintf (p_out_buff," %04x ", i); ++ } ++ ++ // Now the hex code for the specific character. ++ ++ p_out_buff += sprintf (p_out_buff, " %02x", pc[i]); ++ ++ // And buffer a printable ASCII character for later. ++ ++ if ((pc[i] < 0x20) || (pc[i] > 0x7e)) // isprint() may be better. ++ buff[i % 16] = '.'; ++ else ++ buff[i % 16] = pc[i]; ++ buff[(i % 16) + 1] = '\0'; ++ } ++ ++ // Pad out last line if not exactly perLine characters. ++ ++ while ((i % 16) != 0) { ++ p_out_buff += sprintf (p_out_buff, " "); ++ i++; ++ } ++ ++ // And print the final ASCII buffer. ++ ++ p_out_buff += sprintf (p_out_buff, " %s\n", buff); ++ ++ printk(DEVICE_NAME ": %s", out_buff); ++} ++#else ++ #define hexDump(...) ++#endif ++ ++/** ++ * Initializes module at installation ++ */ ++static int bpmp_host_proxy_probe(struct platform_device *pdev) ++{ ++ int i; ++ ++ deb_info("%s, installing module.", __func__); ++ ++ // Read allowed clocks and reset from the device tree ++ // if clocks or resets are not defined, not initialize the module ++ bpmp_ares.clocks_size = of_property_read_variable_u32_array(pdev->dev.of_node, ++ "allowed-clocks", bpmp_ares.clock, 0, BPMP_HOST_MAX_CLOCKS_SIZE); ++ ++ if(bpmp_ares.clocks_size <= 0){ ++ deb_error("No allowed clocks defined"); ++ return EINVAL; ++ } ++ ++ deb_info("bpmp_ares.clocks_size: %d", bpmp_ares.clocks_size); ++ for (i = 0; i < bpmp_ares.clocks_size; i++) { ++ deb_info("bpmp_ares.clock %d", bpmp_ares.clock[i]); ++ } ++ ++ bpmp_ares.resets_size = of_property_read_variable_u32_array(pdev->dev.of_node, ++ "allowed-resets", bpmp_ares.reset, 0, BPMP_HOST_MAX_RESETS_SIZE); ++ ++ if(bpmp_ares.resets_size <= 0){ ++ deb_error("No allowed resets defined"); ++ return EINVAL; ++ } ++ ++ deb_info("bpmp_ares.resets_size: %d", bpmp_ares.resets_size); ++ for (i = 0; i < bpmp_ares.resets_size; i++) { ++ deb_info("bpmp_ares.reset %d", bpmp_ares.reset[i]); ++ } ++ ++ ++ // Allocate a major number for the device. ++ major_number = register_chrdev(0, DEVICE_NAME, &fops); ++ if (major_number < 0) ++ { ++ deb_error("could not register number.\n"); ++ return major_number; ++ } ++ deb_info("registered correctly with major number %d\n", major_number); ++ ++ // Register the device class ++ bpmp_host_proxy_class = class_create(THIS_MODULE, CLASS_NAME); ++ if (IS_ERR(bpmp_host_proxy_class)) ++ { // Check for error and clean up if there is ++ unregister_chrdev(major_number, DEVICE_NAME); ++ deb_error("Failed to register device class\n"); ++ return PTR_ERR(bpmp_host_proxy_class); // Correct way to return an error on a pointer ++ } ++ deb_info("device class registered correctly\n"); ++ ++ // Register the device driver ++ bpmp_host_proxy_device = device_create(bpmp_host_proxy_class, NULL, MKDEV(major_number, 0), NULL, DEVICE_NAME); ++ if (IS_ERR(bpmp_host_proxy_device)) ++ { // Clean up if there is an error ++ class_destroy(bpmp_host_proxy_class); ++ unregister_chrdev(major_number, DEVICE_NAME); ++ deb_error("Failed to create the device\n"); ++ return PTR_ERR(bpmp_host_proxy_device); ++ } ++ ++ deb_info("device class created correctly\n"); // Made it! device was initialized ++ ++ return 0; ++} ++ ++ ++ ++/* ++ * Removes module, sends appropriate message to kernel ++ */ ++static int bpmp_host_proxy_remove(struct platform_device *pdev) ++{ ++ deb_info("removing module.\n"); ++ device_destroy(bpmp_host_proxy_class, MKDEV(major_number, 0)); // remove the device ++ class_unregister(bpmp_host_proxy_class); // unregister the device class ++ class_destroy(bpmp_host_proxy_class); // remove the device class ++ unregister_chrdev(major_number, DEVICE_NAME); // unregister the major number ++ deb_info("Goodbye from the LKM!\n"); ++ unregister_chrdev(major_number, DEVICE_NAME); ++ return 0; ++} ++ ++/* ++ * Opens device module, sends appropriate message to kernel ++ */ ++static int open(struct inode *inodep, struct file *filep) ++{ ++ deb_info("device opened.\n"); ++ return 0; ++} ++ ++/* ++ * Closes device module, sends appropriate message to kernel ++ */ ++static int close(struct inode *inodep, struct file *filep) ++{ ++ deb_info("device closed.\n"); ++ return 0; ++} ++ ++/* ++ * Reads from device, displays in userspace, and deletes the read data ++ */ ++static ssize_t read(struct file *filep, char *buffer, size_t len, loff_t *offset) ++{ ++ deb_info("read stub"); ++ return 0; ++} ++ ++/* ++ * Checks if the msg that wants to transmit through the ++ * bpmp-host is allowed by the device tree configuration ++ */ ++static bool check_if_allowed(struct tegra_bpmp_message *msg) ++{ ++ struct mrq_reset_request *reset_req = NULL; ++ struct mrq_clk_request *clock_req = NULL; ++ uint32_t clk_cmd = 0; ++ int i = 0; ++ ++ // Allow get information mrq ++ if(msg->mrq == MRQ_PING || ++ msg->mrq == MRQ_QUERY_TAG || ++ msg->mrq == MRQ_THREADED_PING || ++ msg->mrq == MRQ_QUERY_ABI || ++ msg->mrq == MRQ_QUERY_FW_TAG ){ ++ return true; ++ } ++ ++ // Check for reset and clock mrq ++ if(msg->mrq == MRQ_RESET){ ++ reset_req = (struct mrq_reset_request*) msg->tx.data; ++ ++ for(i = 0; i < bpmp_ares.resets_size; i++){ ++ if(bpmp_ares.reset[i] == reset_req->reset_id){ ++ return true; ++ } ++ } ++ deb_error("Error, reset not allowed for: %d", reset_req->reset_id); ++ return false; ++ } ++ else if (msg->mrq == MRQ_CLK){ ++ clock_req = (struct mrq_clk_request*) msg->tx.data; ++ ++ for(i = 0; i < bpmp_ares.clocks_size; i++){ ++ // bits[23..0] are the clock id ++ if(bpmp_ares.clock[i] == (clock_req->cmd_and_id & 0x0FFF)){ ++ return true; ++ } ++ } ++ ++ clk_cmd = (clock_req->cmd_and_id >> 24) & 0x000F; ++ ++ // If there is a get info command, allow it no matters the ID ++ if(clk_cmd == CMD_CLK_GET_MAX_CLK_ID || ++ clk_cmd == CMD_CLK_GET_ALL_INFO || ++ clk_cmd == CMD_CLK_GET_PARENT){ ++ return true; ++ } ++ ++ deb_error("Error, clock not allowed for: %d, with command: %d", ++ clock_req->cmd_and_id & 0x0FFF, clk_cmd); ++ return false; ++ } ++ ++ deb_error("Error, msg->mrq %d not allowed", msg->mrq); ++ ++ return false; ++} ++ ++extern int tegra_bpmp_transfer(struct tegra_bpmp *, struct tegra_bpmp_message *); ++extern struct tegra_bpmp *tegra_bpmp_host_device; ++ ++#define BUF_SIZE 1024 ++ ++/* ++ * Writes to the device ++ */ ++ ++static ssize_t write(struct file *filep, const char *buffer, size_t len, loff_t *offset) ++{ ++ ++ int ret = len; ++ struct tegra_bpmp_message *kbuf = NULL; ++ void *txbuf = NULL; ++ void *rxbuf = NULL; ++ void *usertxbuf = NULL; ++ void *userrxbuf = NULL; ++ ++ if (len > 65535) { /* paranoia */ ++ deb_error("count %zu exceeds max # of bytes allowed, " ++ "aborting write\n", len); ++ goto out_nomem; ++ } ++ ++ deb_info("wants to write %zu bytes\n", len); ++ ++ ++ ret = -ENOMEM; ++ kbuf = kmalloc(len, GFP_KERNEL); ++ ++ ++ if (!kbuf) ++ goto out_nomem; ++ ++ memset(kbuf, 0, len); ++ ++ ret = -EFAULT; ++ ++ // Copy header ++ if (copy_from_user(kbuf, buffer, len)) { ++ deb_error("copy_from_user(1) failed\n"); ++ goto out_cfu; ++ } ++ ++ if(kbuf->tx.size > 0){ ++ txbuf = kmalloc(BUF_SIZE, GFP_KERNEL); ++ if (!txbuf) ++ goto out_nomem; ++ memset(txbuf, 0, BUF_SIZE); ++ if (copy_from_user(txbuf, kbuf->tx.data, kbuf->tx.size)) { ++ deb_error("copy_from_user(2) failed\n"); ++ goto out_cfu; ++ } ++ } ++ ++ rxbuf = kmalloc(BUF_SIZE, GFP_KERNEL); ++ if (!rxbuf) ++ goto out_nomem; ++ ++ memset(rxbuf, 0, BUF_SIZE); ++ if (copy_from_user(rxbuf, kbuf->rx.data, kbuf->rx.size)) { ++ deb_error("copy_from_user(3) failed\n"); ++ goto out_cfu; ++ } ++ ++ ++ usertxbuf = (void*)kbuf->tx.data; //save userspace buffers addresses ++ userrxbuf = kbuf->rx.data; ++ ++ ++ kbuf->tx.data = txbuf; //reassing to kernel space buffers ++ kbuf->rx.data = rxbuf; ++ ++ hexDump (DEVICE_NAME ": kbuf", kbuf, len); ++ hexDump (DEVICE_NAME ": txbuf", txbuf, kbuf->tx.size); ++ ++ if(!tegra_bpmp_host_device){ ++ deb_error("host device not initialised, can't do transfer!"); ++ return -EFAULT; ++ } ++ ++ if(!check_if_allowed(kbuf)){ ++ goto out_cfu; ++ } ++ ++ ret = tegra_bpmp_transfer(tegra_bpmp_host_device, (struct tegra_bpmp_message *)kbuf); ++ ++ ++ ++ if (copy_to_user((void *)usertxbuf, kbuf->tx.data, kbuf->tx.size)) { ++ deb_error("copy_to_user(2) failed\n"); ++ goto out_notok; ++ } ++ ++ if (copy_to_user((void *)userrxbuf, kbuf->rx.data, kbuf->rx.size)) { ++ deb_error("copy_to_user(3) failed\n"); ++ goto out_notok; ++ } ++ ++ kbuf->tx.data=usertxbuf; ++ kbuf->rx.data=userrxbuf; ++ ++ if (copy_to_user((void *)buffer, kbuf, len)) { ++ deb_error("copy_to_user(1) failed\n"); ++ goto out_notok; ++ } ++ ++ ++ ++ kfree(kbuf); ++ return len; ++out_notok: ++out_nomem: ++ deb_error("memory allocation failed"); ++out_cfu: ++ kfree(kbuf); ++ kfree(txbuf); ++ kfree(rxbuf); ++ return -EINVAL; ++ ++} ++ ++static const struct of_device_id bpmp_host_proxy_ids[] = { ++ { .compatible = "nvidia,bpmp-host-proxy" }, ++ { } ++}; ++ ++static struct platform_driver bpmp_host_proxy_driver = { ++ .driver = { ++ .name = "bpmp_host_proxy", ++ .of_match_table = bpmp_host_proxy_ids, ++ }, ++ .probe = bpmp_host_proxy_probe, ++ .remove = bpmp_host_proxy_remove, ++}; ++builtin_platform_driver(bpmp_host_proxy_driver); +\ No newline at end of file +diff --git a/drivers/bpmp-host-proxy/bpmp-host-proxy.h b/drivers/bpmp-host-proxy/bpmp-host-proxy.h +new file mode 100644 +index 000000000000..120bb9b046f5 +--- /dev/null ++++ b/drivers/bpmp-host-proxy/bpmp-host-proxy.h +@@ -0,0 +1,17 @@ ++#ifndef __BPMP_HOST_PROXY__H__ ++#define __BPMP_HOST_PROXY__H__ ++ ++#include ++ ++#define BPMP_HOST_MAX_CLOCKS_SIZE 256 ++#define BPMP_HOST_MAX_RESETS_SIZE 256 ++ ++struct bpmp_allowed_res { ++ int clocks_size; ++ uint32_t clock[BPMP_HOST_MAX_CLOCKS_SIZE]; ++ int resets_size; ++ uint32_t reset[BPMP_HOST_MAX_RESETS_SIZE]; ++ ++}; ++ ++#endif +\ No newline at end of file +-- +2.38.5 + diff --git a/modules/hardware/nvidia-jetson-orin/virtualization/common/bpmp-virt-common/patches/0005-bpmp-overlay.patch b/modules/hardware/nvidia-jetson-orin/virtualization/common/bpmp-virt-common/patches/0005-bpmp-overlay.patch new file mode 100644 index 000000000..0aba7588c --- /dev/null +++ b/modules/hardware/nvidia-jetson-orin/virtualization/common/bpmp-virt-common/patches/0005-bpmp-overlay.patch @@ -0,0 +1,18 @@ +diff --git a/kernel-int-overlays.txt b/kernel-int-overlays.txt +index 6a59a925233b..e099229d3706 100644 +--- a/kernel-int-overlays.txt ++++ b/kernel-int-overlays.txt +@@ -3,3 +3,4 @@ nvlink + nvgpu + nvgpu-next + nvidia-t239 ++bpmp-virt +diff --git a/kernel-overlays.txt b/kernel-overlays.txt +index 1a8cf218b2a9..aba8c9131b81 100644 +--- a/kernel-overlays.txt ++++ b/kernel-overlays.txt +@@ -1,3 +1,4 @@ + nvidia + nvlink + nvgpu ++bpmp-virt diff --git a/modules/hardware/nvidia-jetson-orin/virtualization/default.nix b/modules/hardware/nvidia-jetson-orin/virtualization/default.nix new file mode 100644 index 000000000..acf8d716f --- /dev/null +++ b/modules/hardware/nvidia-jetson-orin/virtualization/default.nix @@ -0,0 +1,9 @@ +# Copyright 2022-2023 TII (SSRC) and the Ghaf contributors +# SPDX-License-Identifier: Apache-2.0 +{ + imports = [ + ./common/bpmp-virt-common + ./host/bpmp-virt-host + ./host/uarta-host + ]; +} diff --git a/modules/hardware/nvidia-jetson-orin/virtualization/host/bpmp-virt-host/default.nix b/modules/hardware/nvidia-jetson-orin/virtualization/host/bpmp-virt-host/default.nix new file mode 100644 index 000000000..821483066 --- /dev/null +++ b/modules/hardware/nvidia-jetson-orin/virtualization/host/bpmp-virt-host/default.nix @@ -0,0 +1,51 @@ +# Copyright 2022-2023 TII (SSRC) and the Ghaf contributors +# SPDX-License-Identifier: Apache-2.0 +{ + lib, + pkgs, + config, + ... +}: let + cfg = config.ghaf.hardware.nvidia.virtualization.host.bpmp; +in { + options.ghaf.hardware.nvidia.virtualization.host.bpmp.enable = lib.mkOption { + type = lib.types.bool; + default = false; + description = '' + Enable virtualization host support for NVIDIA Orin + + This option is an implementation level detail and is toggled automatically + by modules that need it. Manually enabling this option is not recommended in + release builds. + ''; + }; + + config = lib.mkIf cfg.enable { + nixpkgs.overlays = [(import ./overlays/qemu)]; + + boot.kernelPatches = [ + { + name = "Bpmp virtualization host proxy device tree"; + patch = ./patches/0001-bpmp-host-proxy-dts.patch; + } + { + name = "Bpmp virtualization host uarta device tree"; + patch = ./patches/0002-bpmp-host-uarta-dts.patch; + } + { + name = "Bpmp virtualization host kernel configuration"; + patch = null; + extraStructuredConfig = with lib.kernel; { + VFIO_PLATFORM = yes; + TEGRA_BPMP_HOST_PROXY = yes; + }; + } + ]; + + # TODO: Consider are these really needed, maybe add only in debug builds? + environment.systemPackages = with pkgs; [ + qemu + dtc + ]; + }; +} diff --git a/modules/hardware/nvidia-jetson-orin/virtualization/host/bpmp-virt-host/overlays/qemu/default.nix b/modules/hardware/nvidia-jetson-orin/virtualization/host/bpmp-virt-host/overlays/qemu/default.nix new file mode 100644 index 000000000..938fc3fab --- /dev/null +++ b/modules/hardware/nvidia-jetson-orin/virtualization/host/bpmp-virt-host/overlays/qemu/default.nix @@ -0,0 +1,11 @@ +# Copyright 2022-2023 TII (SSRC) and the Ghaf contributors +# SPDX-License-Identifier: Apache-2.0 +(_final: prev: { + qemu = prev.qemu.overrideAttrs (_final: prev: { + patches = + prev.patches + ++ [ + ./patches/0001-qemu-v8.1.3_bpmp-virt.patch + ]; + }); +}) diff --git a/modules/hardware/nvidia-jetson-orin/virtualization/host/bpmp-virt-host/overlays/qemu/patches/0001-qemu-v8.1.3_bpmp-virt.patch b/modules/hardware/nvidia-jetson-orin/virtualization/host/bpmp-virt-host/overlays/qemu/patches/0001-qemu-v8.1.3_bpmp-virt.patch new file mode 100644 index 000000000..356e8c01a --- /dev/null +++ b/modules/hardware/nvidia-jetson-orin/virtualization/host/bpmp-virt-host/overlays/qemu/patches/0001-qemu-v8.1.3_bpmp-virt.patch @@ -0,0 +1,357 @@ +From e7d0fd8ca9f4334ee5d980217044e9c1c7cc21f4 Mon Sep 17 00:00:00 2001 +From: Mika Tammi +Date: Sun, 7 Jan 2024 00:24:33 +0200 +Subject: Rebased patch on top of QEMU 8.1.3 + +--- + README.md | 50 +++++++++++ + configure.sh | 1 + + hw/arm/Kconfig | 1 + + hw/arm/virt.c | 5 ++ + hw/misc/Kconfig | 3 + + hw/misc/meson.build | 1 + + hw/misc/nvidia_bpmp_guest.c | 171 ++++++++++++++++++++++++++++++++++++ + hw/misc/nvidia_bpmp_guest.h | 9 ++ + include/hw/arm/virt.h | 1 + + 9 files changed, 242 insertions(+) + create mode 100644 README.md + create mode 100755 configure.sh + create mode 100644 hw/misc/nvidia_bpmp_guest.c + create mode 100644 hw/misc/nvidia_bpmp_guest.h + +diff --git a/README.md b/README.md +new file mode 100644 +index 0000000000..56b9e765ec +--- /dev/null ++++ b/README.md +@@ -0,0 +1,50 @@ ++ ++# Short intstructions: ++ ++ ++1. git clone https://gitlab.com/qemu-project/qemu.git ++2 cd qemu ++3 git checkout -b v8.1.3 ++4 ./configure --target-list=aarm64-softmmu ++5 make -j12 ++ ++ ++ ++Device memory map: ++ ++0x090c0000 + /* Base address, size 0x01000 */ ++ ++ 0x0000 \ Tx buffer ++ 0x01FF / ++ 0x0200 \ Rx buffer ++ 0x03FF / ++ 0x0400 -- Tx size ++ 0x0408 -- Rx size ++ 0x0410 -- Ret ++ 0x0500 -- mrq ++ ++ ++ ++ Data should be aligned to 64bit paragraph. ++ ++ Protocol is: ++ 1. Write data buffers to 0x0000-0x01FF and 0x0200-0x03FF ++ 2. Write buffer sizes to 0x0400 (Tx) and 0x0408 (Rx) ++ 2. Start operation by writing mrq opcode to address 0x0500 ++ 3. Read ret code from 0x0410 and response data from the buffers ++ ++ ++For reading and writing busybox may be used as: ++ ++ busybox devmem 0x090c0000 ++ ++and so on ++ ++For instance, to reset the UARTA (with ID 0x64) you can sed the next ++command: ++ ++ busybox devmem 0x090c0000 64 0x0000006400000001 ++ busybox devmem 0x090c0400 8 0x08 ++ busybox devmem 0x090c0500 8 0x14 ++ ++``` +diff --git a/configure.sh b/configure.sh +new file mode 100755 +index 0000000000..cb82ca69a7 +--- /dev/null ++++ b/configure.sh +@@ -0,0 +1 @@ ++./configure --target-list=aarch64-softmmu --enable-sdl --enable-gtk -enable-vhost-kernel --enable-vhost-net --enable-vhost-user --enable-vhost-user-blk-server --enable-vfio-user-server --enable-vhost-crypto --enable-vhost-vdpa --enable-virglrenderer --enable-virtfs --enable-virtfs-proxy-helper +diff --git a/hw/arm/Kconfig b/hw/arm/Kconfig +index 7e68348440..3653c77f9e 100644 +--- a/hw/arm/Kconfig ++++ b/hw/arm/Kconfig +@@ -32,6 +32,7 @@ config ARM_VIRT + select VIRTIO_MEM_SUPPORTED + select ACPI_CXL + select ACPI_HMAT ++ select NVIDIA_BPMP_GUEST + + config CHEETAH + bool +diff --git a/hw/arm/virt.c b/hw/arm/virt.c +index 7d9dbc2663..5ee4ec077a 100644 +--- a/hw/arm/virt.c ++++ b/hw/arm/virt.c +@@ -79,6 +79,7 @@ + #include "hw/virtio/virtio-md-pci.h" + #include "hw/virtio/virtio-iommu.h" + #include "hw/char/pl011.h" ++#include "hw/misc/nvidia_bpmp_guest.h" + #include "qemu/guest-random.h" + + #define DEFINE_VIRT_MACHINE_LATEST(major, minor, latest) \ +@@ -155,6 +156,7 @@ static const MemMapEntry base_memmap[] = { + [VIRT_NVDIMM_ACPI] = { 0x09090000, NVDIMM_ACPI_IO_LEN}, + [VIRT_PVTIME] = { 0x090a0000, 0x00010000 }, + [VIRT_SECURE_GPIO] = { 0x090b0000, 0x00001000 }, ++ [VIRT_NVIDIA_BPMP_GUEST] = { 0x090c0000, 0x00001000 }, + [VIRT_MMIO] = { 0x0a000000, 0x00000200 }, + /* ...repeating for a total of NUM_VIRTIO_TRANSPORTS, each of that size */ + [VIRT_PLATFORM_BUS] = { 0x0c000000, 0x02000000 }, +@@ -1046,6 +1048,9 @@ static void create_virtio_devices(const VirtMachineState *vms) + hwaddr size = vms->memmap[VIRT_MMIO].size; + MachineState *ms = MACHINE(vms); + ++ /* Create NVIDIA BPMP guest passthru device, possibly need update ftd - WIP */ ++ nvidia_bpmp_guest_create(vms->memmap[VIRT_NVIDIA_BPMP_GUEST].base); ++ + /* We create the transports in forwards order. Since qbus_realize() + * prepends (not appends) new child buses, the incrementing loop below will + * create a list of virtio-mmio buses with decreasing base addresses. +diff --git a/hw/misc/Kconfig b/hw/misc/Kconfig +index 6996d265e4..30b4c34e69 100644 +--- a/hw/misc/Kconfig ++++ b/hw/misc/Kconfig +@@ -11,6 +11,9 @@ config ARMSSE_MHU + config ARMSSE_CPU_PWRCTRL + bool + ++config NVIDIA_BPMP_GUEST ++ bool ++ + config ISA_DEBUG + bool + depends on ISA_BUS +diff --git a/hw/misc/meson.build b/hw/misc/meson.build +index 892f8b91c5..f623cf2c9c 100644 +--- a/hw/misc/meson.build ++++ b/hw/misc/meson.build +@@ -1,4 +1,5 @@ + system_ss.add(when: 'CONFIG_APPLESMC', if_true: files('applesmc.c')) ++system_ss.add(when: 'CONFIG_NVIDIA_BPMP_GUEST', if_true: files('nvidia_bpmp_guest.c')) + system_ss.add(when: 'CONFIG_EDU', if_true: files('edu.c')) + system_ss.add(when: 'CONFIG_FW_CFG_DMA', if_true: files('vmcoreinfo.c')) + system_ss.add(when: 'CONFIG_ISA_DEBUG', if_true: files('debugexit.c')) +diff --git a/hw/misc/nvidia_bpmp_guest.c b/hw/misc/nvidia_bpmp_guest.c +new file mode 100644 +index 0000000000..3facee6d00 +--- /dev/null ++++ b/hw/misc/nvidia_bpmp_guest.c +@@ -0,0 +1,171 @@ ++#include "qemu/osdep.h" ++#include "qemu/log.h" ++#include "qapi/error.h" /* provides error_fatal() handler */ ++#include "hw/sysbus.h" /* provides all sysbus registering func */ ++#include "hw/misc/nvidia_bpmp_guest.h" ++ ++#define TYPE_NVIDIA_BPMP_GUEST "nvidia_bpmp_guest" ++typedef struct NvidiaBpmpGuestState NvidiaBpmpGuestState; ++DECLARE_INSTANCE_CHECKER(NvidiaBpmpGuestState, NVIDIA_BPMP_GUEST, TYPE_NVIDIA_BPMP_GUEST) ++ ++#define TX_BUF 0x0000 ++#define RX_BUF 0x0200 ++#define TX_SIZ 0x0400 ++#define RX_SIZ 0x0408 ++#define RET_COD 0x0410 ++#define MRQ 0x0500 ++ ++#define MEM_SIZE 0x600 ++#define HOST_DEVICE_PATH "/dev/bpmp-host" ++#define MESSAGE_SIZE 0x0200 ++ ++// qemu_log_mask(LOG_UNIMP, "%s: \n", __func__ ); ++ ++struct NvidiaBpmpGuestState ++{ ++ SysBusDevice parent_obj; ++ MemoryRegion iomem; ++ int host_device_fd; ++ uint8_t mem[MEM_SIZE]; ++}; ++ ++// Device memory map: ++ ++// 0x090c0000 + /* Base address, size 0x01000 */ ++ ++// 0x0000 \ Tx buffer ++// 0x01FF / ++// 0x0200 \ Rx buffer ++// 0x03FF / ++// 0x0400 -- Tx size ++// 0x0408 -- Rx size ++// 0x0410 -- Ret ++// 0x0500 -- mrq ++ ++ ++ ++// Data should be aligned to 64bit paragraph. ++ ++// Protocol is: ++// 1. Write data buffers to 0x0000-0x01FF and 0x0200-0x03FF ++// 2. Write buffer sizes to 0x0400 (Tx) and 0x0408 (Rx) ++// 2. Start operation by writing mrq opcode to address 0x0500 ++// 3. Read ret code from 0x0410 and response data from the buffers ++ ++static uint64_t nvidia_bpmp_guest_read(void *opaque, hwaddr addr, unsigned int size) ++{ ++ NvidiaBpmpGuestState *s = opaque; ++ ++ if (addr >= MEM_SIZE) ++ return 0xDEADBEEF; ++ ++ // Cast buffer location as uint64_t ++ return *(uint64_t*)&s->mem[addr]; ++} ++ ++static void nvidia_bpmp_guest_write(void *opaque, hwaddr addr, uint64_t data, unsigned int size) ++{ ++ NvidiaBpmpGuestState *s = opaque; ++ int ret; ++ ++ struct ++ { ++ unsigned int mrq; ++ struct ++ { ++ void *data; ++ size_t size; ++ } tx; ++ struct ++ { ++ void *data; ++ size_t size; ++ int ret; ++ } rx; ++ } messg; ++ ++ memset(&messg, 0, sizeof(messg)); ++ ++ if (addr >= MEM_SIZE){ ++ qemu_log_mask(LOG_UNIMP, "qemu: Error addr >= MEM_SIZE in 0x%lX data: 0x%lX\n", addr, data); ++ return; ++ } ++ ++ switch (addr) ++ { ++ case MRQ: ++ // set up the structure ++ messg.mrq = data; ++ messg.tx.data = &s->mem[TX_BUF]; ++ memcpy(&messg.tx.size, &s->mem[TX_SIZ], sizeof(messg.tx.size)); ++ messg.rx.data = &s->mem[RX_BUF]; ++ memcpy(&messg.rx.size, &s->mem[RX_SIZ], sizeof(messg.rx.size)); ++ ++ ret = write(s->host_device_fd, &messg, sizeof(messg)); // Send the data to the host module ++ if (ret < 0) ++ { ++ qemu_log_mask(LOG_UNIMP, "%s: Failed to write the host device..\n", __func__); ++ return; ++ } ++ ++ memcpy(&s->mem[RET_COD], &messg.rx.ret, sizeof(messg.rx.ret)); ++ memcpy(&s->mem[RX_SIZ], &messg.rx.size, sizeof(messg.rx.size)); ++ ++ break; ++ ++ default: ++ ++ memcpy(&s->mem[addr], &data, size); ++ } ++ ++ return; ++} ++ ++static const MemoryRegionOps nvidia_bpmp_guest_ops = { ++ .read = nvidia_bpmp_guest_read, ++ .write = nvidia_bpmp_guest_write, ++ .endianness = DEVICE_NATIVE_ENDIAN, ++}; ++ ++static void nvidia_bpmp_guest_instance_init(Object *obj) ++{ ++ NvidiaBpmpGuestState *s = NVIDIA_BPMP_GUEST(obj); ++ ++ /* allocate memory map region */ ++ memory_region_init_io(&s->iomem, obj, &nvidia_bpmp_guest_ops, s, TYPE_NVIDIA_BPMP_GUEST, MEM_SIZE); ++ sysbus_init_mmio(SYS_BUS_DEVICE(obj), &s->iomem); ++ ++ s->host_device_fd = open(HOST_DEVICE_PATH, O_RDWR); // Open the device with read/write access ++ ++ if (s->host_device_fd < 0) ++ { ++ qemu_log_mask(LOG_UNIMP, "%s: Failed to open the host device..\n", __func__); ++ return; ++ } ++} ++ ++/* create a new type to define the info related to our device */ ++static const TypeInfo nvidia_bpmp_guest_info = { ++ .name = TYPE_NVIDIA_BPMP_GUEST, ++ .parent = TYPE_SYS_BUS_DEVICE, ++ .instance_size = sizeof(NvidiaBpmpGuestState), ++ .instance_init = nvidia_bpmp_guest_instance_init, ++}; ++ ++static void nvidia_bpmp_guest_register_types(void) ++{ ++ type_register_static(&nvidia_bpmp_guest_info); ++} ++ ++type_init(nvidia_bpmp_guest_register_types) ++ ++ /* ++ * Create the Nvidia BPMP guest device. ++ */ ++ DeviceState *nvidia_bpmp_guest_create(hwaddr addr) ++{ ++ DeviceState *dev = qdev_new(TYPE_NVIDIA_BPMP_GUEST); ++ sysbus_realize_and_unref(SYS_BUS_DEVICE(dev), &error_fatal); ++ sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, addr); ++ return dev; ++} +diff --git a/hw/misc/nvidia_bpmp_guest.h b/hw/misc/nvidia_bpmp_guest.h +new file mode 100644 +index 0000000000..dd4b4221d3 +--- /dev/null ++++ b/hw/misc/nvidia_bpmp_guest.h +@@ -0,0 +1,9 @@ ++#ifndef HW_NVIDIA_BPMP_GUEST_H ++#define HW_NVIDIA_BPMP_GUEST_H ++ ++#include "qom/object.h" ++ ++DeviceState *nvidia_bpmp_guest_create(hwaddr); ++ ++ ++#endif +diff --git a/include/hw/arm/virt.h b/include/hw/arm/virt.h +index e1ddbea96b..fce00ae7e9 100644 +--- a/include/hw/arm/virt.h ++++ b/include/hw/arm/virt.h +@@ -77,6 +77,7 @@ enum { + VIRT_PCIE_MMIO, + VIRT_PCIE_PIO, + VIRT_PCIE_ECAM, ++ VIRT_NVIDIA_BPMP_GUEST, + VIRT_PLATFORM_BUS, + VIRT_GPIO, + VIRT_SECURE_UART, +-- +2.42.0 + diff --git a/modules/hardware/nvidia-jetson-orin/virtualization/host/bpmp-virt-host/patches/0001-bpmp-host-proxy-dts.patch b/modules/hardware/nvidia-jetson-orin/virtualization/host/bpmp-virt-host/patches/0001-bpmp-host-proxy-dts.patch new file mode 100644 index 000000000..25821b77c --- /dev/null +++ b/modules/hardware/nvidia-jetson-orin/virtualization/host/bpmp-virt-host/patches/0001-bpmp-host-proxy-dts.patch @@ -0,0 +1,28 @@ +From f439277b252808fa20182a14a49f647eb22c78ac Mon Sep 17 00:00:00 2001 +From: Julius Koskela +Date: Mon, 9 Oct 2023 01:43:18 +0300 +Subject: [PATCH] Add bpmp host proxy device tree node + +Signed-off-by: Julius Koskela +--- + .../t23x/kernel-dts/tegra234-soc/tegra234-soc-base.dtsi | 7 +++++++ + 1 file changed, 7 insertions(+) + +diff --git a/nvidia/soc/t23x/kernel-dts/tegra234-soc/tegra234-soc-base.dtsi b/nvidia/soc/t23x/kernel-dts/tegra234-soc/tegra234-soc-base.dtsi +index 1cf75f2773ad..24c32b0ebb1c 100644 +--- a/nvidia/soc/t23x/kernel-dts/tegra234-soc/tegra234-soc-base.dtsi ++++ b/nvidia/soc/t23x/kernel-dts/tegra234-soc/tegra234-soc-base.dtsi +@@ -1128,4 +1128,11 @@ tegra_cec: tegra_cec { + reset-names = "cec"; + status = "disabled"; + }; ++ ++ bpmp_host_proxy: bpmp_host_proxy { ++ compatible = "nvidia,bpmp-host-proxy"; ++ allowed-clocks = <155 102>; ++ allowed-resets = <100>; ++ status = "okay"; ++ }; + }; +-- +2.42.0 diff --git a/modules/hardware/nvidia-jetson-orin/virtualization/host/bpmp-virt-host/patches/0002-bpmp-host-uarta-dts.patch b/modules/hardware/nvidia-jetson-orin/virtualization/host/bpmp-virt-host/patches/0002-bpmp-host-uarta-dts.patch new file mode 100644 index 000000000..2be2c43be --- /dev/null +++ b/modules/hardware/nvidia-jetson-orin/virtualization/host/bpmp-virt-host/patches/0002-bpmp-host-uarta-dts.patch @@ -0,0 +1,46 @@ +diff --git a/nvidia/soc/t23x/kernel-dts/tegra234-soc/tegra234-soc-uart.dtsi b/nvidia/soc/t23x/kernel-dts/tegra234-soc/tegra234-soc-uart.dtsi +index 503cb275dcc1..c708ad3f5048 100644 +--- a/nvidia/soc/t23x/kernel-dts/tegra234-soc/tegra234-soc-uart.dtsi ++++ b/nvidia/soc/t23x/kernel-dts/tegra234-soc/tegra234-soc-uart.dtsi +@@ -29,23 +29,24 @@ aliases { + serial7 = &uarth; + }; + +- uarta: serial@3100000 { +- compatible = "nvidia,tegra194-hsuart"; +- iommus = <&smmu_niso0 TEGRA_SID_NISO0_GPCDMA_0>; +- dma-coherent; +- reg = <0x0 0x03100000 0x0 0x10000>; +- reg-shift = <2>; +- interrupts = <0 TEGRA234_IRQ_UARTA 0x04>; +- nvidia,memory-clients = <14>; +- dmas = <&gpcdma 8>, <&gpcdma 8>; +- dma-names = "rx", "tx"; +- clocks = <&bpmp_clks TEGRA234_CLK_UARTA>, +- <&bpmp_clks TEGRA234_CLK_PLLP_OUT0>; +- clock-names = "serial", "parent"; +- resets = <&bpmp_resets TEGRA234_RESET_UARTA>; +- reset-names = "serial"; +- status = "disabled"; +- }; ++ uarta: serial@3100000 { ++ compatible = "nvidia,tegra194-dummy"; ++ //iommus = <&smmu_niso0 TEGRA_SID_NISO0_GPCDMA_0>; ++ iommus = <&smmu_niso0 TEGRA_SID_NISO1_SMMU_TEST>; ++ dma-coherent; ++ reg = <0x0 0x03100000 0x0 0x10000>; ++ reg-shift = <2>; ++ interrupts = <0 TEGRA234_IRQ_UARTA 0x04>; ++ nvidia,memory-clients = <14>; ++ dmas = <&gpcdma 8>, <&gpcdma 8>; ++ dma-names = "rx", "tx"; ++ clocks = <&bpmp_clks TEGRA234_CLK_UARTA>, ++ <&bpmp_clks TEGRA234_CLK_PLLP_OUT0>; ++ clock-names = "serial", "parent"; ++ resets = <&bpmp_resets TEGRA234_RESET_UARTA>; ++ reset-names = "serial"; ++ status = "okay"; ++ }; + + uartb: serial@3110000 { + compatible = "nvidia,tegra194-hsuart"; diff --git a/modules/hardware/nvidia-jetson-orin/virtualization/host/uarta-host/default.nix b/modules/hardware/nvidia-jetson-orin/virtualization/host/uarta-host/default.nix new file mode 100644 index 000000000..2bbf06656 --- /dev/null +++ b/modules/hardware/nvidia-jetson-orin/virtualization/host/uarta-host/default.nix @@ -0,0 +1,46 @@ +# Copyright 2022-2023 TII (SSRC) and the Ghaf contributors +# SPDX-License-Identifier: Apache-2.0 +{ + lib, + pkgs, + config, + ... +}: let + cfg = config.ghaf.hardware.nvidia.passthroughs.host.uarta; +in { + options.ghaf.hardware.nvidia.passthroughs.host.uarta.enable = lib.mkOption { + type = lib.types.bool; + default = false; + description = "Enable UARTA passthrough on Nvidia Orin host"; + }; + + config = lib.mkIf cfg.enable { + ghaf.hardware.nvidia.virtualization.enable = true; + ghaf.hardware.nvidia.virtualization.host.bpmp.enable = true; + + systemd.services.enableVfioPlatform = { + description = "Enable the vfio-platform driver for UARTA"; + wantedBy = ["bindSerial3100000.service"]; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = "yes"; + ExecStart = '' + ${pkgs.bash}/bin/bash -c "echo vfio-platform > /sys/bus/platform/devices/3100000.serial/driver_override" + ''; + }; + }; + + systemd.services.bindSerial3100000 = { + description = "Bind UARTA to the vfio-platform driver"; + wantedBy = ["multi-user.target"]; + after = ["enableVfioPlatform.service"]; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = "yes"; + ExecStart = '' + ${pkgs.bash}/bin/bash -c "echo 3100000.serial > /sys/bus/platform/drivers/vfio-platform/bind" + ''; + }; + }; + }; +} diff --git a/targets/nvidia-jetson-orin/default.nix b/targets/nvidia-jetson-orin/default.nix index 92d498920..6cd20effc 100644 --- a/targets/nvidia-jetson-orin/default.nix +++ b/targets/nvidia-jetson-orin/default.nix @@ -57,6 +57,12 @@ nx.enableNetvmEthernetPCIPassthrough = som == "nx"; }; + hardware.nvidia = { + virtualization.enable = false; + virtualization.host.bpmp.enable = false; + passthroughs.host.uarta.enable = false; + }; + virtualization.microvm-host.enable = true; host.networking.enable = true;