Use Waypipe with VSOCK support

Signed-off-by: Yuri Nesterov <[email protected]>
tiiuae · Sep 29, 2023 · aaeafbc · aaeafbc
1 parent 611be21
commit aaeafbc
Show file tree

Hide file tree

Showing 5 changed files with 154 additions and 11 deletions.
diff --git a/modules/virtualization/microvm/appvm.nix b/modules/virtualization/microvm/appvm.nix
@@ -10,8 +10,15 @@
   cfg = config.ghaf.virtualization.microvm.appvm;
   waypipe-ssh = pkgs.callPackage ../../../user-apps/waypipe-ssh {};
 
-  makeVm = {vm}: let
+  makeVm = {
+    vm,
+    index,
+  }: let
     hostname = "vm-" + vm.name;
+    cid =
+      if vm.cid > 0
+      then vm.cid
+      else cfg.vsockBaseCID + index;
     appvmConfiguration = {
       imports = [
         ({
@@ -73,6 +80,12 @@
                 mac = vm.macAddress;
               }
             ];
+            qemu.extraArgs = [
+              "-M"
+              "q35,accel=kvm:tcg,mem-merge=on,sata=off"
+              "-device"
+              "vhost-vsock-pci,guest-cid=${toString cid}"
+            ];
           };
 
           networking.nat = {
@@ -167,6 +180,14 @@ in {
               '';
               default = [];
             };
+            cid = mkOption {
+              description = ''
+                VSOCK context identifier (CID) for the AppVM
+                Default value 0 means auto-assign using vsockBaseCID and AppVM index
+              '';
+              type = int;
+              default = 0;
+            };
           };
         });
         default = [];
@@ -179,12 +200,23 @@ in {
       '';
       default = [];
     };
+
+    # Base VSOCK CID which is used for auto assigning CIDs for all AppVMs
+    # For example, when it's set to 100, AppVMs will get 100, 101, 102, etc.
+    # It is also possible to override the auto assinged CID using the vms.cid option
+    vsockBaseCID = lib.mkOption {
+      type = lib.types.int;
+      default = 100;
+      description = ''
+        Context Identifier (CID) of the AppVM VSOCK
+      '';
+    };
   };
 
   config = lib.mkIf cfg.enable {
     microvm.vms = (
       let
-        vms = map (vm: {"appvm-${vm.name}" = makeVm {inherit vm;};}) cfg.vms;
+        vms = lib.imap0 (index: vm: {"appvm-${vm.name}" = makeVm {inherit index vm;};}) cfg.vms;
       in
         lib.foldr lib.recursiveUpdate {} vms
     );

diff --git a/modules/virtualization/microvm/guivm.nix b/modules/virtualization/microvm/guivm.nix
@@ -71,6 +71,11 @@
               mac = "02:00:00:02:02:02";
             }
           ];
+
+          qemu.extraArgs = [
+            "-device"
+            "vhost-vsock-pci,guest-cid=${toString cfg.vsockCID}"
+          ];
         };
 
         networking.nat = {
@@ -103,10 +108,35 @@
         };
 
         imports = import ../../module-list.nix;
+
+        # Waypipe service runs in the GUIVM and listens for incoming connections from AppVMs
+        systemd.user.services.waypipe = {
+          enable = true;
+          description = "waypipe";
+          after = ["weston.service"];
+          serviceConfig = {
+            Type = "simple";
+            Environment = [
+              "WAYLAND_DISPLAY=\"wayland-1\""
+              "DISPLAY=\":0\""
+              "XDG_SESSION_TYPE=wayland"
+              "QT_QPA_PLATFORM=\"wayland\"" # Qt Applications
+              "GDK_BACKEND=\"wayland\"" # GTK Applications
+              "XDG_SESSION_TYPE=\"wayland\"" # Electron Applications
+              "SDL_VIDEODRIVER=\"wayland\""
+              "CLUTTER_BACKEND=\"wayland\""
+            ];
+            ExecStart = "${pkgs.waypipe}/bin/waypipe --vsock -s ${toString cfg.waypipePort} client";
+            Restart = "always";
+            RestartSec = "1";
+          };
+          wantedBy = ["ghaf-session.target"];
+        };
       })
     ];
   };
   cfg = config.ghaf.virtualization.microvm.guivm;
+  vsockproxy = pkgs.callPackage ../../../user-apps/vsockproxy {};
 in {
   options.ghaf.virtualization.microvm.guivm = {
     enable = lib.mkEnableOption "GUIVM";
@@ -118,6 +148,28 @@ in {
       '';
       default = [];
     };
+
+    # GUIVM uses a VSOCK which requires a CID
+    # There are several special addresses:
+    # VMADDR_CID_HYPERVISOR (0) is reserved for services built into the hypervisor
+    # VMADDR_CID_LOCAL (1) is the well-known address for local communication (loopback)
+    # VMADDR_CID_HOST (2) is the well-known address of the host
+    # CID 3 is the lowest available number for guest virtual machines
+    vsockCID = lib.mkOption {
+      type = lib.types.int;
+      default = 3;
+      description = ''
+        Context Identifier (CID) of the GUIVM VSOCK
+      '';
+    };
+
+    waypipePort = lib.mkOption {
+      type = lib.types.int;
+      default = 1100;
+      description = ''
+        Waypipe port number to listen for incoming connections from AppVMs
+      '';
+    };
   };
 
   config = lib.mkIf cfg.enable {
@@ -132,5 +184,21 @@ in {
         };
       specialArgs = {inherit lib;};
     };
+
+    # Waypipe in GUIVM needs to communicate with AppVMs over VSOCK
+    # However, VSOCK does not support direct guest to guest communication
+    # The vsockproxy app is used on host as a bridge between AppVMs and GUIVM
+    # It listens for incoming connections from AppVMs and forwards data to GUIVM
+    systemd.services.vsockproxy = {
+      enable = true;
+      description = "vsockproxy";
+      unitConfig = {
+        Type = "simple";
+      };
+      serviceConfig = {
+        ExecStart = "${vsockproxy}/bin/vsockproxy ${toString cfg.waypipePort} ${toString cfg.vsockCID} ${toString cfg.waypipePort}";
+      };
+      wantedBy = ["multi-user.target"];
+    };
   };
 }
diff --git a/overlays/custom-packages.nix b/overlays/custom-packages.nix
@@ -16,7 +16,11 @@
 # nativeBuildInputs and buildInputs where possible.
 # It makes things clear and robust.
 #
-{lib, ...}: {
+{
+  lib,
+  pkgs,
+  ...
+}: {
   nixpkgs.overlays = [
     (final: prev: {
       gala-app = final.callPackage ../user-apps/gala {};
@@ -81,6 +85,16 @@
       qemu_kvm = prev.qemu_kvm.overrideAttrs (_final: prev: {
         patches = prev.patches ++ [./acpi-devices-passthrough.patch];
       });
+      # Waypipe with vsock
+      waypipe = prev.waypipe.overrideAttrs (prevAttrs: {
+        src = pkgs.fetchFromGitLab {
+          domain = "gitlab.freedesktop.org";
+          owner = "nesterov";
+          repo = "waypipe";
+          rev = "2f1ab6a8efd2c1ad0dbcc9f8482b10861743e9c3";
+          sha256 = "sha256-P4y8p4R28j4zp0OX2GspsBKqWvCHqg+nF153LIrRYs8=";
+        };
+      });
     })
   ];
 }
diff --git a/targets/lenovo-x1-carbon.nix b/targets/lenovo-x1-carbon.nix
@@ -32,6 +32,7 @@
         };
       }
     ];
+    guivmConfig = hostConfiguration.config.ghaf.virtualization.microvm.guivm;
     guivmExtraModules = [
       {
         # Early KMS needed for GNOME to work inside GuiVM
@@ -68,17 +69,17 @@
       ({pkgs, ...}: {
         ghaf.graphics.weston.launchers = [
           {
-            path = "${pkgs.waypipe}/bin/waypipe ssh -i ${pkgs.waypipe-ssh}/keys/waypipe-ssh -o StrictHostKeyChecking=no 192.168.101.5 chromium --enable-features=UseOzonePlatform --ozone-platform=wayland";
+            path = "${pkgs.openssh}/bin/ssh -i ${pkgs.waypipe-ssh}/keys/waypipe-ssh -o StrictHostKeyChecking=no 192.168.101.5 ${pkgs.waypipe}/bin/waypipe --vsock -s ${toString guivmConfig.waypipePort} server chromium --enable-features=UseOzonePlatform --ozone-platform=wayland";
             icon = "${pkgs.weston}/share/weston/icon_editor.png";
           }
 
           {
-            path = "${pkgs.waypipe}/bin/waypipe ssh -i ${pkgs.waypipe-ssh}/keys/waypipe-ssh -o StrictHostKeyChecking=no 192.168.101.6 gala --enable-features=UseOzonePlatform --ozone-platform=wayland";
+            path = "${pkgs.openssh}/bin/ssh -i ${pkgs.waypipe-ssh}/keys/waypipe-ssh -o StrictHostKeyChecking=no 192.168.101.6 ${pkgs.waypipe}/bin/waypipe --vsock -s ${toString guivmConfig.waypipePort} server gala --enable-features=UseOzonePlatform --ozone-platform=wayland";
             icon = "${pkgs.weston}/share/weston/icon_editor.png";
           }
 
           {
-            path = "${pkgs.waypipe}/bin/waypipe ssh -i ${pkgs.waypipe-ssh}/keys/waypipe-ssh -o StrictHostKeyChecking=no 192.168.101.7 zathura";
+            path = "${pkgs.openssh}/bin/ssh -i ${pkgs.waypipe-ssh}/keys/waypipe-ssh -o StrictHostKeyChecking=no 192.168.101.7 ${pkgs.waypipe}/bin/waypipe --vsock -s ${toString guivmConfig.waypipePort} server zathura";
             icon = "${pkgs.weston}/share/weston/icon_editor.png";
           }
         ];
@@ -157,11 +158,6 @@
                         nixpkgs.config.pulseaudio = true;
 
                         microvm.qemu.extraArgs = [
-                          # APPVMs use microvm qemu machine which has pcie
-                          # disabled by default, and it also causes other
-                          # problems.
-                          "-M"
-                          "q35,accel=kvm:tcg,mem-merge=on,sata=off"
                           # Lenovo X1 integrated usb webcam
                           "-device"
                           "qemu-xhci"

diff --git a/user-apps/vsockproxy/default.nix b/user-apps/vsockproxy/default.nix
@@ -0,0 +1,33 @@
+# Copyright 2022-2023 TII (SSRC) and the Ghaf contributors
+# SPDX-License-Identifier: Apache-2.0
+{
+  stdenv,
+  pkgs,
+  lib,
+  ...
+}:
+stdenv.mkDerivation {
+  name = "vsockproxy";
+
+  nativeBuildInputs = with pkgs; [meson ninja];
+
+  src = pkgs.fetchFromGitHub {
+    owner = "tiiuae";
+    repo = "vsockproxy";
+    rev = "aad625f9a27ce4c68d9996c65ece8477ace37534";
+    sha256 = "sha256-3WgpDlF8oIdlgwkvl7TPR6WAh+qk0mowzuYiPY0rwaU=";
+  };
+
+  installPhase = ''
+    mkdir -p $out/bin
+    install ./vsockproxy $out/bin/vsockproxy
+  '';
+
+  meta = with lib; {
+    description = "vsockproxy";
+    platforms = [
+      "x86_64-linux"
+      "aarch64-linux"
+    ];
+  };
+}