feat: FAST looking glass! TONS of time on this jesus christ

nixos/default.nix

@@ -26,6 +26,8 @@
       ./user-system-config.nix
 
       ./vm.nix
+      (import ./kvmfr.nix { std = inputs.nix-std.lib; lib = lib; pkgs = pkgs; config = config; })
+      ./libvirtd.nix
     ];
 
   vfio = {
@@ -34,6 +36,36 @@
     applyACSpatch = true;
   };
 
+  virtualisation.kvmfr = {
+    enable = true;
+
+    devices = [
+      {
+        size = 128;
+
+        permissions = {
+          user = "evar";
+          group = "qemu-libvirtd";
+          mode = "0660";
+        };
+      }
+    ];
+  };
+
+  # # https://gist.github.com/j-brn/716a03822d256bc5bf5d77b951c7915c
+  # virtualisation.kvmfr = {
+  #   enable = true;
+
+  #   shm = {
+  #     enable = true;
+
+  #     size = 64;
+  #     user = "evar";
+  #     group = "libvirtd";
+  #     mode = "0660";
+  #   };
+  # };
+
   # Allows referring to this flake by the shorthand `nixos-config`, which lets you do e.g.
   # nix repl nixos-config 
   nix.registry.nixos-config.to = {

nixos/kvmfr.nix

@@ -0,0 +1,140 @@
+{ std, lib, pkgs, config, ... }:
+
+with lib;
+
+let
+  cfg = config.virtualisation.kvmfr;
+
+  sizeFromResolution = resolution:
+    let
+      ceilToPowerOf2 = n:
+        std.num.pow 2 (std.num.bits.bitSize - std.num.bits.countLeadingZeros n);
+      pixelSize = if resolution.pixelFormat == "rgb24" then 3 else 4;
+      bytes = resolution.width * resolution.height * pixelSize * 2;
+    in ceilToPowerOf2 (bytes / 1024 / 1024 + 10);
+
+  deviceSizes = map (device: device.size) cfg.devices;
+
+  devices =
+    imap0 (index: _deviceConfig: "/dev/kvmfr${toString index}") cfg.devices;
+
+  udevPackage = pkgs.writeTextDir "/lib/udev/rules.d/99-kvmfr.rules"
+    (concatStringsSep "\n" (imap0 (index: deviceConfig: ''
+      SUBSYSTEM=="kvmfr", KERNEL=="kvmfr${
+        toString index
+      }", OWNER="${deviceConfig.permissions.user}", GROUP="${deviceConfig.permissions.group}", MODE="${deviceConfig.permissions.mode}", TAG+="systemd"
+    '') cfg.devices));
+
+  apparmorAbstraction =
+    concatStringsSep "\n" (map (device: "${device} rw") devices);
+
+  permissionsType = types.submodule {
+    options = {
+      user = mkOption {
+        type = types.str;
+        default = "root";
+        description = mdDoc "Owner of the shared memory device.";
+      };
+      group = mkOption {
+        type = types.str;
+        default = "root";
+        description = mdDoc "Group of the shared memory device.";
+      };
+      mode = mkOption {
+        type = types.str;
+        default = "0600";
+        description = mdDoc "Mode of the shared memory device.";
+      };
+    };
+  };
+
+  resolutionType = types.submodule {
+    options = {
+      width = mkOption {
+        type = types.number;
+        description = mdDoc
+          "Maximum horizontal video size that should be supported by this device.";
+      };
+
+      height = mkOption {
+        type = types.number;
+        description = mdDoc
+          "Maximum vertical video size that should be supported by this device.";
+      };
+
+      pixelFormat = mkOption {
+        type = types.enum [ "rgba32" "rgb24" ];
+        description = mdDoc "Pixel format to use.";
+        default = "rgba32";
+      };
+    };
+  };
+
+  deviceType = (types.submodule ({ config, options, ... }: {
+    options = {
+      resolution = mkOption {
+        type = types.nullOr resolutionType;
+        default = null;
+        description = mdDoc ''
+          Automatically calculate the minimum device size for a specific resolution. Overrides `size` if set.
+        '';
+      };
+
+      size = mkOption {
+        type = types.number;
+        description = mdDoc ''
+          Size for the kvmfr device in megabytes.
+        '';
+      };
+
+      permissions = mkOption {
+        type = permissionsType;
+        default = { };
+        description = mdDoc "Permissions of the kvmfr device.";
+      };
+    };
+
+    config = {
+      size =
+        mkIf (config.resolution != null) (sizeFromResolution config.resolution);
+    };
+  }));
+in {
+  options.virtualisation.kvmfr = {
+    enable = mkOption {
+      type = types.bool;
+      default = false;
+      description = mdDoc "Whether to enable the kvmfr kernel module.";
+    };
+
+    devices = mkOption {
+      type = types.listOf deviceType;
+      default = [ ];
+      description = mdDoc "List of devices to create.";
+    };
+  };
+
+  config = mkIf cfg.enable {
+    boot.extraModulePackages = with config.boot.kernelPackages; [ kvmfr ];
+    services.udev.packages = optionals (cfg.devices != [ ]) [ udevPackage ];
+
+    environment.etc = {
+      "modules-load.d/kvmfr.conf".text = ''
+        kvmfr
+      '';
+
+      "modprobe.d/kvmfr.conf".text = ''
+        options kvmfr static_size_mb=${
+          concatStringsSep "," (map (size: toString size) deviceSizes)
+        }
+      '';
+
+      "apparmor.d/local/abstractions/libvirt-qemu" =
+        mkIf config.security.apparmor.enable {
+          text = mkIf config.security.apparmor.enable apparmorAbstraction;
+        };
+    };
+
+    virtualisation.libvirtd.deviceACL = devices;
+  };
+}

nixos/libvirtd.nix

@@ -0,0 +1,46 @@
+{ lib, pkgs, config, ... }:
+with lib;
+let
+  cfg = config.virtualisation.libvirtd;
+
+  aclString = with lib.strings;
+    concatMapStringsSep ''
+      ,
+        '' escapeNixString cfg.deviceACL;
+in {
+  options.virtualisation.libvirtd = {
+    deviceACL = mkOption {
+      type = types.listOf types.str;
+      default = [ ];
+      description = "allowed devices";
+    };
+  };
+
+  # All this was part of the rabbit hole of getting looking glass working properly
+  # This below also seems very important
+  # https://forum.level1techs.com/t/new-looking-glass-beta-7-release-candidate-1/208250
+  # same here, wrt cpu max bits possibly preventing shared memory from working
+  # https://forum.level1techs.com/t/looking-glass-b6-and-b7-rc1-not-working-with-new-kernels/222134/7
+  # https://www.kraxel.org/blog/2023/12/qemu-phys-bits/
+  # https://libvirt.org/formatdomain.html#cpu-model-and-topology
+  # 
+  # Needed to make sure to pass
+  # all these different /dev/'s, otherwise qemu won't be able to
+  # start properly. I'm not 100% on either where the user here
+  # got this list, nor which i actually *need*, but either way
+  # this was an immense help:
+  # https://forum.level1techs.com/t/solved-unable-to-connect-to-libvirt-qemu-system-after-changing-to-kernel-module/219006
+
+  # 
+  config.virtualisation.libvirtd.qemu.verbatimConfig = ''
+    namespaces = []
+
+    cgroup_device_acl = [
+      "/dev/null", "/dev/full", "/dev/zero",
+      "/dev/random", "/dev/urandom",
+      "/dev/ptmx", "/dev/kvm",
+      "/dev/userfaultfd",
+      ${aclString}
+    ]
+  '';
+}

nixos/vm.nix

@@ -6,8 +6,8 @@
 }: 
 let
   gpuIDs = [
-    "1002:7480" # Graphics
-    "1002:ab30" # Audio
+    "1002:7480" # Graphics - IOMMU Group 15 / 03:00.0
+    "1002:ab30" # Audio - IOMMU Group 16 / 03:00.1
   ];
 in {
   # https://astrid.tech/2022/09/22/0/nixos-gpu-vfio/ Was a huge gem to find with regard to getting this setup working.
@@ -68,8 +68,12 @@ in {
 
     # These are needed, since I'm not currently trying to
     # reserve a static IP for the bridge interface
-    networking.interfaces.eth0.useDHCP = true;
-    networking.interfaces.winvm0.useDHCP = true;
+    networking.interfaces.winvm0 = {
+      useDHCP = false;
+      ipv4.addresses = [
+        { address = "10.0.5.1"; prefixLength = 16; }
+      ];
+    };
 
     # Trying to ensure the bridge network doesn't cause us to wait
     # on boot
@@ -101,22 +105,22 @@ in {
 
     # note that the VM needs 64 MB for the shmem in side the
     # xml for the full res of the FW laptop
-    systemd.tmpfiles.rules = [
-      "f /dev/shm/scream 0660 evar qemu-libvirtd -"
-      "f /dev/shm/looking-glass 0660 evar qemu-libvirtd -"
-    ];
+    # systemd.tmpfiles.rules = [
+    #   "f /dev/shm/scream 0660 evar qemu-libvirtd -"
+    #   "f /dev/shm/looking-glass 0660 evar qemu-libvirtd -"
+    # ];
 
     # service for hooking up scream for audio
-    systemd.user.services.scream-ivshmem = {
-      enable = true;
-      description = "Scream IVSHMEM";
-      serviceConfig = {
-        ExecStart = "${pkgs.scream}/bin/scream-ivshmem-pulse /dev/shm/scream";
-        Restart = "always";
-      };
-      wantedBy = [ "multi-user.target" ];
-      requires = [ "pulseaudio.service" ];
-    };
+    # systemd.user.services.scream-ivshmem = {
+    #   enable = true;
+    #   description = "Scream IVSHMEM";
+    #   serviceConfig = {
+    #     ExecStart = "${pkgs.scream}/bin/scream-ivshmem-pulse /dev/shm/scream";
+    #     Restart = "always";
+    #   };
+    #   wantedBy = [ "multi-user.target" ];
+    #   requires = [ "pulseaudio.service" ];
+    # };
   
     boot = {
       initrd.kernelModules = [
@@ -166,9 +170,10 @@ in {
           "use sendfile" = "yes";
           #"max protocol" = "smb2";
 
+          "interfaces" = "virbr0";
           # note: localhost is the ipv6 localhost ::1
-          "hosts allow" = "10. 127.0.0.1 localhost";
-          "hosts deny" = "0.0.0.0/0";
+          "hosts allow" = "192.168.122.";
+          # "hosts deny" = "0.0.0.0/0";
           "guest account" = "nobody";
           "map to guest" = "bad user";
         };
@@ -194,5 +199,26 @@ in {
 
     networking.firewall.enable = true;
     networking.firewall.allowPing = true;
+
+    # I got into a stuck state and couldn't start any vm's, whenever I did I got the following:a
+    # $ sudo cat /var/log/libvirt/qemu/win10.log
+    # 2025-01-26T04:41:57.245640Z qemu-system-x86_64: -chardev pty,id=charserial0: Failed to create PTY: Operation not permitted
+    # 2025-01-26 04:41:57.284+0000: shutting down, reason=failed
+    # 
+    # After some searching, several sources stated that this is something with OVH
+    # and the workaround is as below.
+    # 
+    # https://bugzilla.redhat.com/show_bug.cgi?id=1668713
+    # https://www.linuxglobal.com/fixed-libvirtd-qemu-kvm-monitor-unexpectedly-closed-failed-create-chardev-live-migration-virsh-start/
+    fileSystems."devpts" = {
+      device = "devpts";
+      mountPoint = "/dev/pts";
+      fsType = "devpts";
+      noCheck = true;
+      options = [
+        "gid=5"
+        "mode=620"
+      ];
+    };
   };
 }