feat: FAST looking glass! TONS of time on this jesus christ

flake.lock

@@ -193,6 +193,21 @@
         "type": "github"
       }
     },
+    "nix-std": {
+      "locked": {
+        "lastModified": 1710870712,
+        "narHash": "sha256-e+7MJF2gsgTBuOWv4mCimSP0D9+naeFSw9a7N3yEmv4=",
+        "owner": "chessai",
+        "repo": "nix-std",
+        "rev": "31bbc925750cc9d8f828fe55cee1a2bd985e0c00",
+        "type": "github"
+      },
+      "original": {
+        "owner": "chessai",
+        "repo": "nix-std",
+        "type": "github"
+      }
+    },
     "nixos-facter-modules": {
       "locked": {
         "lastModified": 1736931726,
@@ -309,6 +324,7 @@
         "impermanence": "impermanence",
         "muse-sounds-manager": "muse-sounds-manager",
         "nix-index-database": "nix-index-database",
+        "nix-std": "nix-std",
         "nixos-facter-modules": "nixos-facter-modules",
         "nixos-hardware": "nixos-hardware",
         "nixpkgs": "nixpkgs_4"

flake.nix

@@ -3,6 +3,8 @@
     nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable";
     flake-parts.url = "github:hercules-ci/flake-parts";
 
+    nix-std.url = "github:chessai/nix-std";
+
     nixos-hardware.url = "github:NixOS/nixos-hardware";
     disko.url = "github:nix-community/disko";
     disko.inputs.nixpkgs.follows = "nixpkgs";

home/apps.nix

@@ -40,6 +40,7 @@
     # Dev
     pkgs.ptyxis # Terminal emulator
     pkgs.vscodium
+    pkgs.godot_4 
     #pkgs.jetbrains.rider
     #pkgs.jetbrains.rust-rover
     #pkgs.helix

nixos/default.nix

@@ -26,6 +26,8 @@
       ./user-system-config.nix
 
       ./vm.nix
+      (import ./kvmfr.nix { std = inputs.nix-std.lib; lib = lib; pkgs = pkgs; config = config; })
+      ./libvirtd.nix
     ];
 
   vfio = {
@@ -34,6 +36,36 @@
     applyACSpatch = true;
   };
 
+  virtualisation.kvmfr = {
+    enable = true;
+
+    devices = [
+      {
+        size = 128;
+
+        permissions = {
+          user = "evar";
+          group = "qemu-libvirtd";
+          mode = "0660";
+        };
+      }
+    ];
+  };
+
+  # # https://gist.github.com/j-brn/716a03822d256bc5bf5d77b951c7915c
+  # virtualisation.kvmfr = {
+  #   enable = true;
+
+  #   shm = {
+  #     enable = true;
+
+  #     size = 64;
+  #     user = "evar";
+  #     group = "libvirtd";
+  #     mode = "0660";
+  #   };
+  # };
+
   # Allows referring to this flake by the shorthand `nixos-config`, which lets you do e.g.
   # nix repl nixos-config 
   nix.registry.nixos-config.to = {

nixos/kvmfr.nix

@@ -0,0 +1,140 @@
+{ std, lib, pkgs, config, ... }:
+
+with lib;
+
+let
+  cfg = config.virtualisation.kvmfr;
+
+  sizeFromResolution = resolution:
+    let
+      ceilToPowerOf2 = n:
+        std.num.pow 2 (std.num.bits.bitSize - std.num.bits.countLeadingZeros n);
+      pixelSize = if resolution.pixelFormat == "rgb24" then 3 else 4;
+      bytes = resolution.width * resolution.height * pixelSize * 2;
+    in ceilToPowerOf2 (bytes / 1024 / 1024 + 10);
+
+  deviceSizes = map (device: device.size) cfg.devices;
+
+  devices =
+    imap0 (index: _deviceConfig: "/dev/kvmfr${toString index}") cfg.devices;
+
+  udevPackage = pkgs.writeTextDir "/lib/udev/rules.d/99-kvmfr.rules"
+    (concatStringsSep "\n" (imap0 (index: deviceConfig: ''
+      SUBSYSTEM=="kvmfr", KERNEL=="kvmfr${
+        toString index
+      }", OWNER="${deviceConfig.permissions.user}", GROUP="${deviceConfig.permissions.group}", MODE="${deviceConfig.permissions.mode}", TAG+="systemd"
+    '') cfg.devices));
+
+  apparmorAbstraction =
+    concatStringsSep "\n" (map (device: "${device} rw") devices);
+
+  permissionsType = types.submodule {
+    options = {
+      user = mkOption {
+        type = types.str;
+        default = "root";
+        description = mdDoc "Owner of the shared memory device.";
+      };
+      group = mkOption {
+        type = types.str;
+        default = "root";
+        description = mdDoc "Group of the shared memory device.";
+      };
+      mode = mkOption {
+        type = types.str;
+        default = "0600";
+        description = mdDoc "Mode of the shared memory device.";
+      };
+    };
+  };
+
+  resolutionType = types.submodule {
+    options = {
+      width = mkOption {
+        type = types.number;
+        description = mdDoc
+          "Maximum horizontal video size that should be supported by this device.";
+      };
+
+      height = mkOption {
+        type = types.number;
+        description = mdDoc
+          "Maximum vertical video size that should be supported by this device.";
+      };
+
+      pixelFormat = mkOption {
+        type = types.enum [ "rgba32" "rgb24" ];
+        description = mdDoc "Pixel format to use.";
+        default = "rgba32";
+      };
+    };
+  };
+
+  deviceType = (types.submodule ({ config, options, ... }: {
+    options = {
+      resolution = mkOption {
+        type = types.nullOr resolutionType;
+        default = null;
+        description = mdDoc ''
+          Automatically calculate the minimum device size for a specific resolution. Overrides `size` if set.
+        '';
+      };
+
+      size = mkOption {
+        type = types.number;
+        description = mdDoc ''
+          Size for the kvmfr device in megabytes.
+        '';
+      };
+
+      permissions = mkOption {
+        type = permissionsType;
+        default = { };
+        description = mdDoc "Permissions of the kvmfr device.";
+      };
+    };
+
+    config = {
+      size =
+        mkIf (config.resolution != null) (sizeFromResolution config.resolution);
+    };
+  }));
+in {
+  options.virtualisation.kvmfr = {
+    enable = mkOption {
+      type = types.bool;
+      default = false;
+      description = mdDoc "Whether to enable the kvmfr kernel module.";
+    };
+
+    devices = mkOption {
+      type = types.listOf deviceType;
+      default = [ ];
+      description = mdDoc "List of devices to create.";
+    };
+  };
+
+  config = mkIf cfg.enable {
+    boot.extraModulePackages = with config.boot.kernelPackages; [ kvmfr ];
+    services.udev.packages = optionals (cfg.devices != [ ]) [ udevPackage ];
+
+    environment.etc = {
+      "modules-load.d/kvmfr.conf".text = ''
+        kvmfr
+      '';
+
+      "modprobe.d/kvmfr.conf".text = ''
+        options kvmfr static_size_mb=${
+          concatStringsSep "," (map (size: toString size) deviceSizes)
+        }
+      '';
+
+      "apparmor.d/local/abstractions/libvirt-qemu" =
+        mkIf config.security.apparmor.enable {
+          text = mkIf config.security.apparmor.enable apparmorAbstraction;
+        };
+    };
+
+    virtualisation.libvirtd.deviceACL = devices;
+  };
+}

nixos/libvirtd.nix

@@ -0,0 +1,46 @@
+{ lib, pkgs, config, ... }:
+with lib;
+let
+  cfg = config.virtualisation.libvirtd;
+
+  aclString = with lib.strings;
+    concatMapStringsSep ''
+      ,
+        '' escapeNixString cfg.deviceACL;
+in {
+  options.virtualisation.libvirtd = {
+    deviceACL = mkOption {
+      type = types.listOf types.str;
+      default = [ ];
+      description = "allowed devices";
+    };
+  };
+
+  # All this was part of the rabbit hole of getting looking glass working properly
+  # This below also seems very important
+  # https://forum.level1techs.com/t/new-looking-glass-beta-7-release-candidate-1/208250
+  # same here, wrt cpu max bits possibly preventing shared memory from working
+  # https://forum.level1techs.com/t/looking-glass-b6-and-b7-rc1-not-working-with-new-kernels/222134/7
+  # https://www.kraxel.org/blog/2023/12/qemu-phys-bits/
+  # https://libvirt.org/formatdomain.html#cpu-model-and-topology
+  # 
+  # Needed to make sure to pass
+  # all these different /dev/'s, otherwise qemu won't be able to
+  # start properly. I'm not 100% on either where the user here
+  # got this list, nor which i actually *need*, but either way
+  # this was an immense help:
+  # https://forum.level1techs.com/t/solved-unable-to-connect-to-libvirt-qemu-system-after-changing-to-kernel-module/219006
+
+  # 
+  config.virtualisation.libvirtd.qemu.verbatimConfig = ''
+    namespaces = []
+
+    cgroup_device_acl = [
+      "/dev/null", "/dev/full", "/dev/zero",
+      "/dev/random", "/dev/urandom",
+      "/dev/ptmx", "/dev/kvm",
+      "/dev/userfaultfd",
+      ${aclString}
+    ]
+  '';
+}

nixos/vm.nix

@@ -6,8 +6,8 @@
 }: 
 let
   gpuIDs = [
-    "1002:7480" # Graphics
+    "1002:7480" # Graphics - IOMMU Group 15 / 03:00.0
-    "1002:ab30" # Audio
+    "1002:ab30" # Audio - IOMMU Group 16 / 03:00.1
   ];
 in {
   # https://astrid.tech/2022/09/22/0/nixos-gpu-vfio/ Was a huge gem to find with regard to getting this setup working.
@@ -68,8 +68,12 @@ in {
 
     # These are needed, since I'm not currently trying to
     # reserve a static IP for the bridge interface
-    networking.interfaces.eth0.useDHCP = true;
+    networking.interfaces.winvm0 = {
-    networking.interfaces.winvm0.useDHCP = true;
+      useDHCP = false;
+      ipv4.addresses = [
+        { address = "10.0.5.1"; prefixLength = 16; }
+      ];
+    };
 
     # Trying to ensure the bridge network doesn't cause us to wait
     # on boot
@@ -101,22 +105,22 @@ in {
 
     # note that the VM needs 64 MB for the shmem in side the
     # xml for the full res of the FW laptop
-    systemd.tmpfiles.rules = [
+    # systemd.tmpfiles.rules = [
-      "f /dev/shm/scream 0660 evar qemu-libvirtd -"
+    #   "f /dev/shm/scream 0660 evar qemu-libvirtd -"
-      "f /dev/shm/looking-glass 0660 evar qemu-libvirtd -"
+    #   "f /dev/shm/looking-glass 0660 evar qemu-libvirtd -"
-    ];
+    # ];
 
     # service for hooking up scream for audio
-    systemd.user.services.scream-ivshmem = {
+    # systemd.user.services.scream-ivshmem = {
-      enable = true;
+    #   enable = true;
-      description = "Scream IVSHMEM";
+    #   description = "Scream IVSHMEM";
-      serviceConfig = {
+    #   serviceConfig = {
-        ExecStart = "${pkgs.scream}/bin/scream-ivshmem-pulse /dev/shm/scream";
+    #     ExecStart = "${pkgs.scream}/bin/scream-ivshmem-pulse /dev/shm/scream";
-        Restart = "always";
+    #     Restart = "always";
-      };
+    #   };
-      wantedBy = [ "multi-user.target" ];
+    #   wantedBy = [ "multi-user.target" ];
-      requires = [ "pulseaudio.service" ];
+    #   requires = [ "pulseaudio.service" ];
-    };
+    # };
   
     boot = {
       initrd.kernelModules = [
@@ -166,9 +170,10 @@ in {
           "use sendfile" = "yes";
           #"max protocol" = "smb2";
 
+          "interfaces" = "virbr0";
           # note: localhost is the ipv6 localhost ::1
-          "hosts allow" = "10. 127.0.0.1 localhost";
+          "hosts allow" = "192.168.122.";
-          "hosts deny" = "0.0.0.0/0";
+          # "hosts deny" = "0.0.0.0/0";
           "guest account" = "nobody";
           "map to guest" = "bad user";
         };
@@ -194,5 +199,26 @@ in {
 
     networking.firewall.enable = true;
     networking.firewall.allowPing = true;
+
+    # I got into a stuck state and couldn't start any vm's, whenever I did I got the following:a
+    # $ sudo cat /var/log/libvirt/qemu/win10.log
+    # 2025-01-26T04:41:57.245640Z qemu-system-x86_64: -chardev pty,id=charserial0: Failed to create PTY: Operation not permitted
+    # 2025-01-26 04:41:57.284+0000: shutting down, reason=failed
+    # 
+    # After some searching, several sources stated that this is something with OVH
+    # and the workaround is as below.
+    # 
+    # https://bugzilla.redhat.com/show_bug.cgi?id=1668713
+    # https://www.linuxglobal.com/fixed-libvirtd-qemu-kvm-monitor-unexpectedly-closed-failed-create-chardev-live-migration-virsh-start/
+    fileSystems."devpts" = {
+      device = "devpts";
+      mountPoint = "/dev/pts";
+      fsType = "devpts";
+      noCheck = true;
+      options = [
+        "gid=5"
+        "mode=620"
+      ];
+    };
   };
 }

win10.xml

@@ -1,4 +1,4 @@
-<domain type="kvm">
+<domain xmlns:qemu="http://libvirt.org/schemas/domain/qemu/1.0" type="kvm">
   <name>win10</name>
   <uuid>d5c9ecd5-418d-491a-a4f9-6b7c4a4df28d</uuid>
   <metadata>
@@ -6,13 +6,13 @@
       <libosinfo:os id="http://microsoft.com/win/10"/>
     </libosinfo:libosinfo>
   </metadata>
-  <memory unit="KiB">16777216</memory>
+  <memory unit="KiB">25165824</memory>
-  <currentMemory unit="KiB">16777216</currentMemory>
+  <currentMemory unit="KiB">25165824</currentMemory>
   <memoryBacking>
     <source type="memfd"/>
     <access mode="shared"/>
   </memoryBacking>
-  <vcpu placement="static">8</vcpu>
+  <vcpu placement="static">14</vcpu>
   <cputune>
     <vcpupin vcpu="0" cpuset="0"/>
     <vcpupin vcpu="1" cpuset="1"/>
@@ -22,12 +22,18 @@
     <vcpupin vcpu="5" cpuset="5"/>
     <vcpupin vcpu="6" cpuset="6"/>
     <vcpupin vcpu="7" cpuset="7"/>
+    <vcpupin vcpu="8" cpuset="8"/>
+    <vcpupin vcpu="9" cpuset="9"/>
+    <vcpupin vcpu="10" cpuset="10"/>
+    <vcpupin vcpu="11" cpuset="11"/>
+    <vcpupin vcpu="12" cpuset="12"/>
+    <vcpupin vcpu="13" cpuset="13"/>
   </cputune>
   <os>
     <type arch="x86_64" machine="pc-q35-9.2">hvm</type>
     <loader readonly="yes" type="pflash" format="raw">/run/libvirt/nix-ovmf/OVMF_CODE.fd</loader>
     <nvram template="/run/libvirt/nix-ovmf/OVMF_VARS.fd" templateFormat="raw" format="raw">/var/lib/libvirt/qemu/nvram/win10_VARS.fd</nvram>
-    <bootmenu enable="yes"/>
+    <bootmenu enable="no"/>
   </os>
   <features>
     <acpi/>
@@ -49,10 +55,15 @@
       <tlbflush state="on"/>
       <ipi state="on"/>
     </hyperv>
+    <kvm>
+      <hidden state="on"/>
+    </kvm>
     <vmport state="off"/>
+    <ioapic driver="kvm"/>
   </features>
   <cpu mode="host-passthrough" check="none" migratable="on">
-    <topology sockets="1" dies="1" clusters="1" cores="4" threads="2"/>
+    <topology sockets="1" dies="1" clusters="1" cores="7" threads="2"/>
+    <maxphysaddr mode="passthrough" limit="40"/>
     <feature policy="require" name="topoext"/>
   </cpu>
   <clock offset="localtime">
@@ -83,22 +94,6 @@
       <target dev="vdb" bus="virtio"/>
       <address type="pci" domain="0x0000" bus="0x08" slot="0x00" function="0x0"/>
     </disk>
-    <disk type="file" device="cdrom">
-      <driver name="qemu" type="raw"/>
-      <source file="/home/evar/Downloads/Win10_22H2_English_x64v1.iso"/>
-      <target dev="sdb" bus="sata"/>
-      <readonly/>
-      <boot order="2"/>
-      <address type="drive" controller="0" bus="0" target="0" unit="1"/>
-    </disk>
-    <disk type="file" device="cdrom">
-      <driver name="qemu" type="raw"/>
-      <source file="/home/evar/Downloads/virtio-win-0.1.266.iso"/>
-      <target dev="sdc" bus="sata"/>
-      <readonly/>
-      <boot order="3"/>
-      <address type="drive" controller="0" bus="0" target="0" unit="2"/>
-    </disk>
     <controller type="usb" index="0" model="qemu-xhci" ports="15">
       <address type="pci" domain="0x0000" bus="0x02" slot="0x00" function="0x0"/>
     </controller>
@@ -191,23 +186,23 @@
     <filesystem type="mount" accessmode="passthrough">
       <driver type="virtiofs"/>
       <binary path="/run/current-system/sw/bin/virtiofsd"/>
-      <source dir="/home/evar/Virtio Shared"/>
+      <source dir="/home/evar/Virtio Shared/"/>
       <target dir="Virtio Shared"/>
-      <address type="pci" domain="0x0000" bus="0x0b" slot="0x00" function="0x0"/>
+      <address type="pci" domain="0x0000" bus="0x0c" slot="0x00" function="0x0"/>
     </filesystem>
-    <interface type="bridge">
-      <mac address="52:54:00:ef:ae:20"/>
-      <source bridge="winvm0"/>
-      <model type="virtio"/>
-      <link state="up"/>
-      <address type="pci" domain="0x0000" bus="0x09" slot="0x00" function="0x0"/>
-    </interface>
     <interface type="network">
       <mac address="52:54:00:15:1c:97"/>
       <source network="default"/>
       <model type="virtio"/>
       <address type="pci" domain="0x0000" bus="0x0a" slot="0x00" function="0x0"/>
     </interface>
+    <interface type="bridge">
+      <mac address="52:54:00:33:7d:5e"/>
+      <source bridge="winvm0"/>
+      <model type="virtio"/>
+      <link state="up"/>
+      <address type="pci" domain="0x0000" bus="0x0b" slot="0x00" function="0x0"/>
+    </interface>
     <serial type="pty">
       <target type="isa-serial" port="0">
         <model name="isa-serial"/>
@@ -216,6 +211,9 @@
     <console type="pty">
       <target type="serial" port="0"/>
     </console>
+    <console type="pty">
+      <target type="virtio" port="1"/>
+    </console>
     <channel type="unix">
       <target type="virtio" name="org.qemu.guest_agent.0"/>
       <address type="virtio-serial" controller="0" bus="0" port="2"/>
@@ -224,6 +222,12 @@
       <target type="virtio" name="com.redhat.spice.0"/>
       <address type="virtio-serial" controller="0" bus="0" port="1"/>
     </channel>
+    <input type="mouse" bus="virtio">
+      <address type="pci" domain="0x0000" bus="0x07" slot="0x00" function="0x0"/>
+    </input>
+    <input type="keyboard" bus="virtio">
+      <address type="pci" domain="0x0000" bus="0x09" slot="0x00" function="0x0"/>
+    </input>
     <input type="mouse" bus="ps2"/>
     <input type="keyboard" bus="ps2"/>
     <graphics type="spice" port="-1" autoport="no">
@@ -232,14 +236,12 @@
       <gl enable="no"/>
     </graphics>
     <sound model="ich9">
+      <audio id="1"/>
       <address type="pci" domain="0x0000" bus="0x00" slot="0x1b" function="0x0"/>
     </sound>
-    <audio id="1" type="none"/>
+    <audio id="1" type="spice"/>
     <video>
-      <model type="virtio" heads="1" primary="yes">
+      <model type="none"/>
-        <acceleration accel3d="no"/>
-      </model>
-      <address type="pci" domain="0x0000" bus="0x00" slot="0x01" function="0x0"/>
     </video>
     <hostdev mode="subsystem" type="pci" managed="yes">
       <source>
@@ -255,24 +257,18 @@
     </hostdev>
     <hostdev mode="subsystem" type="usb" managed="yes">
       <source>
-        <vendor id="0x320f"/>
+        <vendor id="0x256c"/>
-        <product id="0x5088"/>
+        <product id="0x006b"/>
       </source>
-      <address type="usb" bus="0" port="2"/>
+      <address type="usb" bus="0" port="1"/>
     </hostdev>
     <watchdog model="itco" action="reset"/>
-    <memballoon model="virtio">
+    <memballoon model="none"/>
-      <address type="pci" domain="0x0000" bus="0x07" slot="0x00" function="0x0"/>
-    </memballoon>
-    <shmem name="looking-glass">
-      <model type="ivshmem-plain"/>
-      <size unit="M">64</size>
-      <address type="pci" domain="0x0000" bus="0x10" slot="0x01" function="0x0"/>
-    </shmem>
-    <shmem name="scream">
-      <model type="ivshmem-plain"/>
-      <size unit="M">2</size>
-      <address type="pci" domain="0x0000" bus="0x10" slot="0x02" function="0x0"/>
-    </shmem>
   </devices>
+  <qemu:commandline>
+    <qemu:arg value="-device"/>
+    <qemu:arg value="{&quot;driver&quot;:&quot;ivshmem-plain&quot;,&quot;id&quot;:&quot;shmem0&quot;,&quot;memdev&quot;:&quot;looking-glass&quot;}"/>
+    <qemu:arg value="-object"/>
+    <qemu:arg value="{&quot;qom-type&quot;:&quot;memory-backend-file&quot;,&quot;id&quot;:&quot;looking-glass&quot;,&quot;mem-path&quot;:&quot;/dev/kvmfr0&quot;,&quot;size&quot;:134217728,&quot;share&quot;:true}"/>
+  </qemu:commandline>
 </domain>