{ config, lib, pkgs, ... }: let gpuIDs = [ "1002:7480" # Graphics - IOMMU Group 15 / 03:00.0 "1002:ab30" # Audio - IOMMU Group 16 / 03:00.1 ]; in { # https://astrid.tech/2022/09/22/0/nixos-gpu-vfio/ Was a huge gem to find with regard to getting this setup working. # Absolutely great. # # Web Archive: http://web.archive.org/web/20241229020334/https://astrid.tech/2022/09/22/0/nixos-gpu-vfio/ options.vfio.enable = with lib; mkEnableOption "Configure the machine for VFIO"; options.vfio.earlyKMS = with lib; mkEnableOption "Configure the machine to load the GPU driver during initramfs"; options.vfio.applyACSpatch = with lib; mkEnableOption ''If set, the following things will happen: - The ACS override patch is applied - Applies the i915-vga-arbiter patch - Adds pcie_acs_override=downstream to the command line ''; config = let cfg = config.vfio; in { # Move me services.udev.packages = [ ( let virsh = "${config.virtualisation.libvirtd.package}/bin/virsh"; updateBin = pkgs.writeShellScript "vm-pass-usb-update.sh" '' # todo param vm_name="win10" read -r -d ''\'''\' xml_template <<'EOF'
EOF BUSNUM=$((10#$BUSNUM)) DEVNUM=$((10#$DEVNUM)) if test "$ACTION" = "add" then printf "$xml_template" "$BUSNUM" "$DEVNUM" | \ ${virsh} attach-device --persistent -- "$vm_name" /dev/stdin elif test "$ACTION" = "remove" then printf "$xml_template" "$BUSNUM" "$DEVNUM" | \ ${virsh} detach-device --persistent -- "$vm_name" /dev/stdin fi ''; in pkgs.writeTextDir "/lib/udev/rules.d/60-vm-attach-usb-anker-hub.rules" '' SUBSYSTEM=="usb", ACTION=="add", ATTRS{idVendor}=="0bda", ATTRS{idProduct}=="5411", ATTR{idVendor}!="0bda", ATTR{idProduct}!="5411", RUN+="${updateBin}" SUBSYSTEM=="usb", ACTION=="remove", RUN+="${updateBin}" '' ) ]; # Useful: # https://nixos.mayflower.consulting/blog/2020/06/17/windows-vm-performance/ # TODO: The bridge interface is necessary in order to # have network discovery in the virtual machine. However, # this will cause a massive slowdown in startup time # on the host machine while ethernet is not connected. # Should fix this, but for now it's fine. # # Emily used systemd-analyze a lot in order to help triage # which got us to an alright point. # TODO: Right now, I need to manually start # the network bridge interface with systemctl start network-addresses-winvm0.service, # and potentially toggle the link state in the vm config # to get this working. would be good to fix it. networking.bridges = { "winvm0" = { interfaces = [ "eth0" ]; }; }; networking.dhcpcd.denyInterfaces = [ "winvm0" ]; # TODO: the below doesn't actually work, so I just # disable wait-online entirely. # ensure the bridge network doesn't cause us to wait # on boot # systemd.network.wait-online.ignoredInterfaces = [ "eth0" "winvm0" ]; systemd.network.wait-online.anyInterface = true; boot.initrd.systemd.network.wait-online.ignoredInterfaces = [ "eth0" "winvm0" ]; # These are needed, since I'm not currently trying to # reserve a static IP for the bridge interface networking.defaultGateway = { address = "10.0.0.1"; interface = "eth0"; }; networking.interfaces.winvm0 = { useDHCP = false; ipv4 = { addresses = [ { address = "10.0.5.1"; prefixLength = 32; } ]; }; }; environment.systemPackages = [ # For sharing filesystems # I followed https://www.heiko-sieger.info/sharing-files-between-the-linux-host-and-a-windows-vm-using-virtiofs/ # when I was setting up the windows VM (though I already had the guest tools installed). Worked like a charm! # Note that I temporarily need to add the following to the guest OS filesystem xml # # due to this issue https://github.com/NixOS/nixpkgs/issues/347942 pkgs.virtiofsd pkgs.looking-glass-client pkgs.scream ]; programs.virt-manager.enable = true; users.groups.libvirtd.members = ["evar"]; # let me do stuff with vms # TODO: I don't currently have a satisfying way of passing through # my huion tablet when I connect it. I'm pretty sure # that really good blog post I found initially talks about this # and provides a script for mounting/unmounting stuff dynamically # that I could look into for this. virtualisation.spiceUSBRedirection.enable = true; # allows usb passthrough hardware.graphics.enable = true; # needed for display spice opengl virtualisation.libvirtd = { enable = true; qemu.swtpm.enable = true; # for TPM 2.0 support onBoot = "ignore"; # only start autostart vms, not just ones that were running onShutdown = "shutdown"; # always shut down the vm's cleanly }; # shared memory for looking glass # see https://looking-glass.io/docs/B7-rc1/ivshmem_shm/ # or https://alexbakker.me/post/nixos-pci-passthrough-qemu-vfio.html # note that the VM needs 64 MB for the shmem in side the # xml for the full res of the FW laptop systemd.tmpfiles.rules = [ "f /dev/shm/scream 0660 evar qemu-libvirtd -" ]; # service for hooking up scream for audio systemd.user.services.scream-ivshmem = { enable = true; description = "Scream"; serviceConfig = { ExecStart = "${pkgs.scream}/bin/scream -v -n scream -o pulse -m /dev/shm/scream"; Restart = "always"; }; wantedBy = [ "multi-user.target" ]; requires = [ "pipewire-pulse.service" "pipewire.service" "sound.target" ]; }; boot = { initrd.kernelModules = [ "vfio_pci" "vfio" "vfio_iommu_type1" # "vfio_virqfd" # This is apparently a part of the kernel now ] ++ lib.optional cfg.earlyKMS "amdgpu"; # kernelPatches = [] ++ lib.optional cfg.applyACSpatch # { # name = "add-acs-overrides"; # patch = pkgs.fetchurl { # name = "add-acs-overrides.patch"; # url = "https://aur.archlinux.org/cgit/aur.git/plain/1001-6.8.0-add-acs-overrides.patch?h=linux-vfio"; # sha256 = "1qd68s9r0ppynksbffqn2qbp1whqpbfp93dpccp9griwhx5srx6v"; # }; # }; kernelParams = [ # enable IOMMU "amd_iommu=on" ] ++ lib.optional cfg.enable # isolate the GPU ("vfio-pci.ids=" + lib.concatStringsSep "," gpuIDs); # ++ lib.optional cfg.applyACSpatch "pcie_acs_override=downstream,multifunction"; }; # Samba share. Primarily intended to be used via the # bridged network adapter for speed # # TODO: https://www.samba.org/samba/docs/current/man-html/vfs_btrfs.8.html # to take advantage of btrfs stuff services.samba = { enable = true; openFirewall = true; settings = { global = { "workgroup" = "WORKGROUP"; "server string" = "Atreus"; "netbios name" = "Atreus"; "security" = "user"; "username map" = "${./smb-usernames.map}"; # don't show shares to people who aren't valid to see them "access based share enum" = "yes"; # only allow authenticated users - this might break old windows apps "restrict anonymous" = "2"; "use sendfile" = "yes"; #"max protocol" = "smb2"; "interfaces" = "virbr0"; # note: localhost is the ipv6 localhost ::1 "hosts allow" = "192.168.122. 100.64.0.0/10"; # "hosts deny" = "0.0.0.0/0"; "guest account" = "nobody"; "map to guest" = "bad user"; # Stuff for MacOS # see https://wiki.samba.org/index.php/Configure_Samba_to_Work_Better_with_Mac_OS_X # for additional settings - see manpage for vfs_fruit "vfs objects" = "fruit streams_xattr"; # load in modules, enable APPL extensions - order is critical "fruit:metadata" = "stream"; # stores osx medatadata "fruit:model" = "MacSamba"; # server icon in finder "fruit:veto_appledouble" = "no"; # following stuff generally cleans up files "fruit:zero_file_id" = "yes"; "fruit:wipe_intentionally_left_blank_rfork" = "yes"; "fruit:delete_empty_adfiles" = "yes"; "fruit:posix_rename" = "yes"; "fruit:nfs_aces" = "no"; # prevents macOS clients from motifying the UNIX mode of directories that use NFS ACEs }; "TimeMachineBackup" = { "fruit:time machine" = "yes"; # "fruit:time machine max size" = "SIZE"; }; "Virtio Shared" = { # macos spotlight indexing backend # see smb.conf for other elasticsearch params "spotlight backend" = "elasticsearch"; "path" = "/home/evar/Virtio Shared"; "comment" = "Virtio shared directory"; "valid users" = "evar"; "force user" = "evar"; # POSIX ACE entry maps to Full Control ACL in windows "acl map full control" = "yes"; # allow users with write access to also change perms "dos filemode" = "yes"; # files created by a samba client have posix ace "force create mode" = 0777; "read only" = "no"; "public" = "no"; "guest ok" = "no"; "browseable" = "yes"; }; }; }; services.samba-wsdd = { enable = true; openFirewall = true; }; networking.firewall.enable = true; networking.firewall.allowPing = true; # I got into a stuck state and couldn't start any vm's, whenever I did I got the following:a # $ sudo cat /var/log/libvirt/qemu/win10.log # 2025-01-26T04:41:57.245640Z qemu-system-x86_64: -chardev pty,id=charserial0: Failed to create PTY: Operation not permitted # 2025-01-26 04:41:57.284+0000: shutting down, reason=failed # # After some searching, several sources stated that this is something with OVH # and the workaround is as below. # # https://bugzilla.redhat.com/show_bug.cgi?id=1668713 # https://www.linuxglobal.com/fixed-libvirtd-qemu-kvm-monitor-unexpectedly-closed-failed-create-chardev-live-migration-virsh-start/ fileSystems."devpts" = { device = "devpts"; mountPoint = "/dev/pts"; fsType = "devpts"; noCheck = true; options = [ "gid=5" "mode=620" ]; }; }; }