{
config,
lib,
pkgs,
...
}:
let
gpuIDs = [
"1002:7480" # Graphics - IOMMU Group 15 / 03:00.0
"1002:ab30" # Audio - IOMMU Group 16 / 03:00.1
];
in {
# https://astrid.tech/2022/09/22/0/nixos-gpu-vfio/ Was a huge gem to find with regard to getting this setup working.
# Absolutely great.
#
# Web Archive: http://web.archive.org/web/20241229020334/https://astrid.tech/2022/09/22/0/nixos-gpu-vfio/
options.vfio.enable = with lib;
mkEnableOption "Configure the machine for VFIO";
options.vfio.earlyKMS = with lib;
mkEnableOption "Configure the machine to load the GPU driver during initramfs";
options.vfio.applyACSpatch = with lib;
mkEnableOption
''If set, the following things will happen:
- The ACS override patch is applied
- Applies the i915-vga-arbiter patch
- Adds pcie_acs_override=downstream to the command line
'';
config = let cfg = config.vfio;
in {
# Move me
services.udev.packages = [
(
let
virsh = "${config.virtualisation.libvirtd.package}/bin/virsh";
updateBin = pkgs.writeShellScript "vm-pass-usb-update.sh" ''
# todo param
vm_name="win10"
read -r -d ''\'''\' xml_template <<'EOF'
EOF
BUSNUM=$((10#$BUSNUM))
DEVNUM=$((10#$DEVNUM))
if test "$ACTION" = "add"
then
printf "$xml_template" "$BUSNUM" "$DEVNUM" | \
${virsh} attach-device --persistent -- "$vm_name" /dev/stdin
elif test "$ACTION" = "remove"
then
printf "$xml_template" "$BUSNUM" "$DEVNUM" | \
${virsh} detach-device --persistent -- "$vm_name" /dev/stdin
fi
'';
in
pkgs.writeTextDir "/lib/udev/rules.d/99-vm-attach-usb-anker-hub.rules" ''
SUBSYSTEM=="usb", ACTION=="add", ATTRS{idVendor}=="0bda", ATTRS{idProduct}=="5411", ATTR{idVendor}!="0bda", ATTR{idProduct}!="5411", RUN+="${updateBin}"
SUBSYSTEM=="usb", ACTION=="remove", RUN+="${updateBin}"
''
)
];
# Useful:
# https://nixos.mayflower.consulting/blog/2020/06/17/windows-vm-performance/
# TODO: The bridge interface is necessary in order to
# have network discovery in the virtual machine. However,
# this will cause a massive slowdown in startup time
# on the host machine while ethernet is not connected.
# Should fix this, but for now it's fine.
#
# Emily used systemd-analyze a lot in order to help triage
# which got us to an alright point.
# TODO: Right now, I need to manually start
# the network bridge interface with systemctl start network-addresses-winvm0.service,
# and potentially toggle the link state in the vm config
# to get this working. would be good to fix it.
networking.bridges = {
"winvm0" = {
interfaces = [ "eth0" ];
};
};
environment.systemPackages = [
# For sharing filesystems
# I followed https://www.heiko-sieger.info/sharing-files-between-the-linux-host-and-a-windows-vm-using-virtiofs/
# when I was setting up the windows VM (though I already had the guest tools installed). Worked like a charm!
# Note that I temporarily need to add the following to the guest OS filesystem xml
#
# due to this issue https://github.com/NixOS/nixpkgs/issues/347942
pkgs.virtiofsd
pkgs.looking-glass-client
pkgs.scream
];
# These are needed, since I'm not currently trying to
# reserve a static IP for the bridge interface
networking.interfaces.winvm0 = {
useDHCP = false;
ipv4.addresses = [
{ address = "10.0.5.1"; prefixLength = 16; }
];
};
# Trying to ensure the bridge network doesn't cause us to wait
# on boot
systemd.services.network-addresses-eth0.before = lib.mkForce [ ];
systemd.services.network-addresses-winvm0.before = lib.mkForce [ ];
systemd.services.winvm0-netdev.before = lib.mkForce [ ];
programs.virt-manager.enable = true;
users.groups.libvirtd.members = ["evar"]; # let me do stuff with vms
# TODO: I don't currently have a satisfying way of passing through
# my huion tablet when I connect it. I'm pretty sure
# that really good blog post I found initially talks about this
# and provides a script for mounting/unmounting stuff dynamically
# that I could look into for this.
virtualisation.spiceUSBRedirection.enable = true; # allows usb passthrough
hardware.graphics.enable = true; # needed for display spice opengl
virtualisation.libvirtd = {
enable = true;
qemu.swtpm.enable = true; # for TPM 2.0 support
onBoot = "ignore"; # only start autostart vms, not just ones that were running
onShutdown = "shutdown"; # always shut down the vm's cleanly
};
# shared memory for looking glass
# see https://looking-glass.io/docs/B7-rc1/ivshmem_shm/
# or https://alexbakker.me/post/nixos-pci-passthrough-qemu-vfio.html
# note that the VM needs 64 MB for the shmem in side the
# xml for the full res of the FW laptop
# systemd.tmpfiles.rules = [
# "f /dev/shm/scream 0660 evar qemu-libvirtd -"
# "f /dev/shm/looking-glass 0660 evar qemu-libvirtd -"
# ];
# service for hooking up scream for audio
# systemd.user.services.scream-ivshmem = {
# enable = true;
# description = "Scream IVSHMEM";
# serviceConfig = {
# ExecStart = "${pkgs.scream}/bin/scream-ivshmem-pulse /dev/shm/scream";
# Restart = "always";
# };
# wantedBy = [ "multi-user.target" ];
# requires = [ "pulseaudio.service" ];
# };
boot = {
initrd.kernelModules = [
"vfio_pci"
"vfio"
"vfio_iommu_type1"
# "vfio_virqfd" # This is apparently a part of the kernel now
] ++ lib.optional cfg.earlyKMS "amdgpu";
# kernelPatches = [] ++ lib.optional cfg.applyACSpatch
# {
# name = "add-acs-overrides";
# patch = pkgs.fetchurl {
# name = "add-acs-overrides.patch";
# url = "https://aur.archlinux.org/cgit/aur.git/plain/1001-6.8.0-add-acs-overrides.patch?h=linux-vfio";
# sha256 = "1qd68s9r0ppynksbffqn2qbp1whqpbfp93dpccp9griwhx5srx6v";
# };
# };
kernelParams = [
# enable IOMMU
"amd_iommu=on"
] ++ lib.optional cfg.enable
# isolate the GPU
("vfio-pci.ids=" + lib.concatStringsSep "," gpuIDs);
# ++ lib.optional cfg.applyACSpatch "pcie_acs_override=downstream,multifunction";
};
# Samba share. Primarily intended to be used via the
# bridged network adapter for speed
services.samba = {
enable = true;
openFirewall = true;
settings = {
global = {
"workgroup" = "WORKGROUP";
"server string" = "Atreus";
"netbios name" = "Atreus";
"security" = "user";
# don't show shares to people who aren't valid to see them
"access based share enum" = "yes";
# only allow authenticated users - this might break old windows apps
"restrict anonymous" = "2";
"use sendfile" = "yes";
#"max protocol" = "smb2";
"interfaces" = "virbr0";
# note: localhost is the ipv6 localhost ::1
"hosts allow" = "192.168.122.";
# "hosts deny" = "0.0.0.0/0";
"guest account" = "nobody";
"map to guest" = "bad user";
};
"Virtio Shared" = {
"path" = "/home/evar/Virtio Shared";
"comment" = "Virtio shared directory";
"valid users" = "evar";
"read only" = "no";
"public" = "no";
"guest ok" = "no";
"browseable" = "yes";
};
};
};
services.samba-wsdd = {
enable = true;
openFirewall = true;
};
networking.firewall.enable = true;
networking.firewall.allowPing = true;
# I got into a stuck state and couldn't start any vm's, whenever I did I got the following:a
# $ sudo cat /var/log/libvirt/qemu/win10.log
# 2025-01-26T04:41:57.245640Z qemu-system-x86_64: -chardev pty,id=charserial0: Failed to create PTY: Operation not permitted
# 2025-01-26 04:41:57.284+0000: shutting down, reason=failed
#
# After some searching, several sources stated that this is something with OVH
# and the workaround is as below.
#
# https://bugzilla.redhat.com/show_bug.cgi?id=1668713
# https://www.linuxglobal.com/fixed-libvirtd-qemu-kvm-monitor-unexpectedly-closed-failed-create-chardev-live-migration-virsh-start/
fileSystems."devpts" = {
device = "devpts";
mountPoint = "/dev/pts";
fsType = "devpts";
noCheck = true;
options = [
"gid=5"
"mode=620"
];
};
};
}