diff --git a/flake.lock b/flake.lock index 52c32d1..102cd20 100644 --- a/flake.lock +++ b/flake.lock @@ -193,6 +193,21 @@ "type": "github" } }, + "nix-std": { + "locked": { + "lastModified": 1710870712, + "narHash": "sha256-e+7MJF2gsgTBuOWv4mCimSP0D9+naeFSw9a7N3yEmv4=", + "owner": "chessai", + "repo": "nix-std", + "rev": "31bbc925750cc9d8f828fe55cee1a2bd985e0c00", + "type": "github" + }, + "original": { + "owner": "chessai", + "repo": "nix-std", + "type": "github" + } + }, "nixos-facter-modules": { "locked": { "lastModified": 1736931726, @@ -309,6 +324,7 @@ "impermanence": "impermanence", "muse-sounds-manager": "muse-sounds-manager", "nix-index-database": "nix-index-database", + "nix-std": "nix-std", "nixos-facter-modules": "nixos-facter-modules", "nixos-hardware": "nixos-hardware", "nixpkgs": "nixpkgs_4" diff --git a/flake.nix b/flake.nix index ed64717..cf07479 100644 --- a/flake.nix +++ b/flake.nix @@ -3,6 +3,8 @@ nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; flake-parts.url = "github:hercules-ci/flake-parts"; + nix-std.url = "github:chessai/nix-std"; + nixos-hardware.url = "github:NixOS/nixos-hardware"; disko.url = "github:nix-community/disko"; disko.inputs.nixpkgs.follows = "nixpkgs"; diff --git a/home/apps.nix b/home/apps.nix index 7b2b733..3c95a7f 100644 --- a/home/apps.nix +++ b/home/apps.nix @@ -40,6 +40,7 @@ # Dev pkgs.ptyxis # Terminal emulator pkgs.vscodium + pkgs.godot_4 #pkgs.jetbrains.rider #pkgs.jetbrains.rust-rover #pkgs.helix diff --git a/nixos/default.nix b/nixos/default.nix index d93b0d9..ec804a6 100644 --- a/nixos/default.nix +++ b/nixos/default.nix @@ -26,6 +26,8 @@ ./user-system-config.nix ./vm.nix + (import ./kvmfr.nix { std = inputs.nix-std.lib; lib = lib; pkgs = pkgs; config = config; }) + ./libvirtd.nix ]; vfio = { @@ -34,6 +36,36 @@ applyACSpatch = true; }; + virtualisation.kvmfr = { + enable = true; + + devices = [ + { + size = 128; + + permissions = { + user = "evar"; + group = "qemu-libvirtd"; + mode = "0660"; + }; + } + ]; + }; + + # # https://gist.github.com/j-brn/716a03822d256bc5bf5d77b951c7915c + # virtualisation.kvmfr = { + # enable = true; + + # shm = { + # enable = true; + + # size = 64; + # user = "evar"; + # group = "libvirtd"; + # mode = "0660"; + # }; + # }; + # Allows referring to this flake by the shorthand `nixos-config`, which lets you do e.g. # nix repl nixos-config nix.registry.nixos-config.to = { diff --git a/nixos/kvmfr.nix b/nixos/kvmfr.nix new file mode 100644 index 0000000..aa70cfa --- /dev/null +++ b/nixos/kvmfr.nix @@ -0,0 +1,140 @@ +{ std, lib, pkgs, config, ... }: + +with lib; + +let + cfg = config.virtualisation.kvmfr; + + sizeFromResolution = resolution: + let + ceilToPowerOf2 = n: + std.num.pow 2 (std.num.bits.bitSize - std.num.bits.countLeadingZeros n); + pixelSize = if resolution.pixelFormat == "rgb24" then 3 else 4; + bytes = resolution.width * resolution.height * pixelSize * 2; + in ceilToPowerOf2 (bytes / 1024 / 1024 + 10); + + deviceSizes = map (device: device.size) cfg.devices; + + devices = + imap0 (index: _deviceConfig: "/dev/kvmfr${toString index}") cfg.devices; + + udevPackage = pkgs.writeTextDir "/lib/udev/rules.d/99-kvmfr.rules" + (concatStringsSep "\n" (imap0 (index: deviceConfig: '' + SUBSYSTEM=="kvmfr", KERNEL=="kvmfr${ + toString index + }", OWNER="${deviceConfig.permissions.user}", GROUP="${deviceConfig.permissions.group}", MODE="${deviceConfig.permissions.mode}", TAG+="systemd" + '') cfg.devices)); + + apparmorAbstraction = + concatStringsSep "\n" (map (device: "${device} rw") devices); + + permissionsType = types.submodule { + options = { + user = mkOption { + type = types.str; + default = "root"; + description = mdDoc "Owner of the shared memory device."; + }; + group = mkOption { + type = types.str; + default = "root"; + description = mdDoc "Group of the shared memory device."; + }; + mode = mkOption { + type = types.str; + default = "0600"; + description = mdDoc "Mode of the shared memory device."; + }; + }; + }; + + resolutionType = types.submodule { + options = { + width = mkOption { + type = types.number; + description = mdDoc + "Maximum horizontal video size that should be supported by this device."; + }; + + height = mkOption { + type = types.number; + description = mdDoc + "Maximum vertical video size that should be supported by this device."; + }; + + pixelFormat = mkOption { + type = types.enum [ "rgba32" "rgb24" ]; + description = mdDoc "Pixel format to use."; + default = "rgba32"; + }; + }; + }; + + deviceType = (types.submodule ({ config, options, ... }: { + options = { + resolution = mkOption { + type = types.nullOr resolutionType; + default = null; + description = mdDoc '' + Automatically calculate the minimum device size for a specific resolution. Overrides `size` if set. + ''; + }; + + size = mkOption { + type = types.number; + description = mdDoc '' + Size for the kvmfr device in megabytes. + ''; + }; + + permissions = mkOption { + type = permissionsType; + default = { }; + description = mdDoc "Permissions of the kvmfr device."; + }; + }; + + config = { + size = + mkIf (config.resolution != null) (sizeFromResolution config.resolution); + }; + })); +in { + options.virtualisation.kvmfr = { + enable = mkOption { + type = types.bool; + default = false; + description = mdDoc "Whether to enable the kvmfr kernel module."; + }; + + devices = mkOption { + type = types.listOf deviceType; + default = [ ]; + description = mdDoc "List of devices to create."; + }; + }; + + config = mkIf cfg.enable { + boot.extraModulePackages = with config.boot.kernelPackages; [ kvmfr ]; + services.udev.packages = optionals (cfg.devices != [ ]) [ udevPackage ]; + + environment.etc = { + "modules-load.d/kvmfr.conf".text = '' + kvmfr + ''; + + "modprobe.d/kvmfr.conf".text = '' + options kvmfr static_size_mb=${ + concatStringsSep "," (map (size: toString size) deviceSizes) + } + ''; + + "apparmor.d/local/abstractions/libvirt-qemu" = + mkIf config.security.apparmor.enable { + text = mkIf config.security.apparmor.enable apparmorAbstraction; + }; + }; + + virtualisation.libvirtd.deviceACL = devices; + }; +} \ No newline at end of file diff --git a/nixos/libvirtd.nix b/nixos/libvirtd.nix new file mode 100644 index 0000000..7348a69 --- /dev/null +++ b/nixos/libvirtd.nix @@ -0,0 +1,46 @@ +{ lib, pkgs, config, ... }: +with lib; +let + cfg = config.virtualisation.libvirtd; + + aclString = with lib.strings; + concatMapStringsSep '' + , + '' escapeNixString cfg.deviceACL; +in { + options.virtualisation.libvirtd = { + deviceACL = mkOption { + type = types.listOf types.str; + default = [ ]; + description = "allowed devices"; + }; + }; + + # All this was part of the rabbit hole of getting looking glass working properly + # This below also seems very important + # https://forum.level1techs.com/t/new-looking-glass-beta-7-release-candidate-1/208250 + # same here, wrt cpu max bits possibly preventing shared memory from working + # https://forum.level1techs.com/t/looking-glass-b6-and-b7-rc1-not-working-with-new-kernels/222134/7 + # https://www.kraxel.org/blog/2023/12/qemu-phys-bits/ + # https://libvirt.org/formatdomain.html#cpu-model-and-topology + # + # Needed to make sure to pass + # all these different /dev/'s, otherwise qemu won't be able to + # start properly. I'm not 100% on either where the user here + # got this list, nor which i actually *need*, but either way + # this was an immense help: + # https://forum.level1techs.com/t/solved-unable-to-connect-to-libvirt-qemu-system-after-changing-to-kernel-module/219006 + + # + config.virtualisation.libvirtd.qemu.verbatimConfig = '' + namespaces = [] + + cgroup_device_acl = [ + "/dev/null", "/dev/full", "/dev/zero", + "/dev/random", "/dev/urandom", + "/dev/ptmx", "/dev/kvm", + "/dev/userfaultfd", + ${aclString} + ] + ''; +} \ No newline at end of file diff --git a/nixos/vm.nix b/nixos/vm.nix index 612d735..4711bc6 100644 --- a/nixos/vm.nix +++ b/nixos/vm.nix @@ -6,8 +6,8 @@ }: let gpuIDs = [ - "1002:7480" # Graphics - "1002:ab30" # Audio + "1002:7480" # Graphics - IOMMU Group 15 / 03:00.0 + "1002:ab30" # Audio - IOMMU Group 16 / 03:00.1 ]; in { # https://astrid.tech/2022/09/22/0/nixos-gpu-vfio/ Was a huge gem to find with regard to getting this setup working. @@ -68,8 +68,12 @@ in { # These are needed, since I'm not currently trying to # reserve a static IP for the bridge interface - networking.interfaces.eth0.useDHCP = true; - networking.interfaces.winvm0.useDHCP = true; + networking.interfaces.winvm0 = { + useDHCP = false; + ipv4.addresses = [ + { address = "10.0.5.1"; prefixLength = 16; } + ]; + }; # Trying to ensure the bridge network doesn't cause us to wait # on boot @@ -101,22 +105,22 @@ in { # note that the VM needs 64 MB for the shmem in side the # xml for the full res of the FW laptop - systemd.tmpfiles.rules = [ - "f /dev/shm/scream 0660 evar qemu-libvirtd -" - "f /dev/shm/looking-glass 0660 evar qemu-libvirtd -" - ]; + # systemd.tmpfiles.rules = [ + # "f /dev/shm/scream 0660 evar qemu-libvirtd -" + # "f /dev/shm/looking-glass 0660 evar qemu-libvirtd -" + # ]; # service for hooking up scream for audio - systemd.user.services.scream-ivshmem = { - enable = true; - description = "Scream IVSHMEM"; - serviceConfig = { - ExecStart = "${pkgs.scream}/bin/scream-ivshmem-pulse /dev/shm/scream"; - Restart = "always"; - }; - wantedBy = [ "multi-user.target" ]; - requires = [ "pulseaudio.service" ]; - }; + # systemd.user.services.scream-ivshmem = { + # enable = true; + # description = "Scream IVSHMEM"; + # serviceConfig = { + # ExecStart = "${pkgs.scream}/bin/scream-ivshmem-pulse /dev/shm/scream"; + # Restart = "always"; + # }; + # wantedBy = [ "multi-user.target" ]; + # requires = [ "pulseaudio.service" ]; + # }; boot = { initrd.kernelModules = [ @@ -166,9 +170,10 @@ in { "use sendfile" = "yes"; #"max protocol" = "smb2"; + "interfaces" = "virbr0"; # note: localhost is the ipv6 localhost ::1 - "hosts allow" = "10. 127.0.0.1 localhost"; - "hosts deny" = "0.0.0.0/0"; + "hosts allow" = "192.168.122."; + # "hosts deny" = "0.0.0.0/0"; "guest account" = "nobody"; "map to guest" = "bad user"; }; @@ -194,5 +199,26 @@ in { networking.firewall.enable = true; networking.firewall.allowPing = true; + + # I got into a stuck state and couldn't start any vm's, whenever I did I got the following:a + # $ sudo cat /var/log/libvirt/qemu/win10.log + # 2025-01-26T04:41:57.245640Z qemu-system-x86_64: -chardev pty,id=charserial0: Failed to create PTY: Operation not permitted + # 2025-01-26 04:41:57.284+0000: shutting down, reason=failed + # + # After some searching, several sources stated that this is something with OVH + # and the workaround is as below. + # + # https://bugzilla.redhat.com/show_bug.cgi?id=1668713 + # https://www.linuxglobal.com/fixed-libvirtd-qemu-kvm-monitor-unexpectedly-closed-failed-create-chardev-live-migration-virsh-start/ + fileSystems."devpts" = { + device = "devpts"; + mountPoint = "/dev/pts"; + fsType = "devpts"; + noCheck = true; + options = [ + "gid=5" + "mode=620" + ]; + }; }; } \ No newline at end of file diff --git a/win10.xml b/win10.xml index 6db8aa3..431bb5e 100644 --- a/win10.xml +++ b/win10.xml @@ -1,4 +1,4 @@ - + win10 d5c9ecd5-418d-491a-a4f9-6b7c4a4df28d @@ -6,13 +6,13 @@ - 16777216 - 16777216 + 25165824 + 25165824 - 8 + 14 @@ -22,12 +22,18 @@ + + + + + + hvm /run/libvirt/nix-ovmf/OVMF_CODE.fd /var/lib/libvirt/qemu/nvram/win10_VARS.fd - + @@ -49,10 +55,15 @@ + + + + - + + @@ -83,22 +94,6 @@
- - - - - - -
- - - - - - - -
-
@@ -191,23 +186,23 @@ - + -
+
- - - - - -
-
+ + + + + +
+ @@ -216,6 +211,9 @@ + + +
@@ -224,6 +222,12 @@
+ +
+ + +
+ @@ -232,14 +236,12 @@ +