From c3fa24524573dad1076d6a37db8d3d893e84dd2b Mon Sep 17 00:00:00 2001 From: Fernando Rodrigues Date: Tue, 17 Sep 2024 20:13:28 +0000 Subject: [PATCH 1/2] nixos/xen: format with nixfmt-rfc-style Signed-off-by: Fernando Rodrigues --- nixos/modules/virtualisation/xen-dom0.nix | 408 ++++++++++++---------- 1 file changed, 233 insertions(+), 175 deletions(-) diff --git a/nixos/modules/virtualisation/xen-dom0.nix b/nixos/modules/virtualisation/xen-dom0.nix index 2fb8c6cd45664..56369089473a3 100644 --- a/nixos/modules/virtualisation/xen-dom0.nix +++ b/nixos/modules/virtualisation/xen-dom0.nix @@ -1,6 +1,11 @@ # Xen hypervisor (Dom0) support. -{ config, lib, pkgs, ... }: +{ + config, + lib, + pkgs, + ... +}: with lib; @@ -10,27 +15,41 @@ in { imports = [ - (mkRemovedOptionModule [ "virtualisation" "xen" "qemu" ] "You don't need this option anymore, it will work without it.") - (mkRenamedOptionModule [ "virtualisation" "xen" "qemu-package" ] [ "virtualisation" "xen" "package-qemu" ]) + (mkRemovedOptionModule [ + "virtualisation" + "xen" + "qemu" + ] "You don't need this option anymore, it will work without it.") + (mkRenamedOptionModule + [ + "virtualisation" + "xen" + "qemu-package" + ] + [ + "virtualisation" + "xen" + "package-qemu" + ] + ) ]; ###### interface options = { - virtualisation.xen.enable = - mkOption { - default = false; - type = types.bool; - description = '' - Setting this option enables the Xen hypervisor, a - virtualisation technology that allows multiple virtual - machines, known as *domains*, to run - concurrently on the physical machine. NixOS runs as the - privileged *Domain 0*. This option - requires a reboot to take effect. - ''; - }; + virtualisation.xen.enable = mkOption { + default = false; + type = types.bool; + description = '' + Setting this option enables the Xen hypervisor, a + virtualisation technology that allows multiple virtual + machines, known as *domains*, to run + concurrently on the physical machine. NixOS runs as the + privileged *Domain 0*. This option + requires a reboot to take effect. + ''; + }; virtualisation.xen.package = mkOption { type = types.package; @@ -39,7 +58,10 @@ in description = '' The package used for Xen binary. ''; - relatedPackages = [ "xen" "xen-light" ]; + relatedPackages = [ + "xen" + "xen-light" + ]; }; virtualisation.xen.package-qemu = mkOption { @@ -49,110 +71,109 @@ in description = '' The package with qemu binaries for dom0 qemu and xendomains. ''; - relatedPackages = [ "xen" - { name = "qemu_xen-light"; comment = "For use with pkgs.xen-light."; } - ]; + relatedPackages = [ + "xen" + { + name = "qemu_xen-light"; + comment = "For use with pkgs.xen-light."; + } + ]; }; - virtualisation.xen.bootParams = - mkOption { - default = []; - type = types.listOf types.str; - description = - '' - Parameters passed to the Xen hypervisor at boot time. - ''; - }; + virtualisation.xen.bootParams = mkOption { + default = [ ]; + type = types.listOf types.str; + description = '' + Parameters passed to the Xen hypervisor at boot time. + ''; + }; - virtualisation.xen.domain0MemorySize = - mkOption { - default = 0; - example = 512; - type = types.addCheck types.int (n: n >= 0); - description = - '' - Amount of memory (in MiB) allocated to Domain 0 on boot. - If set to 0, all memory is assigned to Domain 0. - ''; - }; + virtualisation.xen.domain0MemorySize = mkOption { + default = 0; + example = 512; + type = types.addCheck types.int (n: n >= 0); + description = '' + Amount of memory (in MiB) allocated to Domain 0 on boot. + If set to 0, all memory is assigned to Domain 0. + ''; + }; virtualisation.xen.bridge = { - name = mkOption { - default = "xenbr0"; - type = types.str; - description = '' - Name of bridge the Xen domUs connect to. - ''; - }; - - address = mkOption { - type = types.str; - default = "172.16.0.1"; - description = '' - IPv4 address of the bridge. - ''; - }; - - prefixLength = mkOption { - type = types.addCheck types.int (n: n >= 0 && n <= 32); - default = 16; - description = '' - Subnet mask of the bridge interface, specified as the number of - bits in the prefix (`24`). - A DHCP server will provide IP addresses for the whole, remaining - subnet. - ''; - }; - - forwardDns = mkOption { - type = types.bool; - default = false; - description = '' - If set to `true`, the DNS queries from the - hosts connected to the bridge will be forwarded to the DNS - servers specified in /etc/resolv.conf . - ''; - }; + name = mkOption { + default = "xenbr0"; + type = types.str; + description = '' + Name of bridge the Xen domUs connect to. + ''; + }; + address = mkOption { + type = types.str; + default = "172.16.0.1"; + description = '' + IPv4 address of the bridge. + ''; }; - virtualisation.xen.stored = - mkOption { - type = types.path; - description = - '' - Xen Store daemon to use. Defaults to oxenstored of the xen package. - ''; + prefixLength = mkOption { + type = types.addCheck types.int (n: n >= 0 && n <= 32); + default = 16; + description = '' + Subnet mask of the bridge interface, specified as the number of + bits in the prefix (`24`). + A DHCP server will provide IP addresses for the whole, remaining + subnet. + ''; }; + forwardDns = mkOption { + type = types.bool; + default = false; + description = '' + If set to `true`, the DNS queries from the + hosts connected to the bridge will be forwarded to the DNS + servers specified in /etc/resolv.conf . + ''; + }; + + }; + + virtualisation.xen.stored = mkOption { + type = types.path; + description = '' + Xen Store daemon to use. Defaults to oxenstored of the xen package. + ''; + }; + virtualisation.xen.domains = { - extraConfig = mkOption { - type = types.lines; - default = ""; - description = - '' - Options defined here will override the defaults for xendomains. - The default options can be seen in the file included from - /etc/default/xendomains. - ''; - }; + extraConfig = mkOption { + type = types.lines; + default = ""; + description = '' + Options defined here will override the defaults for xendomains. + The default options can be seen in the file included from + /etc/default/xendomains. + ''; }; + }; virtualisation.xen.trace = mkEnableOption "Xen tracing"; }; - ###### implementation config = mkIf cfg.enable { - assertions = [ { - assertion = pkgs.stdenv.isx86_64; - message = "Xen currently not supported on ${pkgs.stdenv.hostPlatform.system}"; - } { - assertion = config.boot.loader.grub.enable && (config.boot.loader.grub.efiSupport == false); - message = "Xen currently does not support EFI boot"; - } ]; + assertions = [ + { + assertion = pkgs.stdenv.isx86_64; + message = "Xen currently not supported on ${pkgs.stdenv.hostPlatform.system}"; + } + { + assertion = config.boot.loader.grub.enable && (config.boot.loader.grub.efiSupport == false); + message = "Xen currently does not support EFI boot"; + } + ]; virtualisation.xen.package = mkDefault pkgs.xen; virtualisation.xen.package-qemu = mkDefault pkgs.xen; @@ -160,13 +181,30 @@ in environment.systemPackages = [ cfg.package ]; - boot.kernelModules = - [ "xen-evtchn" "xen-gntdev" "xen-gntalloc" "xen-blkback" "xen-netback" - "xen-pciback" "evtchn" "gntdev" "netbk" "blkbk" "xen-scsibk" - "usbbk" "pciback" "xen-acpi-processor" "blktap2" "tun" "netxen_nic" - "xen_wdt" "xen-acpi-processor" "xen-privcmd" "xen-scsiback" - "xenfs" - ]; + boot.kernelModules = [ + "xen-evtchn" + "xen-gntdev" + "xen-gntalloc" + "xen-blkback" + "xen-netback" + "xen-pciback" + "evtchn" + "gntdev" + "netbk" + "blkbk" + "xen-scsibk" + "usbbk" + "pciback" + "xen-acpi-processor" + "blktap2" + "tun" + "netxen_nic" + "xen_wdt" + "xen-acpi-processor" + "xen-privcmd" + "xen-scsiback" + "xenfs" + ]; # The xenfs module is needed in system.activationScripts.xen, but # the modprobe command there fails silently. Include xenfs in the @@ -180,51 +218,51 @@ in # Increase the number of loopback devices from the default (8), # which is way too small because every VM virtual disk requires a # loopback device. - boot.extraModprobeConfig = - '' - options loop max_loop=64 - ''; + boot.extraModprobeConfig = '' + options loop max_loop=64 + ''; - virtualisation.xen.bootParams = [] ++ - optionals cfg.trace [ "loglvl=all" "guest_loglvl=all" ] ++ - optional (cfg.domain0MemorySize != 0) "dom0_mem=${toString cfg.domain0MemorySize}M"; - - system.extraSystemBuilderCmds = - '' - ln -s ${cfg.package}/boot/xen.gz $out/xen.gz - echo "${toString cfg.bootParams}" > $out/xen-params - ''; + virtualisation.xen.bootParams = + [ ] + ++ optionals cfg.trace [ + "loglvl=all" + "guest_loglvl=all" + ] + ++ optional (cfg.domain0MemorySize != 0) "dom0_mem=${toString cfg.domain0MemorySize}M"; + + system.extraSystemBuilderCmds = '' + ln -s ${cfg.package}/boot/xen.gz $out/xen.gz + echo "${toString cfg.bootParams}" > $out/xen-params + ''; # Mount the /proc/xen pseudo-filesystem. - system.activationScripts.xen = - '' - if [ -d /proc/xen ]; then - ${pkgs.kmod}/bin/modprobe xenfs 2> /dev/null - ${pkgs.util-linux}/bin/mountpoint -q /proc/xen || \ - ${pkgs.util-linux}/bin/mount -t xenfs none /proc/xen - fi - ''; + system.activationScripts.xen = '' + if [ -d /proc/xen ]; then + ${pkgs.kmod}/bin/modprobe xenfs 2> /dev/null + ${pkgs.util-linux}/bin/mountpoint -q /proc/xen || \ + ${pkgs.util-linux}/bin/mount -t xenfs none /proc/xen + fi + ''; # Domain 0 requires a pvops-enabled kernel. - system.requiredKernelConfig = with config.lib.kernelConfig; - [ (isYes "XEN") - (isYes "X86_IO_APIC") - (isYes "ACPI") - (isYes "XEN_DOM0") - (isYes "PCI_XEN") - (isYes "XEN_DEV_EVTCHN") - (isYes "XENFS") - (isYes "XEN_COMPAT_XENFS") - (isYes "XEN_SYS_HYPERVISOR") - (isYes "XEN_GNTDEV") - (isYes "XEN_BACKEND") - (isModule "XEN_NETDEV_BACKEND") - (isModule "XEN_BLKDEV_BACKEND") - (isModule "XEN_PCIDEV_BACKEND") - (isYes "XEN_BALLOON") - (isYes "XEN_SCRUB_PAGES") - ]; - + system.requiredKernelConfig = with config.lib.kernelConfig; [ + (isYes "XEN") + (isYes "X86_IO_APIC") + (isYes "ACPI") + (isYes "XEN_DOM0") + (isYes "PCI_XEN") + (isYes "XEN_DEV_EVTCHN") + (isYes "XENFS") + (isYes "XEN_COMPAT_XENFS") + (isYes "XEN_SYS_HYPERVISOR") + (isYes "XEN_GNTDEV") + (isYes "XEN_BACKEND") + (isModule "XEN_NETDEV_BACKEND") + (isModule "XEN_BLKDEV_BACKEND") + (isModule "XEN_PCIDEV_BACKEND") + (isYes "XEN_BALLOON") + (isYes "XEN_SCRUB_PAGES") + ]; environment.etc = { @@ -244,12 +282,18 @@ in # Xen provides udev rules. services.udev.packages = [ cfg.package ]; - services.udev.path = [ pkgs.bridge-utils pkgs.iproute2 ]; + services.udev.path = [ + pkgs.bridge-utils + pkgs.iproute2 + ]; systemd.services.xen-store = { description = "Xen Store Daemon"; wantedBy = [ "multi-user.target" ]; - after = [ "network.target" "xen-store.socket" ]; + after = [ + "network.target" + "xen-store.socket" + ]; requires = [ "xen-store.socket" ]; preStart = '' export XENSTORED_ROOTDIR="/var/lib/xenstored" @@ -259,19 +303,23 @@ in mkdir -p /var/log/xen # Running xl requires /var/log/xen and /var/lib/xen, mkdir -p /var/lib/xen # so we create them here unconditionally. grep -q control_d /proc/xen/capabilities - ''; - serviceConfig = if (builtins.compareVersions cfg.package.version "4.8" < 0) then - { ExecStart = '' - ${cfg.stored}${optionalString cfg.trace " -T /var/log/xen/xenstored-trace.log"} --no-fork + ''; + serviceConfig = + if (builtins.compareVersions cfg.package.version "4.8" < 0) then + { + ExecStart = '' + ${cfg.stored}${optionalString cfg.trace " -T /var/log/xen/xenstored-trace.log"} --no-fork ''; - } else { - ExecStart = '' - ${cfg.package}/etc/xen/scripts/launch-xenstore + } + else + { + ExecStart = '' + ${cfg.package}/etc/xen/scripts/launch-xenstore ''; - Type = "notify"; - RemainAfterExit = true; - NotifyAccess = "all"; - }; + Type = "notify"; + RemainAfterExit = true; + NotifyAccess = "all"; + }; postStart = '' ${optionalString (builtins.compareVersions cfg.package.version "4.8" < 0) '' time=0 @@ -290,21 +338,23 @@ in ''} echo "executing xen-init-dom0" ${cfg.package}/lib/xen/bin/xen-init-dom0 - ''; + ''; }; systemd.sockets.xen-store = { description = "XenStore Socket for userspace API"; wantedBy = [ "sockets.target" ]; socketConfig = { - ListenStream = [ "/var/run/xenstored/socket" "/var/run/xenstored/socket_ro" ]; + ListenStream = [ + "/var/run/xenstored/socket" + "/var/run/xenstored/socket_ro" + ]; SocketMode = "0660"; SocketUser = "root"; SocketGroup = "root"; }; }; - systemd.services.xen-console = { description = "Xen Console Daemon"; wantedBy = [ "multi-user.target" ]; @@ -314,17 +364,16 @@ in mkdir -p /var/run/xen ${optionalString cfg.trace "mkdir -p /var/log/xen"} grep -q control_d /proc/xen/capabilities - ''; + ''; serviceConfig = { ExecStart = '' ${cfg.package}/bin/xenconsoled\ ${optionalString ((builtins.compareVersions cfg.package.version "4.8" >= 0)) " -i"}\ ${optionalString cfg.trace " --log=all --log-dir=/var/log/xen"} - ''; + ''; }; }; - systemd.services.xen-qemu = { description = "Xen Qemu Daemon"; wantedBy = [ "multi-user.target" ]; @@ -334,21 +383,22 @@ in ${cfg.package-qemu}/${cfg.package-qemu.qemu-system-i386} \ -xen-attach -xen-domid 0 -name dom0 -M xenpv \ -nographic -monitor /dev/null -serial /dev/null -parallel /dev/null - ''; + ''; }; - systemd.services.xen-watchdog = { description = "Xen Watchdog Daemon"; wantedBy = [ "multi-user.target" ]; - after = [ "xen-qemu.service" "xen-domains.service" ]; + after = [ + "xen-qemu.service" + "xen-domains.service" + ]; serviceConfig.ExecStart = "${cfg.package}/bin/xenwatchdogd 30 15"; serviceConfig.Type = "forking"; serviceConfig.RestartSec = "1"; serviceConfig.Restart = "on-failure"; }; - systemd.services.xen-bridge = { description = "Xen bridge"; wantedBy = [ "multi-user.target" ]; @@ -429,19 +479,27 @@ in ''; }; - systemd.services.xen-domains = { description = "Xen domains - automatically starts, saves and restores Xen domains"; wantedBy = [ "multi-user.target" ]; - after = [ "xen-bridge.service" "xen-qemu.service" ]; - requires = [ "xen-bridge.service" "xen-qemu.service" ]; + after = [ + "xen-bridge.service" + "xen-qemu.service" + ]; + requires = [ + "xen-bridge.service" + "xen-qemu.service" + ]; ## To prevent a race between dhcpcd and xend's bridge setup script ## (which renames eth* to peth* and recreates eth* as a virtual ## device), start dhcpcd after xend. before = [ "dhcpd.service" ]; restartIfChanged = false; serviceConfig.RemainAfterExit = "yes"; - path = [ cfg.package cfg.package-qemu ]; + path = [ + cfg.package + cfg.package-qemu + ]; environment.XENDOM_CONFIG = "${cfg.package}/etc/sysconfig/xendomains"; preStart = "mkdir -p /var/lock/subsys -m 755"; serviceConfig.ExecStart = "${cfg.package}/etc/init.d/xendomains start"; From 9e5f77a3e27f725ef4b3555e47acc21c7dce43c0 Mon Sep 17 00:00:00 2001 From: Fernando Rodrigues Date: Tue, 17 Sep 2024 20:20:50 +0000 Subject: [PATCH 2/2] nixos/xen: refactor dom0 configuration - Cleans up downstream systemd units in favour of using upstream units. - Xen 4.18 on Nixpkgs now supports EFI booting, so we have an EFI boot builder here that runs after systemd-boot-builder.py. - Add more options for setting up dom0 resource limits. - Adds options for the declarative configuration of oxenstored. - Disables the automatic bridge configuration, as it was broken. - Drops legacy BIOS boot - Adds an EFI boot entry builder script. Signed-off-by: Fernando Rodrigues Co-authored-by: Yaroslav Bolyukin --- .../manual/release-notes/rl-2411.section.md | 10 + .../virtualisation/xen-boot-builder.sh | 165 +++ nixos/modules/virtualisation/xen-dom0.nix | 1130 +++++++++++------ 3 files changed, 920 insertions(+), 385 deletions(-) create mode 100755 nixos/modules/virtualisation/xen-boot-builder.sh diff --git a/nixos/doc/manual/release-notes/rl-2411.section.md b/nixos/doc/manual/release-notes/rl-2411.section.md index ab951ddb3c25b..cb6f29f438eb7 100644 --- a/nixos/doc/manual/release-notes/rl-2411.section.md +++ b/nixos/doc/manual/release-notes/rl-2411.section.md @@ -38,6 +38,16 @@ If you experience any issues, please report them. The original Perl script can still be used for now by setting `system.switch.enableNg` to `false`. +- The [Xen Hypervisor](https://xenproject.org) is once again available as a virtualisation option under [`virtualisation.xen`](#opt-virtualisation.xen.enable). + - This release includes Xen [4.17.5](https://wiki.xenproject.org/wiki/Xen_Project_4.17_Release_Notes), [4.18.3](https://wiki.xenproject.org/wiki/Xen_Project_4.18_Release_Notes) and [4.19.0](https://wiki.xenproject.org/wiki/Xen_Project_4.19_Release_Notes), as well as support for booting the hypervisor on EFI systems. + ::: {.warning} + Booting into Xen through a legacy BIOS bootloader or with the legacy script-based Stage 1 initrd have been **deprecated**. Only EFI booting and the new systemd-based Stage 1 initrd are supported. + ::: + - There are two flavours of Xen available by default: `xen`, which includes all built-in components, and `xen-slim`, which replaces the built-in components with their Nixpkgs equivalents. + - The `qemu-xen-traditional` component has been deprecated by upstream Xen, and is no longer available in any of the Xen packages. + - The OCaml-based Xen Store can now be configured using [`virtualisation.xen.store.settings`](#opt-virtualisation.xen.store.settings). + - The `virtualisation.xen.bridge` options have been deprecated in this release cycle. Users who need network bridges are encouraged to set up their own networking configurations. + ## New Modules {#sec-release-24.11-new-modules} - [TaskChampion Sync-Server](https://github.com/GothenburgBitFactory/taskchampion-sync-server), a [Taskwariror 3](https://taskwarrior.org/docs/upgrade-3/) sync server, replacing Taskwarrior 2's sync server named [`taskserver`](https://github.com/GothenburgBitFactory/taskserver). diff --git a/nixos/modules/virtualisation/xen-boot-builder.sh b/nixos/modules/virtualisation/xen-boot-builder.sh new file mode 100755 index 0000000000000..13e1a4e202432 --- /dev/null +++ b/nixos/modules/virtualisation/xen-boot-builder.sh @@ -0,0 +1,165 @@ +# This script is called by ./xen-dom0.nix to create the Xen boot entries. +# shellcheck shell=bash + +# Handle input argument and exit if the flag is invalid. See virtualisation.xen.efi.bootBuilderVerbosity below. +[[ $# -ne 1 ]] && echo -e "\e[1;31merror:\e[0m xenBootBuilder must be called with exactly one verbosity argument. See the \e[1;34mvirtualisation.xen.efi.bootBuilderVerbosity\e[0m option." && exit 1 +case "$1" in + "quiet") true ;; + "default" | "info") echo -n "Installing Xen Hypervisor boot entries..." ;; + "debug") echo -e "\e[1;34mxenBootBuilder:\e[0m called with the '$1' flag" ;; + *) + echo -e "\e[1;31merror:\e[0m xenBootBuilder was called with an invalid argument. See the \e[1;34mvirtualisation.xen.efi.bootBuilderVerbosity\e[0m option." + exit 2 + ;; +esac + +# Get the current Xen generations and store them in an array. This will be used +# for displaying the diff later, if xenBootBuilder was called with `info`. +# We also delete the current Xen entries here, as they'll be rebuilt later if +# the corresponding NixOS generation still exists. +mapfile -t preGenerations < <(find "$efiMountPoint"/loader/entries -type f -name 'xen-*.conf' | sort -V | sed 's_/loader/entries/nixos_/loader/entries/xen_g;s_^.*/xen_xen_g;s_.conf$__g') +if [ "$1" = "debug" ]; then + if ((${#preGenerations[@]} == 0)); then + echo -e "\e[1;34mxenBootBuilder:\e[0m no previous Xen entries." + else + echo -e "\e[1;34mxenBootBuilder:\e[0m deleting the following stale xen entries:" && for debugGen in "${preGenerations[@]}"; do echo " - $debugGen"; done + fi +fi + +# Cleanup all Xen entries. +rm -f "$efiMountPoint"/{loader/entries/xen-*.conf,efi/nixos/xen-*.efi} + +# Main array for storing which generations exist in $efiMountPoint after +# systemd-boot-builder.py builds the main entries. +mapfile -t gens < <(find "$efiMountPoint"/loader/entries -type f -name 'nixos-*.conf' | sort -V) +[ "$1" = "debug" ] && echo -e "\e[1;34mxenBootBuilder:\e[0m found the following NixOS boot entries:" && for debugGen in "${gens[@]}"; do echo " - $debugGen"; done + +# This is the main loop that installs the Xen entries. +for gen in "${gens[@]}"; do + + # We discover the path to Bootspec through the init attribute in the entries, + # as it is equivalent to $toplevel/init. + bootspecFile="$(sed -nr 's/^options init=(.*)\/init.*$/\1/p' "$gen")/boot.json" + [ "$1" = "debug" ] && echo -e "\e[1;34mxenBootBuilder:\e[0m processing bootspec file $bootspecFile" + + # We do nothing if the Bootspec for the current $gen does not contain the Xen + # extension, which is added as a configuration attribute below. + if grep -sq '"org.xenproject.bootspec.v1"' "$bootspecFile"; then + [ "$1" = "debug" ] && echo -e " \e[1;32msuccess:\e[0m found Xen entries in $gen." + + # TODO: Support DeviceTree booting. Xen has some special handling for DeviceTree + # attributes, which will need to be translated in a boot script similar to this + # one. Having a DeviceTree entry is rare, and it is not always required for a + # successful boot, so we don't fail here, only warn with `debug`. + if grep -sq '"devicetree"' "$bootspecFile"; then + echo -e "\n\e[1;33mwarning:\e[0m $gen has a \e[1;34morg.nixos.systemd-boot.devicetree\e[0m Bootspec entry. Xen currently does not support DeviceTree, so this value will be ignored in the Xen boot entries, which may cause them to \e[1;31mfail to boot\e[0m." + else + [ "$1" = "debug" ] && echo -e "\e[1;34mxenBootBuilder:\e[0m no DeviceTree entries found in $gen." + fi + + # Prepare required attributes for `xen.cfg/xen.conf`. It inherits the name of + # the corresponding nixos generation, substituting `nixos` with `xen`: + # `xen-$profile-generation-$number-specialisation-$specialisation.{cfg,conf}` + xenGen=$(echo "$gen" | sed 's_/loader/entries/nixos_/loader/entries/xen_g;s_^.*/xen_xen_g;s_.conf$__g') + bootParams=$(jq -re '."org.xenproject.bootspec.v1".xenParams | join(" ")' "$bootspecFile") + kernel=$(jq -re '."org.nixos.bootspec.v1".kernel | sub("^/nix/store/"; "") | sub("/bzImage"; "-bzImage.efi")' "$bootspecFile") + kernelParams=$(jq -re '."org.nixos.bootspec.v1".kernelParams | join(" ")' "$bootspecFile") + initrd=$(jq -re '."org.nixos.bootspec.v1".initrd | sub("^/nix/store/"; "") | sub("/initrd"; "-initrd.efi")' "$bootspecFile") + init=$(jq -re '."org.nixos.bootspec.v1".init' "$bootspecFile") + title=$(sed -nr 's/^title (.*)$/\1/p' "$gen") + version=$(sed -nr 's/^version (.*)$/\1/p' "$gen") + machineID=$(sed -nr 's/^machine-id (.*)$/\1/p' "$gen") + sortKey=$(sed -nr 's/^sort-key (.*)$/\1/p' "$gen") + + # Write `xen.cfg` to a temporary location prior to UKI creation. + tmpCfg=$(mktemp) + [ "$1" = "debug" ] && echo -ne "\e[1;34mxenBootBuilder:\e[0m writing $xenGen.cfg to temporary file..." + cat > "$tmpCfg" << EOF +[global] +default=xen + +[xen] +options=$bootParams +kernel=$kernel init=$init $kernelParams +ramdisk=$initrd +EOF + [ "$1" = "debug" ] && echo -e "done." + + # Create Xen UKI for $generation. Most of this is lifted from + # https://xenbits.xenproject.org/docs/unstable/misc/efi.html. + [ "$1" = "debug" ] && echo -e "\e[1;34mxenBootBuilder:\e[0m making Xen UKI..." + xenEfi=$(jq -re '."org.xenproject.bootspec.v1".xen' "$bootspecFile") + padding=$(objdump --header --section=".pad" "$xenEfi" | awk '/\.pad/ { printf("0x%016x\n", strtonum("0x"$3) + strtonum("0x"$4))};') + [ "$1" = "debug" ] && echo " - padding: $padding" + objcopy \ + --add-section .config="$tmpCfg" \ + --change-section-vma .config="$padding" \ + "$xenEfi" \ + "$efiMountPoint"/EFI/nixos/"$xenGen".efi + [ "$1" = "debug" ] && echo -e " - \e[1;32msuccessfully built\e[0m $xenGen.efi" + rm -f "$tmpCfg" + + # Write `xen.conf`. + [ "$1" = "debug" ] && echo -ne "\e[1;34mxenBootBuilder:\e[0m writing $xenGen.conf to EFI System Partition..." + cat > "$efiMountPoint"/loader/entries/"$xenGen".conf << EOF +title $title (with Xen Hypervisor) +version $version +efi /EFI/nixos/$xenGen.efi +machine-id $machineID +sort-key $sortKey +EOF + [ "$1" = "debug" ] && echo -e "done." + + # Sometimes, garbage collection weirdness causes a generation to still exist in + # the loader entries, but its Bootspec file was deleted. We consider such a + # generation to be invalid, but we don't write extra code to handle this + # situation, as supressing grep's error messages above is quite enough, and the + # error message below is still technically correct, as no Xen can be found in + # something that does not exist. + else + [ "$1" = "debug" ] && echo -e " \e[1;33mwarning:\e[0m \e[1;31mno Xen found\e[0m in $gen." + fi +done + +# Counterpart to the preGenerations array above. We use it to diff the +# generations created/deleted when callled with the `info` argument. +mapfile -t postGenerations < <(find "$efiMountPoint"/loader/entries -type f -name 'xen-*.conf' | sort -V | sed 's_/loader/entries/nixos_/loader/entries/xen_g;s_^.*/xen_xen_g;s_.conf$__g') + +# In the event the script does nothing, guide the user to debug, as it'll only +# ever run when Xen is enabled, and it makes no sense to enable Xen and not have +# any hypervisor boot entries. +if ((${#postGenerations[@]} == 0)); then + case "$1" in + "default" | "info") echo "none found." && echo -e "If you believe this is an error, set the \e[1;34mvirtualisation.xen.efi.bootBuilderVerbosity\e[0m option to \e[1;34m\"debug\"\e[0m and rebuild to print debug logs." ;; + "debug") echo -e "\e[1;34mxenBootBuilder:\e[0m wrote \e[1;31mno generations\e[0m. Most likely, there were no generations with a valid \e[1;34morg.xenproject.bootspec.v1\e[0m entry." ;; + esac + +# If the script is successful, change the default boot, say "done.", write a +# diff, or print the total files written, depending on the argument this script +# was called with. We use some dumb dependencies here, like `diff` or `bat` for +# colourisation, but they're only included with the `info` argument. +# +# It's also fine to change the default here, as this runs after the +# `systemd-boot-builder.py` script, which overwrites the file, and this script +# does not run after an user disables the Xen module. +else + sed --in-place 's/^default nixos-/default xen-/g' "$efiMountPoint"/loader/loader.conf + case "$1" in + "default" | "info") echo "done." ;; + "debug") echo -e "\e[1;34mxenBootBuilder:\e[0m \e[1;32msuccessfully wrote\e[0m the following generations:" && for debugGen in "${postGenerations[@]}"; do echo " - $debugGen"; done ;; + esac + if [ "$1" = "info" ]; then + if [[ ${#preGenerations[@]} == "${#postGenerations[@]}" ]]; then + echo -e "\e[1;33mNo Change:\e[0m Xen Hypervisor boot entries were refreshed, but their contents are identical." + else + echo -e "\e[1;32mSuccess:\e[0m Changed the following boot entries:" + # We briefly unset errexit and pipefail here, as GNU diff has no option to not fail when files differ. + set +o errexit + set +o pipefail + diff <(echo "${preGenerations[*]}" | tr ' ' '\n') <(echo "${postGenerations[*]}" | tr ' ' '\n') -U 0 | grep --invert-match --extended-regexp '^(@@|---|\+\+\+).*' | sed '1{/^-$/d}' | bat --language diff --theme ansi --paging=never --plain + true + set -o errexit + set -o pipefail + fi + fi +fi diff --git a/nixos/modules/virtualisation/xen-dom0.nix b/nixos/modules/virtualisation/xen-dom0.nix index 56369089473a3..db09710659001 100644 --- a/nixos/modules/virtualisation/xen-dom0.nix +++ b/nixos/modules/virtualisation/xen-dom0.nix @@ -7,148 +7,274 @@ ... }: -with lib; - let cfg = config.virtualisation.xen; + + xenBootBuilder = pkgs.writeShellApplication { + name = "xenBootBuilder"; + runtimeInputs = + (with pkgs; [ + binutils + coreutils + findutils + gawk + gnugrep + gnused + jq + ]) + ++ lib.lists.optionals (cfg.efi.bootBuilderVerbosity == "info") ( + with pkgs; + [ + bat + diffutils + ] + ); + runtimeEnv = { + efiMountPoint = config.boot.loader.efi.efiSysMountPoint; + }; + text = builtins.readFile ./xen-boot-builder.sh; + }; in { - imports = [ - (mkRemovedOptionModule [ - "virtualisation" - "xen" - "qemu" - ] "You don't need this option anymore, it will work without it.") + imports = with lib.modules; [ + (mkRemovedOptionModule + [ + "virtualisation" + "xen" + "bridge" + "name" + ] + "The Xen Network Bridge options are currently unavailable. Please set up your own bridge manually." + ) + (mkRemovedOptionModule + [ + "virtualisation" + "xen" + "bridge" + "address" + ] + "The Xen Network Bridge options are currently unavailable. Please set up your own bridge manually." + ) + (mkRemovedOptionModule + [ + "virtualisation" + "xen" + "bridge" + "prefixLength" + ] + "The Xen Network Bridge options are currently unavailable. Please set up your own bridge manually." + ) + (mkRemovedOptionModule + [ + "virtualisation" + "xen" + "bridge" + "forwardDns" + ] + "The Xen Network Bridge options are currently unavailable. Please set up your own bridge manually." + ) (mkRenamedOptionModule [ "virtualisation" "xen" "qemu-package" ] + [ + "virtualisation" + "xen" + "qemu" + "package" + ] + ) + (mkRenamedOptionModule [ "virtualisation" "xen" "package-qemu" ] + [ + "virtualisation" + "xen" + "qemu" + "package" + ] + ) + (mkRenamedOptionModule + [ + "virtualisation" + "xen" + "stored" + ] + [ + "virtualisation" + "xen" + "store" + "path" + ] ) ]; - ###### interface + ## Interface ## - options = { + options.virtualisation.xen = { - virtualisation.xen.enable = mkOption { - default = false; - type = types.bool; - description = '' - Setting this option enables the Xen hypervisor, a - virtualisation technology that allows multiple virtual - machines, known as *domains*, to run - concurrently on the physical machine. NixOS runs as the - privileged *Domain 0*. This option - requires a reboot to take effect. - ''; + enable = lib.options.mkEnableOption "the Xen Hypervisor, a virtualisation technology defined as a *type-1 hypervisor*, which allows multiple virtual machines, known as *domains*, to run concurrently on the physical machine. NixOS runs as the privileged *Domain 0*. This option requires a reboot into a Xen kernel to take effect"; + + debug = lib.options.mkEnableOption "Xen debug features for Domain 0. This option enables some hidden debugging tests and features, and should not be used in production"; + + trace = lib.options.mkOption { + type = lib.types.bool; + default = cfg.debug; + defaultText = lib.options.literalExpression "false"; + example = true; + description = "Whether to enable Xen debug tracing and logging for Domain 0."; }; - virtualisation.xen.package = mkOption { - type = types.package; - defaultText = literalExpression "pkgs.xen"; - example = literalExpression "pkgs.xen-light"; + package = lib.options.mkOption { + type = lib.types.package; + default = pkgs.xen; + defaultText = lib.options.literalExpression "pkgs.xen"; + example = lib.options.literalExpression "pkgs.xen-slim"; description = '' - The package used for Xen binary. + The package used for Xen Hypervisor. ''; relatedPackages = [ "xen" - "xen-light" + "xen-slim" ]; }; - virtualisation.xen.package-qemu = mkOption { - type = types.package; - defaultText = literalExpression "pkgs.xen"; - example = literalExpression "pkgs.qemu_xen-light"; - description = '' - The package with qemu binaries for dom0 qemu and xendomains. - ''; - relatedPackages = [ - "xen" - { - name = "qemu_xen-light"; - comment = "For use with pkgs.xen-light."; - } - ]; + qemu = { + package = lib.options.mkOption { + type = lib.types.package; + default = pkgs.xen; + defaultText = lib.options.literalExpression "pkgs.xen"; + example = lib.options.literalExpression "pkgs.qemu_xen"; + description = '' + The package with QEMU binaries that runs in Domain 0 + and virtualises the unprivileged domains. + ''; + relatedPackages = [ + "xen" + { + name = "qemu_xen"; + comment = "For use with `pkgs.xen-slim`."; + } + ]; + }; + pidFile = lib.options.mkOption { + type = lib.types.path; + default = "/run/xen/qemu-dom0.pid"; + example = "/var/run/xen/qemu-dom0.pid"; + description = "Path to the QEMU PID file."; + }; }; - virtualisation.xen.bootParams = mkOption { + bootParams = lib.options.mkOption { default = [ ]; - type = types.listOf types.str; - description = '' - Parameters passed to the Xen hypervisor at boot time. + example = '' + [ + "iommu=force:true,qinval:true,debug:true" + "noreboot=true" + "vga=ask" + ] ''; - }; - - virtualisation.xen.domain0MemorySize = mkOption { - default = 0; - example = 512; - type = types.addCheck types.int (n: n >= 0); + type = lib.types.listOf lib.types.str; description = '' - Amount of memory (in MiB) allocated to Domain 0 on boot. - If set to 0, all memory is assigned to Domain 0. + Xen Command Line parameters passed to Domain 0 at boot time. + Note: these are different from `boot.kernelParams`. See + the [Xen documentation](https://xenbits.xenproject.org/docs/unstable/misc/xen-command-line.html) for more information. ''; }; - virtualisation.xen.bridge = { - name = mkOption { - default = "xenbr0"; - type = types.str; + efi = { + bootBuilderVerbosity = lib.options.mkOption { + type = lib.types.enum [ + "default" + "info" + "debug" + "quiet" + ]; + default = "default"; + example = "info"; description = '' - Name of bridge the Xen domUs connect to. + The EFI boot entry builder script should be called with exactly one of the following arguments in order to specify its verbosity: + + - `quiet` supresses all messages. + + - `default` adds a simple "Installing Xen Hypervisor boot entries...done." message to the script. + + - `info` is the same as `default`, but it also prints a diff with information on which generations were altered. + - This option adds two extra dependencies to the script: `diffutils` and `bat`. + + - `debug` prints information messages for every single step of the script. + + This option does not alter the actual functionality of the script, just the number of messages printed when rebuilding the system. ''; }; - address = mkOption { - type = types.str; - default = "172.16.0.1"; + path = lib.options.mkOption { + type = lib.types.path; + default = "${cfg.package.boot}/${cfg.package.efi}"; + defaultText = lib.options.literalExpression "\${config.virtualisation.xen.package.boot}/\${config.virtualisation.xen.package.efi}"; + example = lib.options.literalExpression "\${config.virtualisation.xen.package}/boot/efi/efi/nixos/xen-\${config.virtualisation.xen.package.version}.efi"; description = '' - IPv4 address of the bridge. + Path to xen.efi. `pkgs.xen` is patched to install the xen.efi file + on `$boot/boot/xen.efi`, but an unpatched Xen build may install it + somewhere else, such as `$out/boot/efi/efi/nixos/xen.efi`. Unless + you're building your own Xen derivation, you should leave this + option as the default value. ''; }; + }; - prefixLength = mkOption { - type = types.addCheck types.int (n: n >= 0 && n <= 32); - default = 16; + dom0Resources = { + maxVCPUs = lib.options.mkOption { + default = 0; + example = 4; + type = lib.types.ints.unsigned; description = '' - Subnet mask of the bridge interface, specified as the number of - bits in the prefix (`24`). - A DHCP server will provide IP addresses for the whole, remaining - subnet. + Amount of virtual CPU cores allocated to Domain 0 on boot. + If set to 0, all cores are assigned to Domain 0, and + unprivileged domains will compete with Domain 0 for CPU time. ''; }; - forwardDns = mkOption { - type = types.bool; - default = false; + memory = lib.options.mkOption { + default = 0; + example = 512; + type = lib.types.ints.unsigned; description = '' - If set to `true`, the DNS queries from the - hosts connected to the bridge will be forwarded to the DNS - servers specified in /etc/resolv.conf . + Amount of memory (in MiB) allocated to Domain 0 on boot. + If set to 0, all memory is assigned to Domain 0, and + unprivileged domains will compete with Domain 0 for free RAM. ''; }; + maxMemory = lib.options.mkOption { + default = cfg.dom0Resources.memory; + defaultText = lib.options.literalExpression "config.virtualisation.xen.dom0Resources.memory"; + example = 1024; + type = lib.types.ints.unsigned; + description = '' + Maximum amount of memory (in MiB) that Domain 0 can + dynamically allocate to itself. Does nothing if set + to the same amount as virtualisation.xen.memory, or + if that option is set to 0. + ''; + }; }; - virtualisation.xen.stored = mkOption { - type = types.path; - description = '' - Xen Store daemon to use. Defaults to oxenstored of the xen package. - ''; - }; - - virtualisation.xen.domains = { - extraConfig = mkOption { - type = types.lines; + domains = { + extraConfig = lib.options.mkOption { + type = lib.types.lines; default = ""; + example = '' + XENDOMAINS_SAVE=/persist/xen/save + XENDOMAINS_RESTORE=false + XENDOMAINS_CREATE_USLEEP=10000000 + ''; description = '' Options defined here will override the defaults for xendomains. The default options can be seen in the file included from @@ -157,94 +283,426 @@ in }; }; - virtualisation.xen.trace = mkEnableOption "Xen tracing"; - + store = { + path = lib.options.mkOption { + type = lib.types.path; + default = "${cfg.package}/bin/oxenstored"; + defaultText = lib.options.literalExpression "\${config.virtualisation.xen.package}/bin/oxenstored"; + example = lib.options.literalExpression "\${config.virtualisation.xen.package}/bin/xenstored"; + description = '' + Path to the Xen Store Daemon. This option is useful to + switch between the legacy C-based Xen Store Daemon, and + the newer OCaml-based Xen Store Daemon, `oxenstored`. + ''; + }; + type = lib.options.mkOption { + type = lib.types.enum [ + "c" + "ocaml" + ]; + default = if (lib.strings.hasSuffix "oxenstored" cfg.store.path) then "ocaml" else "c"; + internal = true; + readOnly = true; + description = "Helper internal option that determines the type of the Xen Store Daemon based on cfg.store.path."; + }; + settings = lib.options.mkOption { + default = { }; + example = { + enableMerge = false; + quota.maxWatchEvents = 2048; + quota.enable = true; + conflict.maxHistorySeconds = 0.12; + conflict.burstLimit = 15.0; + xenstored.log.file = "/dev/null"; + xenstored.log.level = "info"; + }; + description = '' + The OCaml-based Xen Store Daemon configuration. This + option does nothing with the C-based `xenstored`. + ''; + type = lib.types.submodule { + options = { + pidFile = lib.options.mkOption { + default = "/run/xen/xenstored.pid"; + example = "/var/run/xen/xenstored.pid"; + type = lib.types.path; + description = "Path to the Xen Store Daemon PID file."; + }; + testEAGAIN = lib.options.mkOption { + default = cfg.debug; + defaultText = lib.options.literalExpression "config.virtualisation.xen.debug"; + example = true; + type = lib.types.bool; + visible = false; + description = "Randomly fail a transaction with EAGAIN. This option is used for debugging purposes only."; + }; + enableMerge = lib.options.mkOption { + default = true; + example = false; + type = lib.types.bool; + description = "Whether to enable transaction merge support."; + }; + conflict = { + burstLimit = lib.options.mkOption { + default = 5.0; + example = 15.0; + type = lib.types.addCheck ( + lib.types.float + // { + name = "nonnegativeFloat"; + description = "nonnegative floating point number, meaning >=0"; + descriptionClass = "nonRestrictiveClause"; + } + ) (n: n >= 0); + description = '' + Limits applied to domains whose writes cause other domains' transaction + commits to fail. Must include decimal point. + + The burst limit is the number of conflicts a domain can cause to + fail in a short period; this value is used for both the initial and + the maximum value of each domain's conflict-credit, which falls by + one point for each conflict caused, and when it reaches zero the + domain's requests are ignored. + ''; + }; + maxHistorySeconds = lib.options.mkOption { + default = 5.0e-2; + example = 1.0; + type = lib.types.addCheck ( + lib.types.float // { description = "nonnegative floating point number, meaning >=0"; } + ) (n: n >= 0); + description = '' + Limits applied to domains whose writes cause other domains' transaction + commits to fail. Must include decimal point. + + The conflict-credit is replenished over time: + one point is issued after each conflict.maxHistorySeconds, so this + is the minimum pause-time during which a domain will be ignored. + ''; + }; + rateLimitIsAggregate = lib.options.mkOption { + default = true; + example = false; + type = lib.types.bool; + description = '' + If the conflict.rateLimitIsAggregate option is `true`, then after each + tick one point of conflict-credit is given to just one domain: the + one at the front of the queue. If `false`, then after each tick each + domain gets a point of conflict-credit. + + In environments where it is known that every transaction will + involve a set of nodes that is writable by at most one other domain, + then it is safe to set this aggregate limit flag to `false` for better + performance. (This can be determined by considering the layout of + the xenstore tree and permissions, together with the content of the + transactions that require protection.) + + A transaction which involves a set of nodes which can be modified by + multiple other domains can suffer conflicts caused by any of those + domains, so the flag must be set to `true`. + ''; + }; + }; + perms = { + enable = lib.options.mkOption { + default = true; + example = false; + type = lib.types.bool; + description = "Whether to enable the node permission system."; + }; + enableWatch = lib.options.mkOption { + default = true; + example = false; + type = lib.types.bool; + description = '' + Whether to enable the watch permission system. + + When this is set to `true`, unprivileged guests can only get watch events + for xenstore entries that they would've been able to read. + + When this is set to `false`, unprivileged guests may get watch events + for xenstore entries that they cannot read. The watch event contains + only the entry name, not the value. + This restores behaviour prior to [XSA-115](https://xenbits.xenproject.org/xsa/advisory-115.html). + ''; + }; + }; + quota = { + enable = lib.options.mkOption { + default = true; + example = false; + type = lib.types.bool; + description = "Whether to enable the quota system."; + }; + maxEntity = lib.options.mkOption { + default = 1000; + example = 1024; + type = lib.types.ints.positive; + description = "Entity limit for transactions."; + }; + maxSize = lib.options.mkOption { + default = 2048; + example = 4096; + type = lib.types.ints.positive; + description = "Size limit for transactions."; + }; + maxWatch = lib.options.mkOption { + default = 100; + example = 256; + type = lib.types.ints.positive; + description = "Maximum number of watches by the Xenstore Watchdog."; + }; + transaction = lib.options.mkOption { + default = 10; + example = 50; + type = lib.types.ints.positive; + description = "Maximum number of transactions."; + }; + maxRequests = lib.options.mkOption { + default = 1024; + example = 1024; + type = lib.types.ints.positive; + description = "Maximum number of requests per transaction."; + }; + maxPath = lib.options.mkOption { + default = 1024; + example = 1024; + type = lib.types.ints.positive; + description = "Path limit for the quota system."; + }; + maxOutstanding = lib.options.mkOption { + default = 1024; + example = 1024; + type = lib.types.ints.positive; + description = "Maximum outstanding requests, i.e. in-flight requests / domain."; + }; + maxWatchEvents = lib.options.mkOption { + default = 1024; + example = 2048; + type = lib.types.ints.positive; + description = "Maximum number of outstanding watch events per watch."; + }; + }; + persistent = lib.options.mkOption { + default = false; + example = true; + type = lib.types.bool; + description = "Whether to activate the filed base backend."; + }; + xenstored = { + log = { + file = lib.options.mkOption { + default = "/var/log/xen/xenstored.log"; + example = "/dev/null"; + type = lib.types.path; + description = "Path to the Xen Store log file."; + }; + level = lib.options.mkOption { + default = if cfg.trace then "debug" else null; + defaultText = lib.options.literalExpression "if (config.virtualisation.xen.trace == true) then \"debug\" else null"; + example = "error"; + type = lib.types.nullOr ( + lib.types.enum [ + "debug" + "info" + "warn" + "error" + ] + ); + description = "Logging level for the Xen Store."; + }; + # The hidden options below have no upstream documentation whatsoever. + # The nb* options appear to alter the log rotation behaviour, and + # the specialOps option appears to affect the Xenbus logging logic. + nbFiles = lib.options.mkOption { + default = 10; + example = 16; + type = lib.types.int; + visible = false; + description = "Set `xenstored-log-nb-files`."; + }; + }; + accessLog = { + file = lib.options.mkOption { + default = "/var/log/xen/xenstored-access.log"; + example = "/var/log/security/xenstored-access.log"; + type = lib.types.path; + description = "Path to the Xen Store access log file."; + }; + nbLines = lib.options.mkOption { + default = 13215; + example = 16384; + type = lib.types.int; + visible = false; + description = "Set `access-log-nb-lines`."; + }; + nbChars = lib.options.mkOption { + default = 180; + example = 256; + type = lib.types.int; + visible = false; + description = "Set `acesss-log-nb-chars`."; + }; + specialOps = lib.options.mkOption { + default = false; + example = true; + type = lib.types.bool; + visible = false; + description = "Set `access-log-special-ops`."; + }; + }; + xenfs = { + kva = lib.options.mkOption { + default = "/proc/xen/xsd_kva"; + example = cfg.store.settings.xenstored.xenfs.kva; + type = lib.types.path; + visible = false; + description = '' + Path to the Xen Store Daemon KVA location inside the XenFS pseudo-filesystem. + While it is possible to alter this value, some drivers may be hardcoded to follow the default paths. + ''; + }; + port = lib.options.mkOption { + default = "/proc/xen/xsd_port"; + example = cfg.store.settings.xenstored.xenfs.port; + type = lib.types.path; + visible = false; + description = '' + Path to the Xen Store Daemon userspace port inside the XenFS pseudo-filesystem. + While it is possible to alter this value, some drivers may be hardcoded to follow the default paths. + ''; + }; + }; + }; + ringScanInterval = lib.options.mkOption { + default = 20; + example = 30; + type = lib.types.addCheck ( + lib.types.int + // { + name = "nonzeroInt"; + description = "nonzero signed integer, meaning !=0"; + descriptionClass = "nonRestrictiveClause"; + } + ) (n: n != 0); + description = '' + Perodic scanning for all the rings as a safenet for lazy clients. + Define the interval in seconds; set to a negative integer to disable. + ''; + }; + }; + }; + }; + }; }; - ###### implementation + ## Implementation ## - config = mkIf cfg.enable { + config = lib.modules.mkIf cfg.enable { assertions = [ { assertion = pkgs.stdenv.isx86_64; - message = "Xen currently not supported on ${pkgs.stdenv.hostPlatform.system}"; + message = "Xen is currently not supported on ${pkgs.stdenv.hostPlatform.system}."; } { - assertion = config.boot.loader.grub.enable && (config.boot.loader.grub.efiSupport == false); - message = "Xen currently does not support EFI boot"; + assertion = + config.boot.loader.systemd-boot.enable + || (config.boot ? lanzaboote) && config.boot.lanzaboote.enable; + message = "Xen only supports booting on systemd-boot or Lanzaboote."; + } + { + assertion = config.boot.initrd.systemd.enable; + message = "Xen does not support the legacy script-based Stage 1 initrd."; + } + { + assertion = cfg.dom0Resources.maxMemory >= cfg.dom0Resources.memory; + message = '' + You have allocated more memory to dom0 than virtualisation.xen.dom0Resources.maxMemory + allows for. Please increase the maximum memory limit, or decrease the default memory allocation. + ''; + } + { + assertion = cfg.debug -> cfg.trace; + message = "Xen's debugging features are enabled, but logging is disabled. This is most likely not what you want."; + } + { + assertion = cfg.store.settings.quota.maxWatchEvents >= cfg.store.settings.quota.maxOutstanding; + message = '' + Upstream Xen recommends that maxWatchEvents be equal to or greater than maxOutstanding, + in order to mitigate denial of service attacks from malicious frontends. + ''; } ]; - virtualisation.xen.package = mkDefault pkgs.xen; - virtualisation.xen.package-qemu = mkDefault pkgs.xen; - virtualisation.xen.stored = mkDefault "${cfg.package}/bin/oxenstored"; - - environment.systemPackages = [ cfg.package ]; - - boot.kernelModules = [ - "xen-evtchn" - "xen-gntdev" - "xen-gntalloc" - "xen-blkback" - "xen-netback" - "xen-pciback" - "evtchn" - "gntdev" - "netbk" - "blkbk" - "xen-scsibk" - "usbbk" - "pciback" - "xen-acpi-processor" - "blktap2" - "tun" - "netxen_nic" - "xen_wdt" - "xen-acpi-processor" - "xen-privcmd" - "xen-scsiback" - "xenfs" - ]; - - # The xenfs module is needed in system.activationScripts.xen, but - # the modprobe command there fails silently. Include xenfs in the - # initrd as a work around. - boot.initrd.kernelModules = [ "xenfs" ]; - - # The radeonfb kernel module causes the screen to go black as soon - # as it's loaded, so don't load it. - boot.blacklistedKernelModules = [ "radeonfb" ]; - - # Increase the number of loopback devices from the default (8), - # which is way too small because every VM virtual disk requires a - # loopback device. - boot.extraModprobeConfig = '' - options loop max_loop=64 - ''; - virtualisation.xen.bootParams = - [ ] - ++ optionals cfg.trace [ + lib.lists.optionals cfg.trace [ "loglvl=all" "guest_loglvl=all" ] - ++ optional (cfg.domain0MemorySize != 0) "dom0_mem=${toString cfg.domain0MemorySize}M"; - - system.extraSystemBuilderCmds = '' - ln -s ${cfg.package}/boot/xen.gz $out/xen.gz - echo "${toString cfg.bootParams}" > $out/xen-params - ''; - - # Mount the /proc/xen pseudo-filesystem. - system.activationScripts.xen = '' - if [ -d /proc/xen ]; then - ${pkgs.kmod}/bin/modprobe xenfs 2> /dev/null - ${pkgs.util-linux}/bin/mountpoint -q /proc/xen || \ - ${pkgs.util-linux}/bin/mount -t xenfs none /proc/xen - fi - ''; + ++ + lib.lists.optional (cfg.dom0Resources.memory != 0) + "dom0_mem=${toString cfg.dom0Resources.memory}M${ + lib.strings.optionalString ( + cfg.dom0Resources.memory != cfg.dom0Resources.maxMemory + ) ",max:${toString cfg.dom0Resources.maxMemory}M" + }" + ++ lib.lists.optional ( + cfg.dom0Resources.maxVCPUs != 0 + ) "dom0_max_vcpus=${toString cfg.dom0Resources.maxVCPUs}"; + + boot = { + kernelModules = [ + "xen-evtchn" + "xen-gntdev" + "xen-gntalloc" + "xen-blkback" + "xen-netback" + "xen-pciback" + "evtchn" + "gntdev" + "netbk" + "blkbk" + "xen-scsibk" + "usbbk" + "pciback" + "xen-acpi-processor" + "blktap2" + "tun" + "netxen_nic" + "xen_wdt" + "xen-acpi-processor" + "xen-privcmd" + "xen-scsiback" + "xenfs" + ]; + + # The xenfs module is needed to mount /proc/xen. + initrd.kernelModules = [ "xenfs" ]; + + # Increase the number of loopback devices from the default (8), + # which is way too small because every VM virtual disk requires a + # loopback device. + extraModprobeConfig = '' + options loop max_loop=64 + ''; + + # Xen Bootspec extension. This extension allows NixOS bootloaders to + # fetch the `xen.efi` path and access the `cfg.bootParams` option. + bootspec.extensions = { + "org.xenproject.bootspec.v1" = { + xen = cfg.efi.path; + xenParams = cfg.bootParams; + }; + }; + + # See the `xenBootBuilder` script in the main `let...in` statement of this file. + loader.systemd-boot.extraInstallCommands = '' + ${lib.meta.getExe xenBootBuilder} ${cfg.efi.bootBuilderVerbosity} + ''; + }; # Domain 0 requires a pvops-enabled kernel. + # All NixOS kernels come with this enabled by default; this is merely a sanity check. system.requiredKernelConfig = with config.lib.kernelConfig; [ (isYes "XEN") (isYes "X86_IO_APIC") @@ -264,247 +722,149 @@ in (isYes "XEN_SCRUB_PAGES") ]; - environment.etc = - { - "xen/xl.conf".source = "${cfg.package}/etc/xen/xl.conf"; - "xen/scripts".source = "${cfg.package}/etc/xen/scripts"; - "default/xendomains".text = '' - source ${cfg.package}/etc/default/xendomains - - ${cfg.domains.extraConfig} - ''; - } - // optionalAttrs (builtins.compareVersions cfg.package.version "4.10" >= 0) { - # in V 4.10 oxenstored requires /etc/xen/oxenstored.conf to start - "xen/oxenstored.conf".source = "${cfg.package}/etc/xen/oxenstored.conf"; - }; + environment = { + systemPackages = [ + cfg.package + cfg.qemu.package + ]; + etc = + # Set up Xen Domain 0 configuration files. + { + "xen/xl.conf".source = "${cfg.package}/etc/xen/xl.conf"; # TODO: Add options to configure xl.conf declaratively. It's worth considering making a new "xl value" type, as it could be reused to produce xl.cfg (domain definition) files. + "xen/scripts-xen" = { + source = "${cfg.package}/etc/xen/scripts/*"; + target = "xen/scripts"; + }; + "default/xencommons".text = '' + source ${cfg.package}/etc/default/xencommons + + XENSTORED="${cfg.store.path}" + QEMU_XEN="${cfg.qemu.package}/${cfg.qemu.package.qemu-system-i386}" + ${lib.strings.optionalString cfg.trace '' + XENSTORED_TRACE=yes + XENCONSOLED_TRACE=all + ''} + ''; + "default/xendomains".text = '' + source ${cfg.package}/etc/default/xendomains + + ${cfg.domains.extraConfig} + ''; + } + # The OCaml-based Xen Store Daemon requires /etc/xen/oxenstored.conf to start. + // lib.attrsets.optionalAttrs (cfg.store.type == "ocaml") { + "xen/oxenstored.conf".text = '' + pid-file = ${cfg.store.settings.pidFile} + test-eagain = ${lib.trivial.boolToString cfg.store.settings.testEAGAIN} + merge-activate = ${toString cfg.store.settings.enableMerge} + conflict-burst-limit = ${toString cfg.store.settings.conflict.burstLimit} + conflict-max-history-seconds = ${toString cfg.store.settings.conflict.maxHistorySeconds} + conflict-rate-limit-is-aggregate = ${toString cfg.store.settings.conflict.rateLimitIsAggregate} + perms-activate = ${toString cfg.store.settings.perms.enable} + perms-watch-activate = ${toString cfg.store.settings.perms.enableWatch} + quota-activate = ${toString cfg.store.settings.quota.enable} + quota-maxentity = ${toString cfg.store.settings.quota.maxEntity} + quota-maxsize = ${toString cfg.store.settings.quota.maxSize} + quota-maxwatch = ${toString cfg.store.settings.quota.maxWatch} + quota-transaction = ${toString cfg.store.settings.quota.transaction} + quota-maxrequests = ${toString cfg.store.settings.quota.maxRequests} + quota-path-max = ${toString cfg.store.settings.quota.maxPath} + quota-maxoutstanding = ${toString cfg.store.settings.quota.maxOutstanding} + quota-maxwatchevents = ${toString cfg.store.settings.quota.maxWatchEvents} + persistent = ${lib.trivial.boolToString cfg.store.settings.persistent} + xenstored-log-file = ${cfg.store.settings.xenstored.log.file} + xenstored-log-level = ${ + if isNull cfg.store.settings.xenstored.log.level then + "null" + else + cfg.store.settings.xenstored.log.level + } + xenstored-log-nb-files = ${toString cfg.store.settings.xenstored.log.nbFiles} + access-log-file = ${cfg.store.settings.xenstored.accessLog.file} + access-log-nb-lines = ${toString cfg.store.settings.xenstored.accessLog.nbLines} + acesss-log-nb-chars = ${toString cfg.store.settings.xenstored.accessLog.nbChars} + access-log-special-ops = ${lib.trivial.boolToString cfg.store.settings.xenstored.accessLog.specialOps} + ring-scan-interval = ${toString cfg.store.settings.ringScanInterval} + xenstored-kva = ${cfg.store.settings.xenstored.xenfs.kva} + xenstored-port = ${cfg.store.settings.xenstored.xenfs.port} + ''; + }; + }; # Xen provides udev rules. services.udev.packages = [ cfg.package ]; - services.udev.path = [ - pkgs.bridge-utils - pkgs.iproute2 - ]; + systemd = { + # Xen provides systemd units. + packages = [ cfg.package ]; - systemd.services.xen-store = { - description = "Xen Store Daemon"; - wantedBy = [ "multi-user.target" ]; - after = [ - "network.target" - "xen-store.socket" + mounts = [ + { + description = "Mount /proc/xen files"; + what = "xenfs"; + where = "/proc/xen"; + type = "xenfs"; + unitConfig = { + ConditionPathExists = "/proc/xen"; + RefuseManualStop = "true"; + }; + } ]; - requires = [ "xen-store.socket" ]; - preStart = '' - export XENSTORED_ROOTDIR="/var/lib/xenstored" - rm -f "$XENSTORED_ROOTDIR"/tdb* &>/dev/null - - mkdir -p /var/run - mkdir -p /var/log/xen # Running xl requires /var/log/xen and /var/lib/xen, - mkdir -p /var/lib/xen # so we create them here unconditionally. - grep -q control_d /proc/xen/capabilities - ''; - serviceConfig = - if (builtins.compareVersions cfg.package.version "4.8" < 0) then - { - ExecStart = '' - ${cfg.stored}${optionalString cfg.trace " -T /var/log/xen/xenstored-trace.log"} --no-fork - ''; - } - else - { + + services = { + + # While this service is installed by the `xen` package, it shouldn't be used in dom0. + xendriverdomain.enable = false; + + xenstored = { + wantedBy = [ "multi-user.target" ]; + preStart = '' + export XENSTORED_ROOTDIR="/var/lib/xenstored" + rm -f "$XENSTORED_ROOTDIR"/tdb* &>/dev/null + mkdir -p /var/{run,log,lib}/xen + ''; + }; + + xen-init-dom0 = { + restartIfChanged = false; + wantedBy = [ "multi-user.target" ]; + }; + + xen-qemu-dom0-disk-backend = { + wantedBy = [ "multi-user.target" ]; + serviceConfig = { + PIDFile = cfg.qemu.pidFile; ExecStart = '' - ${cfg.package}/etc/xen/scripts/launch-xenstore + ${cfg.qemu.package}/${cfg.qemu.package.qemu-system-i386} \ + -xen-domid 0 -xen-attach -name dom0 -nographic -M xenpv \ + -daemonize -monitor /dev/null -serial /dev/null -parallel \ + /dev/null -nodefaults -no-user-config -pidfile \ + ${cfg.qemu.pidFile} ''; - Type = "notify"; - RemainAfterExit = true; - NotifyAccess = "all"; }; - postStart = '' - ${optionalString (builtins.compareVersions cfg.package.version "4.8" < 0) '' - time=0 - timeout=30 - # Wait for xenstored to actually come up, timing out after 30 seconds - while [ $time -lt $timeout ] && ! `${cfg.package}/bin/xenstore-read -s / >/dev/null 2>&1` ; do - time=$(($time+1)) - sleep 1 - done - - # Exit if we timed out - if ! [ $time -lt $timeout ] ; then - echo "Could not start Xenstore Daemon" - exit 1 - fi - ''} - echo "executing xen-init-dom0" - ${cfg.package}/lib/xen/bin/xen-init-dom0 - ''; - }; + }; - systemd.sockets.xen-store = { - description = "XenStore Socket for userspace API"; - wantedBy = [ "sockets.target" ]; - socketConfig = { - ListenStream = [ - "/var/run/xenstored/socket" - "/var/run/xenstored/socket_ro" - ]; - SocketMode = "0660"; - SocketUser = "root"; - SocketGroup = "root"; - }; - }; + xenconsoled.wantedBy = [ "multi-user.target" ]; - systemd.services.xen-console = { - description = "Xen Console Daemon"; - wantedBy = [ "multi-user.target" ]; - after = [ "xen-store.service" ]; - requires = [ "xen-store.service" ]; - preStart = '' - mkdir -p /var/run/xen - ${optionalString cfg.trace "mkdir -p /var/log/xen"} - grep -q control_d /proc/xen/capabilities - ''; - serviceConfig = { - ExecStart = '' - ${cfg.package}/bin/xenconsoled\ - ${optionalString ((builtins.compareVersions cfg.package.version "4.8" >= 0)) " -i"}\ - ${optionalString cfg.trace " --log=all --log-dir=/var/log/xen"} - ''; + xen-watchdog = { + wantedBy = [ "multi-user.target" ]; + serviceConfig = { + RestartSec = "1"; + Restart = "on-failure"; + }; + }; + + xendomains = { + restartIfChanged = false; + path = [ + cfg.package + cfg.qemu.package + ]; + preStart = "mkdir -p /var/lock/subsys -m 755"; + wantedBy = [ "multi-user.target" ]; + }; }; }; - - systemd.services.xen-qemu = { - description = "Xen Qemu Daemon"; - wantedBy = [ "multi-user.target" ]; - after = [ "xen-console.service" ]; - requires = [ "xen-store.service" ]; - serviceConfig.ExecStart = '' - ${cfg.package-qemu}/${cfg.package-qemu.qemu-system-i386} \ - -xen-attach -xen-domid 0 -name dom0 -M xenpv \ - -nographic -monitor /dev/null -serial /dev/null -parallel /dev/null - ''; - }; - - systemd.services.xen-watchdog = { - description = "Xen Watchdog Daemon"; - wantedBy = [ "multi-user.target" ]; - after = [ - "xen-qemu.service" - "xen-domains.service" - ]; - serviceConfig.ExecStart = "${cfg.package}/bin/xenwatchdogd 30 15"; - serviceConfig.Type = "forking"; - serviceConfig.RestartSec = "1"; - serviceConfig.Restart = "on-failure"; - }; - - systemd.services.xen-bridge = { - description = "Xen bridge"; - wantedBy = [ "multi-user.target" ]; - before = [ "xen-domains.service" ]; - preStart = '' - mkdir -p /var/run/xen - touch /var/run/xen/dnsmasq.pid - touch /var/run/xen/dnsmasq.etherfile - touch /var/run/xen/dnsmasq.leasefile - - IFS='-' read -a data <<< `${pkgs.sipcalc}/bin/sipcalc ${cfg.bridge.address}/${toString cfg.bridge.prefixLength} | grep Usable\ range` - export XEN_BRIDGE_IP_RANGE_START="${"\${data[1]//[[:blank:]]/}"}" - export XEN_BRIDGE_IP_RANGE_END="${"\${data[2]//[[:blank:]]/}"}" - - IFS='-' read -a data <<< `${pkgs.sipcalc}/bin/sipcalc ${cfg.bridge.address}/${toString cfg.bridge.prefixLength} | grep Network\ address` - export XEN_BRIDGE_NETWORK_ADDRESS="${"\${data[1]//[[:blank:]]/}"}" - - IFS='-' read -a data <<< `${pkgs.sipcalc}/bin/sipcalc ${cfg.bridge.address}/${toString cfg.bridge.prefixLength} | grep Network\ mask` - export XEN_BRIDGE_NETMASK="${"\${data[1]//[[:blank:]]/}"}" - - echo "${cfg.bridge.address} host gw dns" > /var/run/xen/dnsmasq.hostsfile - - cat < /var/run/xen/dnsmasq.conf - no-daemon - pid-file=/var/run/xen/dnsmasq.pid - interface=${cfg.bridge.name} - except-interface=lo - bind-interfaces - auth-zone=xen.local,$XEN_BRIDGE_NETWORK_ADDRESS/${toString cfg.bridge.prefixLength} - domain=xen.local - addn-hosts=/var/run/xen/dnsmasq.hostsfile - expand-hosts - strict-order - no-hosts - bogus-priv - ${optionalString (!cfg.bridge.forwardDns) '' - no-resolv - no-poll - auth-server=dns.xen.local,${cfg.bridge.name} - ''} - filterwin2k - clear-on-reload - domain-needed - dhcp-hostsfile=/var/run/xen/dnsmasq.etherfile - dhcp-authoritative - dhcp-range=$XEN_BRIDGE_IP_RANGE_START,$XEN_BRIDGE_IP_RANGE_END - dhcp-no-override - no-ping - dhcp-leasefile=/var/run/xen/dnsmasq.leasefile - EOF - - # DHCP - ${pkgs.iptables}/bin/iptables -w -I INPUT -i ${cfg.bridge.name} -p tcp -s $XEN_BRIDGE_NETWORK_ADDRESS/${toString cfg.bridge.prefixLength} --sport 68 --dport 67 -j ACCEPT - ${pkgs.iptables}/bin/iptables -w -I INPUT -i ${cfg.bridge.name} -p udp -s $XEN_BRIDGE_NETWORK_ADDRESS/${toString cfg.bridge.prefixLength} --sport 68 --dport 67 -j ACCEPT - # DNS - ${pkgs.iptables}/bin/iptables -w -I INPUT -i ${cfg.bridge.name} -p tcp -d ${cfg.bridge.address} --dport 53 -m state --state NEW,ESTABLISHED -j ACCEPT - ${pkgs.iptables}/bin/iptables -w -I INPUT -i ${cfg.bridge.name} -p udp -d ${cfg.bridge.address} --dport 53 -m state --state NEW,ESTABLISHED -j ACCEPT - - ${pkgs.bridge-utils}/bin/brctl addbr ${cfg.bridge.name} - ${pkgs.inetutils}/bin/ifconfig ${cfg.bridge.name} ${cfg.bridge.address} - ${pkgs.inetutils}/bin/ifconfig ${cfg.bridge.name} netmask $XEN_BRIDGE_NETMASK - ${pkgs.inetutils}/bin/ifconfig ${cfg.bridge.name} up - ''; - serviceConfig.ExecStart = "${pkgs.dnsmasq}/bin/dnsmasq --conf-file=/var/run/xen/dnsmasq.conf"; - postStop = '' - IFS='-' read -a data <<< `${pkgs.sipcalc}/bin/sipcalc ${cfg.bridge.address}/${toString cfg.bridge.prefixLength} | grep Network\ address` - export XEN_BRIDGE_NETWORK_ADDRESS="${"\${data[1]//[[:blank:]]/}"}" - - ${pkgs.inetutils}/bin/ifconfig ${cfg.bridge.name} down - ${pkgs.bridge-utils}/bin/brctl delbr ${cfg.bridge.name} - - # DNS - ${pkgs.iptables}/bin/iptables -w -D INPUT -i ${cfg.bridge.name} -p udp -d ${cfg.bridge.address} --dport 53 -m state --state NEW,ESTABLISHED -j ACCEPT - ${pkgs.iptables}/bin/iptables -w -D INPUT -i ${cfg.bridge.name} -p tcp -d ${cfg.bridge.address} --dport 53 -m state --state NEW,ESTABLISHED -j ACCEPT - # DHCP - ${pkgs.iptables}/bin/iptables -w -D INPUT -i ${cfg.bridge.name} -p udp -s $XEN_BRIDGE_NETWORK_ADDRESS/${toString cfg.bridge.prefixLength} --sport 68 --dport 67 -j ACCEPT - ${pkgs.iptables}/bin/iptables -w -D INPUT -i ${cfg.bridge.name} -p tcp -s $XEN_BRIDGE_NETWORK_ADDRESS/${toString cfg.bridge.prefixLength} --sport 68 --dport 67 -j ACCEPT - ''; - }; - - systemd.services.xen-domains = { - description = "Xen domains - automatically starts, saves and restores Xen domains"; - wantedBy = [ "multi-user.target" ]; - after = [ - "xen-bridge.service" - "xen-qemu.service" - ]; - requires = [ - "xen-bridge.service" - "xen-qemu.service" - ]; - ## To prevent a race between dhcpcd and xend's bridge setup script - ## (which renames eth* to peth* and recreates eth* as a virtual - ## device), start dhcpcd after xend. - before = [ "dhcpd.service" ]; - restartIfChanged = false; - serviceConfig.RemainAfterExit = "yes"; - path = [ - cfg.package - cfg.package-qemu - ]; - environment.XENDOM_CONFIG = "${cfg.package}/etc/sysconfig/xendomains"; - preStart = "mkdir -p /var/lock/subsys -m 755"; - serviceConfig.ExecStart = "${cfg.package}/etc/init.d/xendomains start"; - serviceConfig.ExecStop = "${cfg.package}/etc/init.d/xendomains stop"; - }; - }; + meta.maintainers = with lib.maintainers; [ sigmasquadron ]; }