From 5ee6467bd32a15bf76c56e1ebd823c64c6999f49 Mon Sep 17 00:00:00 2001
From: Moritz Sanft <58110325+msanft@users.noreply.github.com>
Date: Fri, 6 Sep 2024 16:06:15 +0200
Subject: [PATCH 1/3] nixos: add support for dm-verity

Co-authored-by: nikstur <nikstur@outlook.com>
Co-authored-by: WilliButz <willibutz@posteo.de>
---
 .../manual/release-notes/rl-2411.section.md   |  3 +
 nixos/modules/module-list.nix                 |  1 +
 .../modules/system/boot/systemd/dm-verity.nix | 61 +++++++++++++++++++
 3 files changed, 65 insertions(+)
 create mode 100644 nixos/modules/system/boot/systemd/dm-verity.nix

diff --git a/nixos/doc/manual/release-notes/rl-2411.section.md b/nixos/doc/manual/release-notes/rl-2411.section.md
index 6659fddf4cee1..805418f971a88 100644
--- a/nixos/doc/manual/release-notes/rl-2411.section.md
+++ b/nixos/doc/manual/release-notes/rl-2411.section.md
@@ -46,6 +46,9 @@
   If you experience any issues, please report them.
   The original Perl script can still be used for now by setting `system.switch.enableNg` to `false`.
 
+- Support for mounting filesystems from block devices protected with [dm-verity](https://docs.kernel.org/admin-guide/device-mapper/verity.html)
+  was added through the `boot.initrd.systemd.dmVerity` option.
+
 - The [Xen Hypervisor](https://xenproject.org) is once again available as a virtualisation option under [`virtualisation.xen`](#opt-virtualisation.xen.enable).
   - This release includes Xen [4.17.5](https://wiki.xenproject.org/wiki/Xen_Project_4.17_Release_Notes), [4.18.3](https://wiki.xenproject.org/wiki/Xen_Project_4.18_Release_Notes) and [4.19.0](https://wiki.xenproject.org/wiki/Xen_Project_4.19_Release_Notes), as well as support for booting the hypervisor on EFI systems.
   ::: {.warning}
diff --git a/nixos/modules/module-list.nix b/nixos/modules/module-list.nix
index 9de27a31067b4..97ea85cfb89ac 100644
--- a/nixos/modules/module-list.nix
+++ b/nixos/modules/module-list.nix
@@ -1625,6 +1625,7 @@
   ./system/boot/stage-2.nix
   ./system/boot/systemd.nix
   ./system/boot/systemd/coredump.nix
+  ./system/boot/systemd/dm-verity.nix
   ./system/boot/systemd/initrd-secrets.nix
   ./system/boot/systemd/initrd.nix
   ./system/boot/systemd/journald.nix
diff --git a/nixos/modules/system/boot/systemd/dm-verity.nix b/nixos/modules/system/boot/systemd/dm-verity.nix
new file mode 100644
index 0000000000000..d800777558013
--- /dev/null
+++ b/nixos/modules/system/boot/systemd/dm-verity.nix
@@ -0,0 +1,61 @@
+{ config, lib, ... }:
+
+let
+  cfg = config.boot.initrd.systemd.dmVerity;
+in
+{
+  options = {
+    boot.initrd.systemd.dmVerity = {
+      enable = lib.mkEnableOption "dm-verity" // {
+        description = ''
+          Mount verity-protected block devices in the initrd.
+
+          Enabling this option allows to use `systemd-veritysetup` and
+          `systemd-veritysetup-generator` in the initrd.
+        '';
+      };
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    assertions = [
+      {
+        assertion = config.boot.initrd.systemd.enable;
+        message = ''
+          'boot.initrd.systemd.dmVerity.enable' requires 'boot.initrd.systemd.enable' to be enabled.
+        '';
+      }
+    ];
+
+    boot.initrd = {
+      availableKernelModules = [
+        "dm_mod"
+        "dm_verity"
+      ];
+
+      # dm-verity needs additional udev rules from LVM to work.
+      services.lvm.enable = true;
+
+      # The additional targets and store paths allow users to integrate verity-protected devices
+      # through the systemd tooling.
+      systemd = {
+        additionalUpstreamUnits = [
+          "veritysetup-pre.target"
+          "veritysetup.target"
+          "remote-veritysetup.target"
+        ];
+
+        storePaths = [
+          "${config.boot.initrd.systemd.package}/lib/systemd/systemd-veritysetup"
+          "${config.boot.initrd.systemd.package}/lib/systemd/system-generators/systemd-veritysetup-generator"
+        ];
+      };
+    };
+  };
+
+  meta.maintainers = with lib.maintainers; [
+    msanft
+    nikstur
+    willibutz
+  ];
+}

From 942588c6866d55a252bdb3a7a0d4eef84162e987 Mon Sep 17 00:00:00 2001
From: WilliButz <willibutz@posteo.de>
Date: Wed, 18 Sep 2024 19:26:04 +0200
Subject: [PATCH 2/3] nixos/repart-verity-store: init

This module provides some abstraction for a multi-stage build to create
a dm-verity protected NixOS repart image.

The opinionated approach realized by this module is to first create an
immutable, verity-protected nix store partition, then embed the root
hash of the corresponding verity hash partition in a UKI, that is then
injected into the ESP of the resulting image.
The UKI can then precisely identify the corresponding data from which
the entire system is bootstrapped.

The module comes with a script that checks the UKI used in the final
image corresponds to the intermediate image created in the first step.
This is necessary to notice incompatible substitutions of
non-reproducible store paths, for example when working with distributed
builds, or when offline-signing the UKI.
---
 .../modules/image/assert_uki_repart_match.py  |  78 +++++++
 nixos/modules/image/repart-verity-store.nix   | 209 ++++++++++++++++++
 nixos/modules/image/repart.nix                |   4 +
 3 files changed, 291 insertions(+)
 create mode 100644 nixos/modules/image/assert_uki_repart_match.py
 create mode 100644 nixos/modules/image/repart-verity-store.nix

diff --git a/nixos/modules/image/assert_uki_repart_match.py b/nixos/modules/image/assert_uki_repart_match.py
new file mode 100644
index 0000000000000..e0f266cf15bfb
--- /dev/null
+++ b/nixos/modules/image/assert_uki_repart_match.py
@@ -0,0 +1,78 @@
+import json
+import sys
+
+store_verity_type = "@NIX_STORE_VERITY@"  # replaced at import by Nix
+
+
+def extract_uki_cmdline_params(ukify_json: dict) -> dict[str, str]:
+    """
+    Return a dict of the parameters in the .cmdline section of the UKI
+    Exits early if "usrhash" is not included.
+    """
+    cmdline = ukify_json.get(".cmdline", {}).get("text")
+    if cmdline is None:
+        print("Failed to get cmdline from ukify output")
+
+    params = {}
+    for param in cmdline.split():
+        key, val = param.partition("=")[::2]
+        params[key] = val
+
+    if "usrhash" not in params:
+        print(
+            f"UKI cmdline does not contain a usrhash:\n{cmdline}"
+        )
+        exit(1)
+
+    return params
+
+
+def hashes_match(partition: dict[str, str], expected: str) -> bool:
+    """
+    Checks if the value of the "roothash" key in the passed partition object matches `expected`.
+    """
+    if partition.get("roothash") != expected:
+        pretty_part = json.dumps(partition, indent=2)
+        print(
+            f"hash mismatch, expected to find roothash {expected} in:\n{pretty_part}"
+        )
+        return False
+    else:
+        return True
+
+
+def check_partitions(
+    partitions: list[dict], uki_params: dict[str, str]
+) -> bool:
+    """
+    Checks if the usrhash from `uki_params` has a matching roothash
+    for the corresponding partition in `partitions`.
+    """
+    for part in partitions:
+        if part.get("type") == store_verity_type:
+            expected = uki_params["usrhash"]
+            return hashes_match(part, expected)
+
+    return False
+
+
+def main() -> None:
+    ukify_json = json.load(sys.stdin)
+    repart_json_output = sys.argv[1]
+
+    with open(repart_json_output, "r") as r:
+        repart_json = json.load(r)
+
+    uki_params = extract_uki_cmdline_params(ukify_json)
+
+    if check_partitions(repart_json, uki_params):
+        print("UKI and repart verity hashes match")
+    else:
+        print("Compatibility check for UKI and image failed!")
+        print(f"UKI cmdline parameters:\n{uki_params}")
+        print(f"repart config: {repart_json_output}")
+        exit(1)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/nixos/modules/image/repart-verity-store.nix b/nixos/modules/image/repart-verity-store.nix
new file mode 100644
index 0000000000000..3f341ca421f26
--- /dev/null
+++ b/nixos/modules/image/repart-verity-store.nix
@@ -0,0 +1,209 @@
+# opinionated module that can be used to build nixos images with
+# a dm-verity protected nix store
+{
+  config,
+  pkgs,
+  lib,
+  ...
+}:
+let
+  cfg = config.image.repart.verityStore;
+
+  verityMatchKey = "store";
+
+  # TODO: make these and other arch mappings available from systemd-lib for example
+  partitionTypes = {
+    usr =
+      {
+        "x86_64" = "usr-x86-64";
+        "arm64" = "usr-arm64";
+      }
+      ."${pkgs.stdenv.hostPlatform.linuxArch}";
+
+    usr-verity =
+      {
+        "x86_64" = "usr-x86-64-verity";
+        "arm64" = "usr-arm64-verity";
+      }
+      ."${pkgs.stdenv.hostPlatform.linuxArch}";
+  };
+
+  verityHashCheck =
+    pkgs.buildPackages.writers.writePython3Bin "assert_uki_repart_match.py"
+      {
+        flakeIgnore = [ "E501" ]; # ignores PEP8's line length limit of 79 (black defaults to 88 characters)
+      }
+      (
+        builtins.replaceStrings [ "@NIX_STORE_VERITY@" ] [
+          partitionTypes.usr-verity
+        ] (builtins.readFile ./assert_uki_repart_match.py)
+      );
+in
+{
+  options.image.repart.verityStore = {
+    enable = lib.mkEnableOption "building images with a dm-verity protected nix store";
+
+    ukiPath = lib.mkOption {
+      type = lib.types.str;
+      default = "/EFI/Linux/${config.system.boot.loader.ukiFile}";
+      defaultText = "/EFI/Linux/\${config.system.boot.loader.ukiFile}";
+      description = ''
+        Specify the location on the ESP where the UKI is placed.
+      '';
+    };
+
+    partitionIds = {
+      esp = lib.mkOption {
+        type = lib.types.str;
+        default = "00-esp";
+        description = ''
+          Specify the attribute name of the ESP.
+        '';
+      };
+      store-verity = lib.mkOption {
+        type = lib.types.str;
+        default = "10-store-verity";
+        description = ''
+          Specify the attribute name of the store's dm-verity hash partition.
+        '';
+      };
+      store = lib.mkOption {
+        type = lib.types.str;
+        default = "20-store";
+        description = ''
+          Specify the attribute name of the store partition.
+        '';
+      };
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    boot.initrd.systemd.dmVerity.enable = true;
+
+    image.repart.partitions = {
+      # dm-verity hash partition
+      ${cfg.partitionIds.store-verity}.repartConfig = {
+        Type = partitionTypes.usr-verity;
+        Verity = "hash";
+        VerityMatchKey = lib.mkDefault verityMatchKey;
+        Label = lib.mkDefault "store-verity";
+      };
+      # dm-verity data partition that contains the nix store
+      ${cfg.partitionIds.store} = {
+        storePaths = [ config.system.build.toplevel ];
+        repartConfig = {
+          Type = partitionTypes.usr;
+          Verity = "data";
+          Format = lib.mkDefault "erofs";
+          VerityMatchKey = lib.mkDefault verityMatchKey;
+          Label = lib.mkDefault "store";
+        };
+      };
+
+    };
+
+    system.build = {
+
+      # intermediate system image without ESP
+      intermediateImage =
+        (config.system.build.image.override {
+          # always disable compression for the intermediate image
+          compression.enable = false;
+        }).overrideAttrs
+          (
+            _: previousAttrs: {
+              # make it easier to identify the intermediate image in build logs
+              pname = "${previousAttrs.pname}-intermediate";
+
+              # do not prepare the ESP, this is done in the final image
+              systemdRepartFlags = previousAttrs.systemdRepartFlags ++ [ "--defer-partitions=esp" ];
+
+              # the image will be self-contained so we can drop references
+              # to the closure that was used to build it
+              unsafeDiscardReferences.out = true;
+            }
+          );
+
+      # UKI with embedded usrhash from intermediateImage
+      uki =
+        let
+          inherit (config.system.boot.loader) ukiFile;
+          cmdline = "init=${config.system.build.toplevel}/init ${toString config.boot.kernelParams}";
+        in
+        # override the default UKI
+        lib.mkOverride 99 (
+          pkgs.runCommand ukiFile
+            {
+              nativeBuildInputs = [
+                pkgs.jq
+                pkgs.systemdUkify
+              ];
+            }
+            ''
+              mkdir -p $out
+
+              # Extract the usrhash from the output of the systemd-repart invocation for the intermediate image.
+              usrhash=$(jq -r \
+                '.[] | select(.type=="${partitionTypes.usr-verity}") | .roothash' \
+                ${config.system.build.intermediateImage}/repart-output.json
+              )
+
+              # Build UKI with the embedded usrhash.
+              ukify build \
+                  --config=${config.boot.uki.configFile} \
+                  --cmdline="${cmdline} usrhash=$usrhash" \
+                  --output="$out/${ukiFile}"
+            ''
+        );
+
+      # final system image that is created from the intermediate image by injecting the UKI from above
+      finalImage =
+        (config.system.build.image.override {
+          # continue building with existing intermediate image
+          createEmpty = false;
+        }).overrideAttrs
+          (
+            finalAttrs: previousAttrs:
+            let
+              copyUki = "CopyFiles=${config.system.build.uki}/${config.system.boot.loader.ukiFile}:${cfg.ukiPath}";
+            in
+            {
+              nativeBuildInputs = previousAttrs.nativeBuildInputs ++ [
+                pkgs.systemdUkify
+                verityHashCheck
+              ];
+
+              postPatch = ''
+                # add entry to inject UKI into ESP
+                echo '${copyUki}' >> $finalRepartDefinitions/${cfg.partitionIds.esp}.conf
+              '';
+
+              preBuild = ''
+                # check that we build the final image with the same intermediate image for
+                # which the injected UKI was built by comparing the UKI cmdline with the repart output
+                # of the intermediate image
+                #
+                # This is necessary to notice incompatible substitutions of
+                # non-reproducible store paths, for example when working with distributed
+                # builds, or when offline-signing the UKI.
+                ukify --json=short inspect ${config.system.build.uki}/${config.system.boot.loader.ukiFile} \
+                  | assert_uki_repart_match.py "${config.system.build.intermediateImage}/repart-output.json"
+
+                # copy the uncompressed intermediate image, so that systemd-repart picks it up
+                cp -v ${config.system.build.intermediateImage}/${config.image.repart.imageFileBasename}.raw .
+                chmod +w ${config.image.repart.imageFileBasename}.raw
+              '';
+
+              # the image will be self-contained so we can drop references
+              # to the closure that was used to build it
+              unsafeDiscardReferences.out = true;
+            }
+          );
+    };
+  };
+
+  meta.maintainers = with lib.maintainers; [
+    nikstur
+    willibutz
+  ];
+}
diff --git a/nixos/modules/image/repart.nix b/nixos/modules/image/repart.nix
index e471f9485cd03..b2f03d96ad35f 100644
--- a/nixos/modules/image/repart.nix
+++ b/nixos/modules/image/repart.nix
@@ -69,6 +69,10 @@ let
   }) opts;
 in
 {
+  imports = [
+    ./repart-verity-store.nix
+  ];
+
   options.image.repart = {
 
     name = lib.mkOption {

From 56d038e17dfb518937eb132b94b39bc25a1e513a Mon Sep 17 00:00:00 2001
From: WilliButz <willibutz@posteo.de>
Date: Wed, 18 Sep 2024 19:26:31 +0200
Subject: [PATCH 3/3] nixos/tests/appliance-repart-image-verity-store: init

This test should illustrate how to build a verity-protected NixOS image
with systemd-repart, using the opinionated image.repart.verityStore module.
---
 nixos/tests/all-tests.nix                     |   1 +
 .../appliance-repart-image-verity-store.nix   | 130 ++++++++++++++++++
 2 files changed, 131 insertions(+)
 create mode 100644 nixos/tests/appliance-repart-image-verity-store.nix

diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix
index 27d5b79c95c4f..0aff8978cb62c 100644
--- a/nixos/tests/all-tests.nix
+++ b/nixos/tests/all-tests.nix
@@ -128,6 +128,7 @@ in {
   apcupsd = handleTest ./apcupsd.nix {};
   apfs = runTest ./apfs.nix;
   appliance-repart-image = runTest ./appliance-repart-image.nix;
+  appliance-repart-image-verity-store = runTest ./appliance-repart-image-verity-store.nix;
   apparmor = handleTest ./apparmor.nix {};
   archi = handleTest ./archi.nix {};
   aria2 = handleTest ./aria2.nix {};
diff --git a/nixos/tests/appliance-repart-image-verity-store.nix b/nixos/tests/appliance-repart-image-verity-store.nix
new file mode 100644
index 0000000000000..3834d0a468ab3
--- /dev/null
+++ b/nixos/tests/appliance-repart-image-verity-store.nix
@@ -0,0 +1,130 @@
+# similar to the appliance-repart-image test but with a dm-verity
+# protected nix store and tmpfs as rootfs
+{ lib, ... }:
+
+{
+  name = "appliance-repart-image-verity-store";
+
+  meta.maintainers = with lib.maintainers; [
+    nikstur
+    willibutz
+  ];
+
+  nodes.machine =
+    {
+      config,
+      lib,
+      pkgs,
+      ...
+    }:
+    let
+      inherit (config.image.repart.verityStore) partitionIds;
+    in
+    {
+      imports = [ ../modules/image/repart.nix ];
+
+      virtualisation.fileSystems = lib.mkVMOverride {
+        "/" = {
+          fsType = "tmpfs";
+          options = [ "mode=0755" ];
+        };
+
+        "/usr" = {
+          device = "/dev/mapper/usr";
+          # explicitly mount it read-only otherwise systemd-remount-fs will fail
+          options = [ "ro" ];
+          fsType = config.image.repart.partitions.${partitionIds.store}.repartConfig.Format;
+        };
+
+        # bind-mount the store
+        "/nix/store" = {
+          device = "/usr/nix/store";
+          options = [ "bind" ];
+        };
+      };
+
+      image.repart = {
+        verityStore = {
+          enable = true;
+          # by default the module works with systemd-boot, for simplicity this test directly boots the UKI
+          ukiPath = "/EFI/BOOT/BOOT${lib.toUpper config.nixpkgs.hostPlatform.efiArch}.EFI";
+        };
+
+        name = "appliance-verity-store-image";
+
+        partitions = {
+          ${partitionIds.esp} = {
+            # the UKI is injected into this partition by the verityStore module
+            repartConfig = {
+              Type = "esp";
+              Format = "vfat";
+              SizeMinBytes = if config.nixpkgs.hostPlatform.isx86_64 then "64M" else "96M";
+            };
+          };
+          ${partitionIds.store-verity}.repartConfig = {
+            Minimize = "best";
+          };
+          ${partitionIds.store}.repartConfig = {
+            Minimize = "best";
+          };
+        };
+      };
+
+      virtualisation = {
+        directBoot.enable = false;
+        mountHostNixStore = false;
+        useEFIBoot = true;
+      };
+
+      boot = {
+        loader.grub.enable = false;
+        initrd.systemd.enable = true;
+      };
+
+      system.image = {
+        id = "nixos-appliance";
+        version = "1";
+      };
+
+      # don't create /usr/bin/env
+      # this would require some extra work on read-only /usr
+      # and it is not a strict necessity
+      system.activationScripts.usrbinenv = lib.mkForce "";
+    };
+
+  testScript =
+    { nodes, ... }: # python
+    ''
+      import os
+      import subprocess
+      import tempfile
+
+      tmp_disk_image = tempfile.NamedTemporaryFile()
+
+      subprocess.run([
+        "${nodes.machine.virtualisation.qemu.package}/bin/qemu-img",
+        "create",
+        "-f",
+        "qcow2",
+        "-b",
+        "${nodes.machine.system.build.finalImage}/${nodes.machine.image.repart.imageFile}",
+        "-F",
+        "raw",
+        tmp_disk_image.name,
+      ])
+
+      os.environ['NIX_DISK_IMAGE'] = tmp_disk_image.name
+
+      machine.wait_for_unit("default.target")
+
+      with subtest("Running with volatile root"):
+        machine.succeed("findmnt --kernel --type tmpfs /")
+
+      with subtest("/nix/store is backed by dm-verity protected fs"):
+        verity_info = machine.succeed("dmsetup info --target verity usr")
+        assert "ACTIVE" in verity_info,f"unexpected verity info: {verity_info}"
+
+        backing_device = machine.succeed("df --output=source /nix/store | tail -n1").strip()
+        assert "/dev/mapper/usr" == backing_device,"unexpected backing device: {backing_device}"
+    '';
+}