zfs-2.2.9 patch set

.github/ISSUE_TEMPLATE/feature_request.md

@@ -14,7 +14,7 @@ Please check our issue tracker before opening a new feature request.
 Filling out the following template will help other contributors better understand your proposed feature.
 -->
 
-### Describe the feature would like to see added to OpenZFS
+### Describe the feature you would like to see added to OpenZFS
 
 <!--
 Provide a clear and concise description of the feature.

.github/PULL_REQUEST_TEMPLATE.md

@@ -2,11 +2,6 @@
 
 <!--- Provide a general summary of your changes in the Title above -->
 
-<!---
-Documentation on ZFS Buildbot options can be found at
-https://openzfs.github.io/openzfs-docs/Developer%20Resources/Buildbot%20Options.html
--->
-
 ### Motivation and Context
 <!--- Why is this change required? What problem does it solve? -->
 <!--- If it fixes an open issue, please link to the issue here. -->

.github/codeql-cpp.yml

@@ -2,3 +2,4 @@ name: "Custom CodeQL Analysis"
 
 queries:
   - uses: ./.github/codeql/custom-queries/cpp/deprecatedFunctionUsage.ql
+  - uses: ./.github/codeql/custom-queries/cpp/dslDatasetHoldReleMismatch.ql

.github/codeql/custom-queries/cpp/dslDatasetHoldReleMismatch.ql

@@ -0,0 +1,34 @@
+/**
+ * @name Detect mismatched dsl_dataset_hold/_rele pairs
+ * @description Flags instances of issue #12014 where
+ *   - a dataset held with dsl_dataset_hold_obj() ends up in dsl_dataset_rele_flags(), or
+ *   - a dataset held with dsl_dataset_hold_obj_flags() ends up in dsl_dataset_rele().
+ * @kind problem
+ * @severity error
+ * @tags correctness
+ * @id cpp/dslDatasetHoldReleMismatch
+ */
+
+import cpp
+
+from Variable ds, Call holdCall, Call releCall, string message
+where
+    ds.getType().toString() = "dsl_dataset_t *" and
+    holdCall.getASuccessor*() = releCall and
+    (
+        (holdCall.getTarget().getName() = "dsl_dataset_hold_obj_flags" and
+         holdCall.getArgument(4).(AddressOfExpr).getOperand().(VariableAccess).getTarget() = ds and
+         releCall.getTarget().getName() = "dsl_dataset_rele" and
+         releCall.getArgument(0).(VariableAccess).getTarget() = ds and
+         message = "Held with dsl_dataset_hold_obj_flags but released with dsl_dataset_rele")
+        or
+        (holdCall.getTarget().getName() = "dsl_dataset_hold_obj" and
+         holdCall.getArgument(3).(AddressOfExpr).getOperand().(VariableAccess).getTarget() = ds and
+         releCall.getTarget().getName() = "dsl_dataset_rele_flags" and
+         releCall.getArgument(0).(VariableAccess).getTarget() = ds and
+         message = "Held with dsl_dataset_hold_obj but released with dsl_dataset_rele_flags")
+    )
+select releCall,
+       "Mismatched release: held with $@ but released with " + releCall.getTarget().getName() + " for dataset $@",
+       holdCall, holdCall.getTarget().getName(),
+       ds, ds.toString()

.github/workflows/scripts/generate-ci-type.py

@@ -65,7 +65,7 @@ def output_type(type, reason):
 
     # check last (HEAD) commit message
     last_commit_message_raw = subprocess.run([
-        'git', 'show', '-s', '--format=%B', 'HEAD'
+        'git', 'show', '-s', '--format=%B', head
     ], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 
     for line in last_commit_message_raw.stdout.decode().splitlines():

.github/workflows/scripts/qemu-1-setup.sh

@@ -6,6 +6,13 @@
 
 set -eu
 
+# We've been seeing this script take over 15min to run.  This may or
+# may not be normal.  Just to get a little more insight, print out
+# a message to stdout with the top running process, and do this every
+# 30 seconds.  We can delete this watchdog later once we get a better
+# handle on what the timeout value should be.
+(while [ 1 ] ; do sleep 30 && echo "[watchdog: $(ps -eo cmd --sort=-pcpu  | head -n 2 | tail -n 1)}')]"; done) &
+
 # install needed packages
 export DEBIAN_FRONTEND="noninteractive"
 sudo apt-get -y update
@@ -65,3 +72,6 @@ sudo zpool create -f -o ashift=12 zpool $SSD1 $SSD2 -O relatime=off \
 for i in /sys/block/s*/queue/scheduler; do
   echo "none" | sudo tee $i
 done
+
+# Kill off our watchdog
+kill $(jobs -p)

.github/workflows/scripts/qemu-2-start.sh

@@ -12,10 +12,10 @@ OS="$1"
 # OS variant (virt-install --os-variant list)
 OSv=$OS
 
-# compressed with .zst extension
-REPO="https://github.com/mcmilk/openzfs-freebsd-images"
-FREEBSD="$REPO/releases/download/v2025-04-13"
-URLzs=""
+# FreeBSD urls's
+FREEBSD_REL="https://download.freebsd.org/releases/CI-IMAGES"
+FREEBSD_SNAP="https://download.freebsd.org/snapshots/CI-IMAGES"
+URLxz=""
 
 # Ubuntu mirrors
 UBMIRROR="https://cloud-images.ubuntu.com"
@@ -25,6 +25,10 @@ UBMIRROR="https://cloud-images.ubuntu.com"
 # default nic model for vm's
 NIC="virtio"
 
+# additional options for virt-install
+OPTS[0]=""
+OPTS[1]=""
+
 case "$OS" in
   almalinux8)
     OSNAME="AlmaLinux 8"
@@ -61,6 +65,14 @@ case "$OS" in
     OSNAME="Debian 12"
     URL="https://cloud.debian.org/images/cloud/bookworm/latest/debian-12-generic-amd64.qcow2"
     ;;
+  debian13)
+    OSNAME="Debian 13"
+    # TODO: Overwrite OSv to debian13 for virt-install until it's added to osinfo
+    OSv="debian12"
+    URL="https://cloud.debian.org/images/cloud/trixie/latest/debian-13-generic-amd64.qcow2"
+    OPTS[0]="--boot"
+    OPTS[1]="uefi=on"
+    ;;
   fedora41)
     OSNAME="Fedora 41"
     OSv="fedora-unknown"
@@ -71,50 +83,56 @@ case "$OS" in
     OSv="fedora-unknown"
     URL="https://download.fedoraproject.org/pub/fedora/linux/releases/42/Cloud/x86_64/images/Fedora-Cloud-Base-Generic-42-1.1.x86_64.qcow2"
     ;;
-  freebsd13-4r)
-    OSNAME="FreeBSD 13.4-RELEASE"
-    OSv="freebsd13.0"
-    URLzs="$FREEBSD/amd64-freebsd-13.4-RELEASE.qcow2.zst"
-    BASH="/usr/local/bin/bash"
-    NIC="rtl8139"
-    ;;
   freebsd13-5r)
-    OSNAME="FreeBSD 13.5-RELEASE"
+    FreeBSD="13.5-RELEASE"
+    OSNAME="FreeBSD $FreeBSD"
     OSv="freebsd13.0"
-    URLzs="$FREEBSD/amd64-freebsd-13.5-RELEASE.qcow2.zst"
-    BASH="/usr/local/bin/bash"
+    URLxz="$FREEBSD_REL/$FreeBSD/amd64/Latest/FreeBSD-$FreeBSD-amd64-BASIC-CI.raw.xz"
+    KSRC="$FREEBSD_REL/../amd64/$FreeBSD/src.txz"
     NIC="rtl8139"
     ;;
-  freebsd14-1r)
-    OSNAME="FreeBSD 14.1-RELEASE"
+  freebsd14-2r)
+    FreeBSD="14.2-RELEASE"
+    OSNAME="FreeBSD $FreeBSD"
     OSv="freebsd14.0"
-    URLzs="$FREEBSD/amd64-freebsd-14.1-RELEASE.qcow2.zst"
-    BASH="/usr/local/bin/bash"
+    KSRC="$FREEBSD_REL/../amd64/$FreeBSD/src.txz"
+    URLxz="$FREEBSD_REL/$FreeBSD/amd64/Latest/FreeBSD-$FreeBSD-amd64-BASIC-CI.raw.xz"
     ;;
-  freebsd14-2r)
-    OSNAME="FreeBSD 14.2-RELEASE"
+  freebsd14-3r)
+    FreeBSD="14.3-RELEASE"
+    OSNAME="FreeBSD $FreeBSD"
     OSv="freebsd14.0"
-    URLzs="$FREEBSD/amd64-freebsd-14.2-RELEASE.qcow2.zst"
-    BASH="/usr/local/bin/bash"
+    URLxz="$FREEBSD_REL/$FreeBSD/amd64/Latest/FreeBSD-$FreeBSD-amd64-BASIC-CI.raw.xz"
+    KSRC="$FREEBSD_REL/../amd64/$FreeBSD/src.txz"
     ;;
   freebsd13-5s)
-    OSNAME="FreeBSD 13.5-STABLE"
+    FreeBSD="13.5-STABLE"
+    OSNAME="FreeBSD $FreeBSD"
     OSv="freebsd13.0"
-    URLzs="$FREEBSD/amd64-freebsd-13.5-STABLE.qcow2.zst"
-    BASH="/usr/local/bin/bash"
+    URLxz="$FREEBSD_SNAP/$FreeBSD/amd64/Latest/FreeBSD-$FreeBSD-amd64-BASIC-CI.raw.xz"
+    KSRC="$FREEBSD_SNAP/../amd64/$FreeBSD/src.txz"
     NIC="rtl8139"
     ;;
-  freebsd14-2s)
-    OSNAME="FreeBSD 14.2-STABLE"
+  freebsd14-3s)
+    FreeBSD="14.3-STABLE"
+    OSNAME="FreeBSD $FreeBSD"
     OSv="freebsd14.0"
-    URLzs="$FREEBSD/amd64-freebsd-14.2-STABLE.qcow2.zst"
-    BASH="/usr/local/bin/bash"
+    URLxz="$FREEBSD_SNAP/$FreeBSD/amd64/Latest/FreeBSD-$FreeBSD-amd64-BASIC-CI-ufs.raw.xz"
+    KSRC="$FREEBSD_SNAP/../amd64/$FreeBSD/src.txz"
     ;;
   freebsd15-0c)
-    OSNAME="FreeBSD 15.0-CURRENT"
+    FreeBSD="15.0-ALPHA4"
+    OSNAME="FreeBSD $FreeBSD"
+    OSv="freebsd14.0"
+    URLxz="$FREEBSD_SNAP/$FreeBSD/amd64/Latest/FreeBSD-$FreeBSD-amd64-BASIC-CI-ufs.raw.xz"
+    KSRC="$FREEBSD_SNAP/../amd64/$FreeBSD/src.txz"
+    ;;
+  freebsd16-0c)
+    FreeBSD="16.0-CURRENT"
+    OSNAME="FreeBSD $FreeBSD"
     OSv="freebsd14.0"
-    URLzs="$FREEBSD/amd64-freebsd-15.0-CURRENT.qcow2.zst"
-    BASH="/usr/local/bin/bash"
+    URLxz="$FREEBSD_SNAP/$FreeBSD/amd64/Latest/FreeBSD-$FreeBSD-amd64-BASIC-CI-ufs.raw.xz"
+    KSRC="$FREEBSD_SNAP/../amd64/$FreeBSD/src.txz"
     ;;
   tumbleweed)
     OSNAME="openSUSE Tumbleweed"
@@ -168,31 +186,37 @@ echo "CPU=\"$CPU\"" >> $ENV
 sudo mkdir -p "/mnt/tests"
 sudo chown -R $(whoami) /mnt/tests
 
+DISK="/dev/zvol/zpool/openzfs"
+sudo zfs create -ps -b 64k -V 80g zpool/openzfs
+while true; do test -b $DISK && break; sleep 1; done
+
 # we are downloading via axel, curl and wget are mostly slower and
 # require more return value checking
-IMG="/mnt/tests/cloudimg.qcow2"
-if [ ! -z "$URLzs" ]; then
-  echo "Loading image $URLzs ..."
-  time axel -q -o "$IMG.zst" "$URLzs"
-  zstd -q -d --rm "$IMG.zst"
+IMG="/mnt/tests/cloud-image"
+if [ ! -z "$URLxz" ]; then
+  echo "Loading $URLxz ..."
+  time axel -q -o "$IMG" "$URLxz"
+  echo "Loading $KSRC ..."
+  time axel -q -o ~/src.txz $KSRC
 else
-  echo "Loading image $URL ..."
+  echo "Loading $URL ..."
   time axel -q -o "$IMG" "$URL"
 fi
 
-DISK="/dev/zvol/zpool/openzfs"
-FORMAT="raw"
-sudo zfs create -ps -b 64k -V 80g zpool/openzfs
-while true; do test -b $DISK && break; sleep 1; done
 echo "Importing VM image to zvol..."
-sudo qemu-img dd -f qcow2 -O raw if=$IMG of=$DISK bs=4M
+if [ ! -z "$URLxz" ]; then
+  xzcat -T0 $IMG | sudo dd of=$DISK bs=4M
+else
+  sudo qemu-img dd -f qcow2 -O raw if=$IMG of=$DISK bs=4M
+fi
 rm -f $IMG
 
 PUBKEY=$(cat ~/.ssh/id_ed25519.pub)
-cat <<EOF > /tmp/user-data
+if [ ${OS:0:7} != "freebsd" ]; then
+  cat <<EOF > /tmp/user-data
 #cloud-config
 
-fqdn: $OS
+hostname: $OS
 
 users:
 - name: root
@@ -208,6 +232,19 @@ growpart:
   devices: ['/']
   ignore_growroot_disabled: false
 EOF
+else
+  cat <<EOF > /tmp/user-data
+#cloud-config
+
+hostname: $OS
+
+# minimized config without sudo for nuageinit of FreeBSD
+growpart:
+  mode: auto
+  devices: ['/']
+  ignore_growroot_disabled: false
+EOF
+fi
 
 sudo virsh net-update default add ip-dhcp-host \
   "<host mac='52:54:00:83:79:00' ip='192.168.122.10'/>" --live --config
@@ -223,15 +260,8 @@ sudo virt-install \
   --graphics none \
   --network bridge=virbr0,model=$NIC,mac='52:54:00:83:79:00' \
   --cloud-init user-data=/tmp/user-data \
-  --disk $DISK,bus=virtio,cache=none,format=$FORMAT,driver.discard=unmap \
-  --import --noautoconsole >/dev/null
-
-# enable KSM on Linux
-if [ ${OS:0:7} != "freebsd" ]; then
-  sudo virsh dommemstat --domain "openzfs" --period 5
-  sudo virsh node-memory-tune 100 50 1
-  echo 1 | sudo tee /sys/kernel/mm/ksm/run > /dev/null
-fi
+  --disk $DISK,bus=virtio,cache=none,format=raw,driver.discard=unmap \
+  --import --noautoconsole ${OPTS[0]} ${OPTS[1]} >/dev/null
 
 # Give the VMs hostnames so we don't have to refer to them with
 # hardcoded IP addresses.
@@ -252,3 +282,29 @@ StrictHostKeyChecking no
 # small timeout, used in while loops later
 ConnectTimeout 1
 EOF
+
+if [ ${OS:0:7} != "freebsd" ]; then
+  # enable KSM on Linux
+  sudo virsh dommemstat --domain "openzfs" --period 5
+  sudo virsh node-memory-tune 100 50 1
+  echo 1 | sudo tee /sys/kernel/mm/ksm/run > /dev/null
+else
+  # on FreeBSD we need some more init stuff, because of nuageinit
+  BASH="/usr/local/bin/bash"
+  while pidof /usr/bin/qemu-system-x86_64 >/dev/null; do
+    ssh 2>/dev/null root@vm0 "uname -a" && break
+  done
+  ssh root@vm0 "env IGNORE_OSVERSION=yes pkg install -y bash ca_root_nss git qemu-guest-agent python3 py311-cloud-init"
+  ssh root@vm0 "chsh -s $BASH root"
+  ssh root@vm0 'sysrc qemu_guest_agent_enable="YES"'
+  ssh root@vm0 'sysrc cloudinit_enable="YES"'
+  ssh root@vm0 "pw add user zfs -w no -s $BASH"
+  ssh root@vm0 'mkdir -p ~zfs/.ssh'
+  ssh root@vm0 'echo "zfs ALL=(ALL:ALL) NOPASSWD: ALL" >> /usr/local/etc/sudoers'
+  ssh root@vm0 'echo "PubkeyAuthentication yes" >> /etc/ssh/sshd_config'
+  scp ~/.ssh/id_ed25519.pub "root@vm0:~zfs/.ssh/authorized_keys"
+  ssh root@vm0 'chown -R zfs ~zfs'
+  ssh root@vm0 'service sshd restart'
+  scp ~/src.txz "root@vm0:/tmp/src.txz"
+  ssh root@vm0 'tar -C / -zxf /tmp/src.txz'
+fi

.github/workflows/scripts/qemu-3-deps-vm.sh

@@ -20,14 +20,15 @@ function archlinux() {
   sudo pacman -Sy --noconfirm base-devel bc cpio cryptsetup dhclient dkms \
     fakeroot fio gdb inetutils jq less linux linux-headers lsscsi nfs-utils \
     parted pax perf python-packaging python-setuptools qemu-guest-agent ksh \
-    samba sysstat rng-tools rsync wget xxhash
+    samba strace sysstat rng-tools rsync wget xxhash
   echo "##[endgroup]"
 }
 
 function debian() {
   export DEBIAN_FRONTEND="noninteractive"
 
   echo "##[group]Running apt-get update+upgrade"
+  sudo sed -i '/[[:alpha:]]-backports/d' /etc/apt/sources.list
   sudo apt-get update -y
   sudo apt-get upgrade -y
   echo "##[endgroup]"
@@ -40,9 +41,10 @@ function debian() {
     libelf-dev libffi-dev libmount-dev libpam0g-dev libselinux-dev libssl-dev \
     libtool libtool-bin libudev-dev libunwind-dev linux-headers-$(uname -r) \
     lsscsi nfs-kernel-server pamtester parted python3 python3-all-dev \
-    python3-cffi python3-dev python3-distlib python3-packaging \
+    python3-cffi python3-dev python3-distlib python3-packaging libtirpc-dev \
     python3-setuptools python3-sphinx qemu-guest-agent rng-tools rpm2cpio \
-    rsync samba sysstat uuid-dev watchdog wget xfslibs-dev  xxhash zlib1g-dev
+    rsync samba strace sysstat uuid-dev watchdog wget xfslibs-dev xxhash \
+    zlib1g-dev
   echo "##[endgroup]"
 }
 
@@ -51,7 +53,7 @@ function freebsd() {
 
   echo "##[group]Install Development Tools"
   sudo pkg install -y autoconf automake autotools base64 checkbashisms fio \
-    gdb gettext gettext-runtime git gmake gsed jq ksh93 lcov libtool lscpu \
+    gdb gettext gettext-runtime git gmake gsed jq ksh lcov libtool lscpu \
     pkgconf python python3 pamtester pamtester qemu-guest-agent rsync xxhash
   sudo pkg install -xy \
     '^samba4[[:digit:]]+$' \
@@ -86,8 +88,8 @@ function rhel() {
     libuuid-devel lsscsi mdadm nfs-utils openssl-devel pam-devel pamtester \
     parted perf python3 python3-cffi python3-devel python3-packaging \
     kernel-devel python3-setuptools qemu-guest-agent rng-tools rpcgen \
-    rpm-build rsync samba sysstat systemd watchdog wget xfsprogs-devel xxhash \
-    zlib-devel
+    rpm-build rsync samba strace sysstat systemd watchdog wget xfsprogs-devel \
+    xxhash zlib-devel
   echo "##[endgroup]"
 }
 
@@ -103,7 +105,7 @@ function install_fedora_experimental_kernel {
   our_version="$1"
   sudo dnf -y copr enable @kernel-vanilla/stable
   sudo dnf -y copr enable @kernel-vanilla/mainline
-  all="$(sudo dnf list --showduplicates kernel-*)"
+  all="$(sudo dnf list --showduplicates kernel-* python3-perf* perf* bpftool*)"
   echo "Available versions:"
   echo "$all"