Add a diagnostic kstat for obtaining pool status

This kstat output does not require taking the spa_namespace
lock, as in the case for 'zpool status'. It can be used for
investigations when pools are in a hung state while holding
global locks required for a traditional 'zpool status' to
proceed.

This kstat is not safe to use in conditions where pools are
in the process of configuration changes (i.e., adding/removing
devices).  Therefore, this kstat is not intended to be a general
replacement or alternative to using 'zpool status'.

Sponsored-by: Wasabi Technology, Inc.
Sponsored-By: Klara Inc.

Co-authored-by: Don Brady <[email protected]>
Signed-off-by: Don Brady <[email protected]>

include/Makefile.am

@@ -176,6 +176,8 @@ COMMON_H = \
 
 
 KERNEL_H = \
+	sys/jprint.h \
+	sys/spa_json_stats.h \
 	sys/zfs_ioctl.h \
 	sys/zfs_ioctl_impl.h \
 	sys/zfs_onexit.h \

include/os/freebsd/spl/sys/kstat.h

@@ -67,8 +67,8 @@ struct list_head {};
 #define	KSTAT_FLAG_VIRTUAL	0x01
 #define	KSTAT_FLAG_VAR_SIZE	0x02
 #define	KSTAT_FLAG_WRITABLE	0x04
-#define	KSTAT_FLAG_PERSISTENT	0x08
-#define	KSTAT_FLAG_DORMANT	0x10
+#define	KSTAT_FLAG_RESTRICTED	0x08
+#define	KSTAT_FLAG_UNUSED	0x10
 #define	KSTAT_FLAG_INVALID	0x20
 #define	KSTAT_FLAG_LONGSTRINGS	0x40
 #define	KSTAT_FLAG_NO_HEADERS	0x80

include/os/linux/spl/sys/kstat.h

@@ -66,8 +66,8 @@
 #define	KSTAT_FLAG_VIRTUAL	0x01
 #define	KSTAT_FLAG_VAR_SIZE	0x02
 #define	KSTAT_FLAG_WRITABLE	0x04
-#define	KSTAT_FLAG_PERSISTENT	0x08
-#define	KSTAT_FLAG_DORMANT	0x10
+#define	KSTAT_FLAG_RESTRICTED	0x08
+#define	KSTAT_FLAG_UNUSED	0x10
 #define	KSTAT_FLAG_INVALID	0x20
 #define	KSTAT_FLAG_LONGSTRINGS	0x40
 #define	KSTAT_FLAG_NO_HEADERS	0x80

include/sys/jprint.h

@@ -0,0 +1,70 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://opensource.org/licenses/CDDL-1.0.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright (c) 2024, Klara Inc.
+ */
+
+#ifndef _SYS_JPRINT_H
+#define	_SYS_JPRINT_H
+
+/* maximum stack nesting */
+#define	JP_MAX_STACK	32
+
+enum jp_type {
+	JP_OBJECT = 1,
+	JP_ARRAY
+};
+
+struct jp_stack {
+	enum jp_type type;
+	int nelem;
+};
+
+typedef struct jprint {
+	char *buffer;		/* pointer to application's buffer */
+	size_t buflen;		/* length of buffer */
+	char *bufp;		/* current write position in buffer */
+	char tmpbuf[32];	/* local buffer for conversions */
+	int error;		/* error code */
+	int ncall;		/* API call number on which error occurred */
+	struct jp_stack		/* stack of array/object nodes */
+	    stack[JP_MAX_STACK];
+	int stackp;
+} jprint_t;
+
+/* error return codes */
+#define	JPRINT_OK		0	/* no error */
+#define	JPRINT_BUF_FULL		1	/* output buffer full */
+#define	JPRINT_NEST_ERROR	2	/* nesting error */
+#define	JPRINT_STACK_FULL	3	/* array/object nesting  */
+#define	JPRINT_STACK_EMPTY	4	/* stack underflow error */
+#define	JPRINT_OPEN		5	/* not all objects closed */
+#define	JPRINT_FMT		6	/* format error */
+
+const char *jp_errorstring(int err);
+int jp_error(jprint_t *jp);
+void jp_open(jprint_t *jp, char *buffer, size_t buflen);
+int jp_close(jprint_t *jp);
+int jp_errorpos(jprint_t *jp);
+int jp_printf(jprint_t *jp, const char *fmt, ...);
+
+#endif	/* _SYS_JPRINT_H */

include/sys/spa.h

@@ -771,6 +771,9 @@ extern void spa_scan_stat_init(spa_t *spa);
 extern int spa_scan_get_stats(spa_t *spa, pool_scan_stat_t *ps);
 extern int bpobj_enqueue_alloc_cb(void *arg, const blkptr_t *bp, dmu_tx_t *tx);
 extern int bpobj_enqueue_free_cb(void *arg, const blkptr_t *bp, dmu_tx_t *tx);
+extern void spa_add_spares(spa_t *spa, nvlist_t *config);
+extern void spa_add_l2cache(spa_t *spa, nvlist_t *config);
+extern void spa_add_feature_stats(spa_t *spa, nvlist_t *config);
 
 #define	SPA_ASYNC_CONFIG_UPDATE			0x01
 #define	SPA_ASYNC_REMOVE			0x02

include/sys/spa_impl.h

@@ -35,6 +35,7 @@
 #include <sys/spa.h>
 #include <sys/spa_checkpoint.h>
 #include <sys/spa_log_spacemap.h>
+#include <sys/spa_json_stats.h>
 #include <sys/vdev.h>
 #include <sys/vdev_rebuild.h>
 #include <sys/vdev_removal.h>
@@ -442,6 +443,7 @@ struct spa {
 	uint64_t	spa_autotrim;		/* automatic background trim? */
 	uint64_t	spa_errata;		/* errata issues detected */
 	spa_stats_t	spa_stats;		/* assorted spa statistics */
+	spa_json_stats_t spa_json_stats;	/* diagnostic status in JSON */
 	spa_keystore_t	spa_keystore;		/* loaded crypto keys */
 
 	/* arc_memory_throttle() parameters during low memory condition */

include/sys/spa_json_stats.h

@@ -0,0 +1,40 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://opensource.org/licenses/CDDL-1.0.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright (c) 2024, Klara Inc.
+ */
+
+#ifndef _SYS_SPA_JSON_STATS_H
+#define	_SYS_SPA_JSON_STATS_H
+
+#include <sys/zfs_context.h>
+#include <sys/spa_impl.h>
+#include <sys/kstat.h>
+
+typedef struct spa_json_stats {
+	kmutex_t	lock;
+	kstat_t		*kstat;
+} spa_json_stats_t;
+
+extern int spa_generate_json_stats(spa_t *spa, char *buf, size_t size);
+
+#endif	/* _SYS_SPA_JSON_STATS_H */

lib/libspl/include/sys/kstat.h

@@ -239,24 +239,13 @@ typedef struct kstat {
  *		The ks_snapshot routine (see below) does not need to check for
  *		this; permission checking is handled in the kstat driver.
  *
- *	KSTAT_FLAG_PERSISTENT:
+ *	KSTAT_FLAG_RESTRICTED:
  *
- *		Indicates that this kstat is to be persistent over time.
- *		For persistent kstats, kstat_delete() simply marks the
- *		kstat as dormant; a subsequent kstat_create() reactivates
- *		the kstat.  This feature is provided so that statistics
- *		are not lost across driver close/open (e.g., raw disk I/O
- *		on a disk with no mounted partitions.)
- *		NOTE: Persistent kstats cannot be virtual, since ks_data
- *		points to garbage as soon as the driver goes away.
+ *		Indicates that this kstat has restricted access and is
+ *		not world readable.
  *
  * The following flags are maintained by the kstat framework:
  *
- *	KSTAT_FLAG_DORMANT:
- *
- *		For persistent kstats, indicates that the kstat is in the
- *		dormant state (e.g., the corresponding device is closed).
- *
  *	KSTAT_FLAG_INVALID:
  *
  *		This flag is set when a kstat is in a transitional state,
@@ -268,8 +257,8 @@ typedef struct kstat {
 #define	KSTAT_FLAG_VIRTUAL		0x01
 #define	KSTAT_FLAG_VAR_SIZE		0x02
 #define	KSTAT_FLAG_WRITABLE		0x04
-#define	KSTAT_FLAG_PERSISTENT		0x08
-#define	KSTAT_FLAG_DORMANT		0x10
+#define	KSTAT_FLAG_RESTRICTED		0x08
+#define	KSTAT_FLAG_UNUSED		0x10
 #define	KSTAT_FLAG_INVALID		0x20
 #define	KSTAT_FLAG_LONGSTRINGS		0x40
 #define	KSTAT_FLAG_NO_HEADERS		0x80
@@ -722,33 +711,8 @@ extern void	kstat_init(void);	/* initialize kstat framework */
  * you must NOT be holding that kstat's ks_lock.  Otherwise, you may
  * deadlock with a kstat reader.
  *
- * Persistent kstats
- *
- * From the provider's point of view, persistence is transparent.  The only
- * difference between ephemeral (normal) kstats and persistent kstats
- * is that you pass KSTAT_FLAG_PERSISTENT to kstat_create().  Magically,
- * this has the effect of making your data visible even when you're
- * not home.  Persistence is important to tools like iostat, which want
- * to get a meaningful picture of disk activity.  Without persistence,
- * raw disk i/o statistics could never accumulate: they would come and
- * go with each open/close of the raw device.
- *
- * The magic of persistence works by slightly altering the behavior of
- * kstat_create() and kstat_delete().  The first call to kstat_create()
- * creates a new kstat, as usual.  However, kstat_delete() does not
- * actually delete the kstat: it performs one final update of the data
- * (i.e., calls the ks_update routine), marks the kstat as dormant, and
- * sets the ks_lock, ks_update, ks_private, and ks_snapshot fields back
- * to their default values (since they might otherwise point to garbage,
- * e.g. if the provider is going away).  kstat clients can still access
- * the dormant kstat just like a live kstat; they just continue to see
- * the final data values as long as the kstat remains dormant.
- * All subsequent kstat_create() calls simply find the already-existing,
- * dormant kstat and return a pointer to it, without altering any fields.
- * The provider then performs its usual initialization sequence, and
- * calls kstat_install().  kstat_install() uses the old data values to
- * initialize the native data (i.e., ks_update is called with KSTAT_WRITE),
- * thus making it seem like you were never gone.
+ * Persistent kstats are not implemented since there is no persistent
+ * namespace for them to reside.
  */
 
 extern kstat_t *kstat_create(const char *, int, const char *, const char *,

lib/libzpool/Makefile.am

@@ -113,6 +113,7 @@ nodist_libzpool_la_SOURCES = \
 	module/zfs/fm.c \
 	module/zfs/gzip.c \
 	module/zfs/hkdf.c \
+	module/zfs/jprint.c \
 	module/zfs/lz4.c \
 	module/zfs/lz4_zfs.c \
 	module/zfs/lzjb.c \
@@ -132,6 +133,7 @@ nodist_libzpool_la_SOURCES = \
 	module/zfs/spa_config.c \
 	module/zfs/spa_errlog.c \
 	module/zfs/spa_history.c \
+	module/zfs/spa_json_stats.c \
 	module/zfs/spa_log_spacemap.c \
 	module/zfs/spa_misc.c \
 	module/zfs/spa_stats.c \

module/Kbuild.in

@@ -356,6 +356,7 @@ ZFS_OBJS := \
 	fm.o \
 	gzip.o \
 	hkdf.o \
+	jprint.o \
 	lz4.o \
 	lz4_zfs.o \
 	lzjb.o \
@@ -375,6 +376,7 @@ ZFS_OBJS := \
 	spa_config.o \
 	spa_errlog.o \
 	spa_history.o \
+	spa_json_stats.o \
 	spa_log_spacemap.o \
 	spa_misc.o \
 	spa_stats.o \

module/Makefile.bsd

@@ -284,6 +284,7 @@ SRCS+=	abd.c \
 	edonr_zfs.c \
 	fm.c \
 	gzip.c \
+	jprint.c \
 	lz4.c \
 	lz4_zfs.c \
 	lzjb.c \
@@ -305,6 +306,7 @@ SRCS+=	abd.c \
 	spa_config.c \
 	spa_errlog.c \
 	spa_history.c \
+	spa_json_stats.c \
 	spa_log_spacemap.c \
 	spa_misc.c \
 	spa_stats.c \

module/os/linux/spl/spl-kstat.c

@@ -654,7 +654,7 @@ __kstat_install(kstat_t *ksp)
 	ASSERT(ksp);
 	mode_t mode;
 	/* Specify permission modes for different kstats */
-	if (strncmp(ksp->ks_proc.kpe_name, "dbufs", KSTAT_STRLEN) == 0) {
+	if (ksp->ks_flags & KSTAT_FLAG_RESTRICTED) {
 		mode = 0600;
 	} else {
 		mode = 0644;

module/zfs/dbuf_stats.c

@@ -189,7 +189,7 @@ dbuf_stats_hash_table_init(dbuf_hash_table_t *hash)
 	dsh->hash = hash;
 
 	ksp = kstat_create("zfs", 0, "dbufs", "misc",
-	    KSTAT_TYPE_RAW, 0, KSTAT_FLAG_VIRTUAL);
+	    KSTAT_TYPE_RAW, 0, KSTAT_FLAG_VIRTUAL | KSTAT_FLAG_RESTRICTED);
 	dsh->kstat = ksp;
 
 	if (ksp) {