Physical rewrite

cmd/zdb/zdb.c

@@ -208,7 +208,7 @@ sublivelist_verify_blkptr(void *arg, const blkptr_t *bp, boolean_t free,
 				sublivelist_verify_block_t svb = {
 				    .svb_dva = bp->blk_dva[i],
 				    .svb_allocated_txg =
-				    BP_GET_LOGICAL_BIRTH(bp)
+				    BP_GET_BIRTH(bp)
 				};
 
 				if (zfs_btree_find(&sv->sv_leftover, &svb,
@@ -2568,7 +2568,7 @@ snprintf_blkptr_compact(char *blkbuf, size_t buflen, const blkptr_t *bp,
 		    (u_longlong_t)BP_GET_PSIZE(bp),
 		    (u_longlong_t)BP_GET_FILL(bp),
 		    (u_longlong_t)BP_GET_LOGICAL_BIRTH(bp),
-		    (u_longlong_t)BP_GET_BIRTH(bp));
+		    (u_longlong_t)BP_GET_PHYSICAL_BIRTH(bp));
 		if (bp_freed)
 			(void) snprintf(blkbuf + strlen(blkbuf),
 			    buflen - strlen(blkbuf), " %s", "FREE");
@@ -2618,7 +2618,7 @@ visit_indirect(spa_t *spa, const dnode_phys_t *dnp,
 {
 	int err = 0;
 
-	if (BP_GET_LOGICAL_BIRTH(bp) == 0)
+	if (BP_GET_BIRTH(bp) == 0)
 		return (0);
 
 	print_indirect(spa, bp, zb, dnp);
@@ -2806,7 +2806,7 @@ dump_bptree_cb(void *arg, const blkptr_t *bp, dmu_tx_t *tx)
 	(void) arg, (void) tx;
 	char blkbuf[BP_SPRINTF_LEN];
 
-	if (BP_GET_LOGICAL_BIRTH(bp) != 0) {
+	if (BP_GET_BIRTH(bp) != 0) {
 		snprintf_blkptr(blkbuf, sizeof (blkbuf), bp);
 		(void) printf("\t%s\n", blkbuf);
 	}
@@ -2847,7 +2847,7 @@ dump_bpobj_cb(void *arg, const blkptr_t *bp, boolean_t bp_freed, dmu_tx_t *tx)
 	(void) arg, (void) tx;
 	char blkbuf[BP_SPRINTF_LEN];
 
-	ASSERT(BP_GET_LOGICAL_BIRTH(bp) != 0);
+	ASSERT(BP_GET_BIRTH(bp) != 0);
 	snprintf_blkptr_compact(blkbuf, sizeof (blkbuf), bp, bp_freed);
 	(void) printf("\t%s\n", blkbuf);
 	return (0);
@@ -5921,11 +5921,11 @@ zdb_count_block(zdb_cb_t *zcb, zilog_t *zilog, const blkptr_t *bp,
 		 * entry back to the block pointer before we claim it.
 		 */
 		if (v == DDT_PHYS_FLAT) {
-			ASSERT3U(BP_GET_BIRTH(bp), ==,
+			ASSERT3U(BP_GET_PHYSICAL_BIRTH(bp), ==,
 			    ddt_phys_birth(dde->dde_phys, v));
 			tempbp = *bp;
 			ddt_bp_fill(dde->dde_phys, v, &tempbp,
-			    BP_GET_BIRTH(bp));
+			    BP_GET_PHYSICAL_BIRTH(bp));
 			bp = &tempbp;
 		}
 
@@ -6151,7 +6151,7 @@ zdb_blkptr_cb(spa_t *spa, zilog_t *zilog, const blkptr_t *bp,
 	if (zb->zb_level == ZB_DNODE_LEVEL)
 		return (0);
 
-	if (dump_opt['b'] >= 5 && BP_GET_LOGICAL_BIRTH(bp) > 0) {
+	if (dump_opt['b'] >= 5 && BP_GET_BIRTH(bp) > 0) {
 		char blkbuf[BP_SPRINTF_LEN];
 		snprintf_blkptr(blkbuf, sizeof (blkbuf), bp);
 		(void) printf("objset %llu object %llu "

cmd/zdb/zdb_il.c

@@ -176,7 +176,7 @@ zil_prt_rec_write(zilog_t *zilog, int txtype, const void *arg)
 
 	if (lr->lr_common.lrc_reclen == sizeof (lr_write_t)) {
 		(void) printf("%shas blkptr, %s\n", tab_prefix,
-		    !BP_IS_HOLE(bp) && BP_GET_LOGICAL_BIRTH(bp) >=
+		    !BP_IS_HOLE(bp) && BP_GET_BIRTH(bp) >=
 		    spa_min_claim_txg(zilog->zl_spa) ?
 		    "will claim" : "won't claim");
 		print_log_bp(bp, tab_prefix);
@@ -189,7 +189,7 @@ zil_prt_rec_write(zilog_t *zilog, int txtype, const void *arg)
 			(void) printf("%s<hole>\n", tab_prefix);
 			return;
 		}
-		if (BP_GET_LOGICAL_BIRTH(bp) < zilog->zl_header->zh_claim_txg) {
+		if (BP_GET_BIRTH(bp) < zilog->zl_header->zh_claim_txg) {
 			(void) printf("%s<block already committed>\n",
 			    tab_prefix);
 			return;
@@ -240,7 +240,7 @@ zil_prt_rec_write_enc(zilog_t *zilog, int txtype, const void *arg)
 
 	if (lr->lr_common.lrc_reclen == sizeof (lr_write_t)) {
 		(void) printf("%shas blkptr, %s\n", tab_prefix,
-		    !BP_IS_HOLE(bp) && BP_GET_LOGICAL_BIRTH(bp) >=
+		    !BP_IS_HOLE(bp) && BP_GET_BIRTH(bp) >=
 		    spa_min_claim_txg(zilog->zl_spa) ?
 		    "will claim" : "won't claim");
 		print_log_bp(bp, tab_prefix);
@@ -476,7 +476,7 @@ print_log_block(zilog_t *zilog, const blkptr_t *bp, void *arg,
 
 	if (claim_txg != 0)
 		claim = "already claimed";
-	else if (BP_GET_LOGICAL_BIRTH(bp) >= spa_min_claim_txg(zilog->zl_spa))
+	else if (BP_GET_BIRTH(bp) >= spa_min_claim_txg(zilog->zl_spa))
 		claim = "will claim";
 	else
 		claim = "won't claim";

cmd/zfs/zfs_main.c

@@ -440,7 +440,7 @@ get_usage(zfs_help_t idx)
 		return (gettext("\tredact <snapshot> <bookmark> "
 		    "<redaction_snapshot> ...\n"));
 	case HELP_REWRITE:
-		return (gettext("\trewrite [-rvx] [-o <offset>] [-l <length>] "
+		return (gettext("\trewrite [-Prvx] [-o <offset>] [-l <length>] "
 		    "<directory|file ...>\n"));
 	case HELP_JAIL:
 		return (gettext("\tjail <jailid|jailname> <filesystem>\n"));
@@ -9177,8 +9177,11 @@ zfs_do_rewrite(int argc, char **argv)
 	zfs_rewrite_args_t args;
 	memset(&args, 0, sizeof (args));
 
-	while ((c = getopt(argc, argv, "l:o:rvx")) != -1) {
+	while ((c = getopt(argc, argv, "Pl:o:rvx")) != -1) {
 		switch (c) {
+		case 'P':
+			args.flags |= ZFS_REWRITE_PHYSICAL;
+			break;
 		case 'l':
 			args.len = strtoll(optarg, NULL, 0);
 			break;

include/sys/dbuf.h

@@ -164,6 +164,7 @@ typedef struct dbuf_dirty_record {
 			boolean_t dr_nopwrite;
 			boolean_t dr_brtwrite;
 			boolean_t dr_diowrite;
+			boolean_t dr_rewrite;
 			boolean_t dr_has_raw_params;
 
 			/* Override and raw params are mutually exclusive. */

include/sys/dmu.h

@@ -822,6 +822,7 @@ struct blkptr *dmu_buf_get_blkptr(dmu_buf_t *db);
  */
 void dmu_buf_will_dirty(dmu_buf_t *db, dmu_tx_t *tx);
 void dmu_buf_will_dirty_flags(dmu_buf_t *db, dmu_tx_t *tx, dmu_flags_t flags);
+void dmu_buf_will_rewrite(dmu_buf_t *db, dmu_tx_t *tx);
 boolean_t dmu_buf_is_dirty(dmu_buf_t *db, dmu_tx_t *tx);
 void dmu_buf_set_crypt_params(dmu_buf_t *db_fake, boolean_t byteorder,
     const uint8_t *salt, const uint8_t *iv, const uint8_t *mac, dmu_tx_t *tx);

include/sys/dmu_traverse.h

@@ -59,6 +59,13 @@ typedef int (blkptr_cb_t)(spa_t *spa, zilog_t *zilog, const blkptr_t *bp,
  */
 #define	TRAVERSE_NO_DECRYPT		(1<<5)
 
+/*
+ * Always use logical birth time for birth time comparisons.  This is useful
+ * for operations that care about user data changes rather than physical
+ * block rewrites (e.g., incremental replication).
+ */
+#define	TRAVERSE_LOGICAL		(1<<6)
+
 /* Special traverse error return value to indicate skipping of children */
 #define	TRAVERSE_VISIT_NO_CHILDREN	-1
 

include/sys/fs/zfs.h

@@ -1627,6 +1627,9 @@ typedef struct zfs_rewrite_args {
 	uint64_t	arg;
 } zfs_rewrite_args_t;
 
+/* zfs_rewrite_args flags */
+#define	ZFS_REWRITE_PHYSICAL	0x1	/* Preserve logical birth time. */
+
 #define	ZFS_IOC_REWRITE		_IOW(0x83, 3, zfs_rewrite_args_t)
 
 /*

include/sys/spa.h

@@ -140,7 +140,7 @@ typedef struct zio_cksum_salt {
  *	+-------+-------+-------+-------+-------+-------+-------+-------+
  * 6	|BDX|lvl| type	| cksum |E| comp|    PSIZE	|     LSIZE	|
  *	+-------+-------+-------+-------+-------+-------+-------+-------+
- * 7	|			padding					|
+ * 7	|R|			padding					|
  *	+-------+-------+-------+-------+-------+-------+-------+-------+
  * 8	|			padding					|
  *	+-------+-------+-------+-------+-------+-------+-------+-------+
@@ -175,6 +175,7 @@ typedef struct zio_cksum_salt {
  * E		blkptr_t contains embedded data (see below)
  * lvl		level of indirection
  * type		DMU object type
+ * R		rewrite (reallocated/rewritten at phys birth TXG)
  * phys birth	txg when dva[0] was written; zero if same as logical birth txg
  *              note that typically all the dva's would be written in this
  *              txg, but they could be different if they were moved by
@@ -204,7 +205,7 @@ typedef struct zio_cksum_salt {
  *	+-------+-------+-------+-------+-------+-------+-------+-------+
  * 6	|BDX|lvl| type	| cksum |E| comp|    PSIZE	|     LSIZE	|
  *	+-------+-------+-------+-------+-------+-------+-------+-------+
- * 7	|			padding					|
+ * 7	|R|			padding					|
  *	+-------+-------+-------+-------+-------+-------+-------+-------+
  * 8	|			padding					|
  *	+-------+-------+-------+-------+-------+-------+-------+-------+
@@ -373,7 +374,8 @@ typedef enum bp_embedded_type {
 typedef struct blkptr {
 	dva_t		blk_dva[SPA_DVAS_PER_BP]; /* Data Virtual Addresses */
 	uint64_t	blk_prop;	/* size, compression, type, etc	    */
-	uint64_t	blk_pad[2];	/* Extra space for the future	    */
+	uint64_t	blk_prop2;	/* additional properties	    */
+	uint64_t	blk_pad;	/* Extra space for the future	    */
 	uint64_t	blk_birth_word[2];
 	uint64_t	blk_fill;	/* fill count			    */
 	zio_cksum_t	blk_cksum;	/* 256-bit checksum		    */
@@ -476,32 +478,51 @@ typedef struct blkptr {
 #define	BP_GET_FREE(bp)			BF64_GET((bp)->blk_fill, 0, 1)
 #define	BP_SET_FREE(bp, x)		BF64_SET((bp)->blk_fill, 0, 1, x)
 
+/*
+ * Block birth time macros for different use cases:
+ * - BP_GET_LOGICAL_BIRTH(): When the block was logically modified by user.
+ *   To be used with a focus on user data, like incremental replication.
+ * - BP_GET_PHYSICAL_BIRTH(): When the block was physically written to disks.
+ *   For regular writes is equal to logical birth.  For dedup and block cloning
+ *   can be smaller than logical birth.  For remapped and rewritten blocks can
+ *   be bigger. To be used with focus on physical disk content: ARC, DDT, scrub.
+ * - BP_GET_RAW_PHYSICAL_BIRTH(): Raw physical birth value.  Zero if equal
+ *   to logical birth.  Should only be used for BP copying and debugging.
+ * - BP_GET_BIRTH(): When the block was allocated, which is a physical birth
+ *   for rewritten blocks (rewrite flag set) or logical birth otherwise.
+ */
 #define	BP_GET_LOGICAL_BIRTH(bp)	(bp)->blk_birth_word[1]
 #define	BP_SET_LOGICAL_BIRTH(bp, x)	((bp)->blk_birth_word[1] = (x))
 
-#define	BP_GET_PHYSICAL_BIRTH(bp)	(bp)->blk_birth_word[0]
+#define	BP_GET_RAW_PHYSICAL_BIRTH(bp)	(bp)->blk_birth_word[0]
 #define	BP_SET_PHYSICAL_BIRTH(bp, x)	((bp)->blk_birth_word[0] = (x))
 
-#define	BP_GET_BIRTH(bp)					\
-	(BP_IS_EMBEDDED(bp) ? 0 : 				\
-	BP_GET_PHYSICAL_BIRTH(bp) ? BP_GET_PHYSICAL_BIRTH(bp) :	\
+#define	BP_GET_PHYSICAL_BIRTH(bp)					\
+	(BP_IS_EMBEDDED(bp) ? 0 : 					\
+	BP_GET_RAW_PHYSICAL_BIRTH(bp) ? BP_GET_RAW_PHYSICAL_BIRTH(bp) :	\
 	BP_GET_LOGICAL_BIRTH(bp))
 
-#define	BP_SET_BIRTH(bp, logical, physical)	\
-{						\
-	ASSERT(!BP_IS_EMBEDDED(bp));		\
-	BP_SET_LOGICAL_BIRTH(bp, logical);	\
-	BP_SET_PHYSICAL_BIRTH(bp, 		\
-	    ((logical) == (physical) ? 0 : (physical))); \
+#define	BP_GET_BIRTH(bp)					\
+	((BP_IS_EMBEDDED(bp) || !BP_GET_REWRITE(bp)) ?		\
+	BP_GET_LOGICAL_BIRTH(bp) : BP_GET_PHYSICAL_BIRTH(bp))
+
+#define	BP_SET_BIRTH(bp, logical, physical)			\
+{								\
+	ASSERT(!BP_IS_EMBEDDED(bp));				\
+	BP_SET_LOGICAL_BIRTH(bp, logical);			\
+	BP_SET_PHYSICAL_BIRTH(bp, 				\
+	    ((logical) == (physical) ? 0 : (physical)));	\
 }
 
 #define	BP_GET_FILL(bp)				\
-	((BP_IS_ENCRYPTED(bp)) ? BF64_GET((bp)->blk_fill, 0, 32) : \
-	((BP_IS_EMBEDDED(bp)) ? 1 : (bp)->blk_fill))
+	(BP_IS_EMBEDDED(bp) ? 1 : 			\
+	BP_IS_ENCRYPTED(bp) ? BF64_GET((bp)->blk_fill, 0, 32) : \
+	(bp)->blk_fill)
 
 #define	BP_SET_FILL(bp, fill)			\
 {						\
-	if (BP_IS_ENCRYPTED(bp))			\
+	ASSERT(!BP_IS_EMBEDDED(bp));		\
+	if (BP_IS_ENCRYPTED(bp))		\
 		BF64_SET((bp)->blk_fill, 0, 32, fill); \
 	else					\
 		(bp)->blk_fill = fill;		\
@@ -516,6 +537,15 @@ typedef struct blkptr {
 	BF64_SET((bp)->blk_fill, 32, 32, iv2);	\
 }
 
+#define	BP_GET_REWRITE(bp)			\
+	(BP_IS_EMBEDDED(bp) ? 0 : BF64_GET((bp)->blk_prop2, 63, 1))
+
+#define	BP_SET_REWRITE(bp, x)			\
+{						\
+	ASSERT(!BP_IS_EMBEDDED(bp));		\
+	BF64_SET((bp)->blk_prop2, 63, 1, x);	\
+}
+
 #define	BP_IS_METADATA(bp)	\
 	(BP_GET_LEVEL(bp) > 0 || DMU_OT_IS_METADATA(BP_GET_TYPE(bp)))
 
@@ -545,7 +575,7 @@ typedef struct blkptr {
 	(dva1)->dva_word[0] == (dva2)->dva_word[0])
 
 #define	BP_EQUAL(bp1, bp2)	\
-	(BP_GET_BIRTH(bp1) == BP_GET_BIRTH(bp2) &&	\
+	(BP_GET_PHYSICAL_BIRTH(bp1) == BP_GET_PHYSICAL_BIRTH(bp2) &&	\
 	BP_GET_LOGICAL_BIRTH(bp1) == BP_GET_LOGICAL_BIRTH(bp2) &&	\
 	DVA_EQUAL(&(bp1)->blk_dva[0], &(bp2)->blk_dva[0]) &&	\
 	DVA_EQUAL(&(bp1)->blk_dva[1], &(bp2)->blk_dva[1]) &&	\
@@ -588,8 +618,8 @@ typedef struct blkptr {
 {						\
 	BP_ZERO_DVAS(bp);			\
 	(bp)->blk_prop = 0;			\
-	(bp)->blk_pad[0] = 0;			\
-	(bp)->blk_pad[1] = 0;			\
+	(bp)->blk_prop2 = 0;			\
+	(bp)->blk_pad = 0;			\
 	(bp)->blk_birth_word[0] = 0;		\
 	(bp)->blk_birth_word[1] = 0;		\
 	(bp)->blk_fill = 0;			\
@@ -696,7 +726,7 @@ typedef struct blkptr {
 		    (u_longlong_t)BP_GET_LSIZE(bp),			\
 		    (u_longlong_t)BP_GET_PSIZE(bp),			\
 		    (u_longlong_t)BP_GET_LOGICAL_BIRTH(bp),		\
-		    (u_longlong_t)BP_GET_BIRTH(bp),			\
+		    (u_longlong_t)BP_GET_PHYSICAL_BIRTH(bp),		\
 		    (u_longlong_t)BP_GET_FILL(bp),			\
 		    ws,							\
 		    (u_longlong_t)bp->blk_cksum.zc_word[0],		\

include/sys/zio.h

@@ -374,6 +374,7 @@ typedef struct zio_prop {
 	boolean_t		zp_encrypt;
 	boolean_t		zp_byteorder;
 	boolean_t		zp_direct_write;
+	boolean_t		zp_rewrite;
 	uint8_t			zp_salt[ZIO_DATA_SALT_LEN];
 	uint8_t			zp_iv[ZIO_DATA_IV_LEN];
 	uint8_t			zp_mac[ZIO_DATA_MAC_LEN];

include/zfeature_common.h

@@ -89,6 +89,7 @@ typedef enum spa_feature {
 	SPA_FEATURE_LARGE_MICROZAP,
 	SPA_FEATURE_DYNAMIC_GANG_HEADER,
 	SPA_FEATURE_BLOCK_CLONING_ENDIAN,
+	SPA_FEATURE_PHYSICAL_REWRITE,
 	SPA_FEATURES
 } spa_feature_t;
 

lib/libzdb/libzdb.c

@@ -93,9 +93,9 @@ livelist_compare(const void *larg, const void *rarg)
 	 * Since we're storing blkptrs without cancelling FREE/ALLOC pairs,
 	 * it's possible the offsets are equal. In that case, sort by txg
 	 */
-	if (BP_GET_LOGICAL_BIRTH(l) < BP_GET_LOGICAL_BIRTH(r)) {
+	if (BP_GET_BIRTH(l) < BP_GET_BIRTH(r)) {
 		return (-1);
-	} else if (BP_GET_LOGICAL_BIRTH(l) > BP_GET_LOGICAL_BIRTH(r)) {
+	} else if (BP_GET_BIRTH(l) > BP_GET_BIRTH(r)) {
 		return (+1);
 	}
 	return (0);

lib/libzfs/libzfs.abi

@@ -638,7 +638,7 @@
     <elf-symbol name='fletcher_4_superscalar_ops' size='128' type='object-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='libzfs_config_ops' size='16' type='object-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='sa_protocol_names' size='16' type='object-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
-    <elf-symbol name='spa_feature_table' size='2576' type='object-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
+    <elf-symbol name='spa_feature_table' size='2632' type='object-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='zfeature_checks_disable' size='4' type='object-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='zfs_deleg_perm_tab' size='528' type='object-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='zfs_history_event_names' size='328' type='object-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
@@ -6398,7 +6398,8 @@
       <enumerator name='SPA_FEATURE_LARGE_MICROZAP' value='43'/>
       <enumerator name='SPA_FEATURE_DYNAMIC_GANG_HEADER' value='44'/>
       <enumerator name='SPA_FEATURE_BLOCK_CLONING_ENDIAN' value='45'/>
-      <enumerator name='SPA_FEATURES' value='46'/>
+      <enumerator name='SPA_FEATURE_PHYSICAL_REWRITE' value='46'/>
+      <enumerator name='SPA_FEATURES' value='47'/>
     </enum-decl>
     <typedef-decl name='spa_feature_t' type-id='33ecb627' id='d6618c78'/>
     <qualified-type-def type-id='80f4b756' const='yes' id='b99c00c9'/>
@@ -9605,8 +9606,8 @@
     </function-decl>
   </abi-instr>
   <abi-instr address-size='64' path='module/zcommon/zfeature_common.c' language='LANG_C99'>
-    <array-type-def dimensions='1' type-id='83f29ca2' size-in-bits='20608' id='b9408bab'>
-      <subrange length='46' type-id='7359adad' id='8b86bc1b'/>
+    <array-type-def dimensions='1' type-id='83f29ca2' size-in-bits='21056' id='fd43354e'>
+      <subrange length='47' type-id='7359adad' id='8f8900fe'/>
     </array-type-def>
     <enum-decl name='zfeature_flags' id='6db816a4'>
       <underlying-type type-id='9cac1fee'/>
@@ -9684,7 +9685,7 @@
     <pointer-type-def type-id='611586a1' size-in-bits='64' id='2e243169'/>
     <qualified-type-def type-id='eaa32e2f' const='yes' id='83be723c'/>
     <pointer-type-def type-id='83be723c' size-in-bits='64' id='7acd98a2'/>
-    <var-decl name='spa_feature_table' type-id='b9408bab' mangled-name='spa_feature_table' visibility='default' elf-symbol-id='spa_feature_table'/>
+    <var-decl name='spa_feature_table' type-id='fd43354e' mangled-name='spa_feature_table' visibility='default' elf-symbol-id='spa_feature_table'/>
     <var-decl name='zfeature_checks_disable' type-id='c19b74c3' mangled-name='zfeature_checks_disable' visibility='default' elf-symbol-id='zfeature_checks_disable'/>
     <function-decl name='opendir' visibility='default' binding='global' size-in-bits='64'>
       <parameter type-id='80f4b756'/>