Add last_scrubbed_txg property and option to scrub from last saved txg

The `last_scrubbed_txg` property indicates the transaction
group (TXG) up to which the most recent scrub operation has
checked and repaired the dataset. This provides administrators
with insight into the data integrity status of their pool at a
specific point in time.

Sponsored-By: Wasabi Technology, Inc.
Sponsored-By: Klara Inc.
Signed-off-by: Mariusz Zaborski <[email protected]>

Author: oshogbo

cmd/zpool/zpool_main.c

@@ -512,7 +512,8 @@ get_usage(zpool_help_t idx)
 		return (gettext("\tinitialize [-c | -s | -u] [-w] <pool> "
 		    "[<device> ...]\n"));
 	case HELP_SCRUB:
-		return (gettext("\tscrub [-s | -p] [-w] [-e] <pool> ...\n"));
+		return (gettext("\tscrub [-e | -s | -p | -C] [-w] "
+		    "<pool> ...\n"));
 	case HELP_RESILVER:
 		return (gettext("\tresilver <pool> ...\n"));
 	case HELP_TRIM:
@@ -8429,12 +8430,13 @@ wait_callback(zpool_handle_t *zhp, void *data)
 }
 
 /*
- * zpool scrub [-s | -p] [-w] [-e] <pool> ...
+ * zpool scrub [-e | -s | -p | -C] [-w] <pool> ...
  *
  *	-e	Only scrub blocks in the error log.
  *	-s	Stop.  Stops any in-progress scrub.
  *	-p	Pause. Pause in-progress scrub.
  *	-w	Wait.  Blocks until scrub has completed.
+ *	-C	Scrub from last saved txg.
  */
 int
 zpool_do_scrub(int argc, char **argv)
@@ -8450,9 +8452,10 @@ zpool_do_scrub(int argc, char **argv)
 	boolean_t is_error_scrub = B_FALSE;
 	boolean_t is_pause = B_FALSE;
 	boolean_t is_stop = B_FALSE;
+	boolean_t is_txg_continue = B_FALSE;
 
 	/* check options */
-	while ((c = getopt(argc, argv, "spwe")) != -1) {
+	while ((c = getopt(argc, argv, "spweC")) != -1) {
 		switch (c) {
 		case 'e':
 			is_error_scrub = B_TRUE;
@@ -8466,6 +8469,9 @@ zpool_do_scrub(int argc, char **argv)
 		case 'w':
 			wait = B_TRUE;
 			break;
+		case 'C':
+			is_txg_continue = B_TRUE;
+			break;
 		case '?':
 			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
 			    optopt);
@@ -8477,6 +8483,18 @@ zpool_do_scrub(int argc, char **argv)
 		(void) fprintf(stderr, gettext("invalid option "
 		    "combination :-s and -p are mutually exclusive\n"));
 		usage(B_FALSE);
+	} else if (is_pause && is_txg_continue) {
+		(void) fprintf(stderr, gettext("invalid option "
+		    "combination :-p and -C are mutually exclusive\n"));
+		usage(B_FALSE);
+	} else if (is_stop && is_txg_continue) {
+		(void) fprintf(stderr, gettext("invalid option "
+		    "combination :-s and -C are mutually exclusive\n"));
+		usage(B_FALSE);
+	} else if (is_error_scrub && is_txg_continue) {
+		(void) fprintf(stderr, gettext("invalid option "
+		    "combination :-e and -C are mutually exclusive\n"));
+		usage(B_FALSE);
 	} else {
 		if (is_error_scrub)
 			cb.cb_type = POOL_SCAN_ERRORSCRUB;
@@ -8485,6 +8503,8 @@ zpool_do_scrub(int argc, char **argv)
 			cb.cb_scrub_cmd = POOL_SCRUB_PAUSE;
 		} else if (is_stop) {
 			cb.cb_type = POOL_SCAN_NONE;
+		} else if (is_txg_continue) {
+			cb.cb_scrub_cmd = POOL_SCRUB_FROM_LAST_TXG;
 		} else {
 			cb.cb_scrub_cmd = POOL_SCRUB_NORMAL;
 		}

include/sys/dmu.h

@@ -381,6 +381,7 @@ typedef struct dmu_buf {
 #define	DMU_POOL_CREATION_VERSION	"creation_version"
 #define	DMU_POOL_SCAN			"scan"
 #define	DMU_POOL_ERRORSCRUB		"error_scrub"
+#define	DMU_POOL_LAST_SCRUBBED_TXG	"last_scrubbed_txg"
 #define	DMU_POOL_FREE_BPOBJ		"free_bpobj"
 #define	DMU_POOL_BPTREE_OBJ		"bptree_obj"
 #define	DMU_POOL_EMPTY_BPOBJ		"empty_bpobj"

include/sys/fs/zfs.h

@@ -265,6 +265,7 @@ typedef enum {
 	ZPOOL_PROP_DEDUP_TABLE_SIZE,
 	ZPOOL_PROP_DEDUP_TABLE_QUOTA,
 	ZPOOL_PROP_DEDUPCACHED,
+	ZPOOL_PROP_LAST_SCRUBBED_TXG,
 	ZPOOL_NUM_PROPS
 } zpool_prop_t;
 
@@ -1088,6 +1089,7 @@ typedef enum pool_scan_func {
 typedef enum pool_scrub_cmd {
 	POOL_SCRUB_NORMAL = 0,
 	POOL_SCRUB_PAUSE,
+	POOL_SCRUB_FROM_LAST_TXG,
 	POOL_SCRUB_FLAGS_END
 } pool_scrub_cmd_t;
 

include/sys/spa.h

@@ -1082,6 +1082,7 @@ extern uint64_t spa_get_deadman_failmode(spa_t *spa);
 extern void spa_set_deadman_failmode(spa_t *spa, const char *failmode);
 extern boolean_t spa_suspended(spa_t *spa);
 extern uint64_t spa_bootfs(spa_t *spa);
+extern uint64_t spa_get_last_scrubbed_txg(spa_t *spa);
 extern uint64_t spa_delegation(spa_t *spa);
 extern objset_t *spa_meta_objset(spa_t *spa);
 extern space_map_t *spa_syncing_log_sm(spa_t *spa);

include/sys/spa_impl.h

@@ -318,6 +318,7 @@ struct spa {
 	uint64_t	spa_scan_pass_scrub_spent_paused; /* total paused */
 	uint64_t	spa_scan_pass_exam;	/* examined bytes per pass */
 	uint64_t	spa_scan_pass_issued;	/* issued bytes per pass */
+	uint64_t	spa_scrubbed_last_txg;	/* last txg scrubbed */
 
 	/* error scrub pause time in milliseconds */
 	uint64_t	spa_scan_pass_errorscrub_pause;

lib/libzfs/libzfs.abi

@@ -3132,7 +3132,8 @@
       <enumerator name='ZPOOL_PROP_DEDUP_TABLE_SIZE' value='36'/>
       <enumerator name='ZPOOL_PROP_DEDUP_TABLE_QUOTA' value='37'/>
       <enumerator name='ZPOOL_PROP_DEDUPCACHED' value='38'/>
-      <enumerator name='ZPOOL_NUM_PROPS' value='39'/>
+      <enumerator name='ZPOOL_PROP_LAST_SCRUBBED_TXG' value='39'/>
+      <enumerator name='ZPOOL_NUM_PROPS' value='40'/>
     </enum-decl>
     <typedef-decl name='zpool_prop_t' type-id='af1ba157' id='5d0c23fb'/>
     <typedef-decl name='regoff_t' type-id='95e97e5e' id='54a2a2a8'/>
@@ -5984,7 +5985,8 @@
       <underlying-type type-id='9cac1fee'/>
       <enumerator name='POOL_SCRUB_NORMAL' value='0'/>
       <enumerator name='POOL_SCRUB_PAUSE' value='1'/>
-      <enumerator name='POOL_SCRUB_FLAGS_END' value='2'/>
+      <enumerator name='POOL_SCRUB_FROM_LAST_TXG' value='2'/>
+      <enumerator name='POOL_SCRUB_FLAGS_END' value='3'/>
     </enum-decl>
     <typedef-decl name='pool_scrub_cmd_t' type-id='a1474cbd' id='b51cf3c2'/>
     <enum-decl name='zpool_errata' id='d9abbf54'>

man/man7/zpoolprops.7

@@ -28,7 +28,7 @@
 .\" Copyright (c) 2021, Colm Buckley <[email protected]>
 .\" Copyright (c) 2023, Klara Inc.
 .\"
-.Dd July 29, 2024
+.Dd November 18, 2024
 .Dt ZPOOLPROPS 7
 .Os
 .
@@ -135,6 +135,19 @@ A unique identifier for the pool.
 The current health of the pool.
 Health can be one of
 .Sy ONLINE , DEGRADED , FAULTED , OFFLINE, REMOVED , UNAVAIL .
+.It Sy last_scrubbed_txg
+Indicates the transaction group (TXG) up to which the most recent scrub
+operation has checked and repaired the dataset.
+This provides insight into the data integrity status of their pool at
+a specific point in time.
+.Xr zpool-scrub 8
+can utilize this property to scan only data that has changed since the last
+scrub completed, when given the
+.Fl C
+flag.
+This property is not updated when performing an error scrub with the
+.Fl e
+flag.
 .It Sy leaked
 Space not released while
 .Sy freeing

man/man8/zpool-scrub.8

@@ -26,7 +26,7 @@
 .\" Copyright 2017 Nexenta Systems, Inc.
 .\" Copyright (c) 2017 Open-E, Inc. All Rights Reserved.
 .\"
-.Dd June 22, 2023
+.Dd November 18, 2024
 .Dt ZPOOL-SCRUB 8
 .Os
 .
@@ -36,9 +36,8 @@
 .Sh SYNOPSIS
 .Nm zpool
 .Cm scrub
-.Op Fl s Ns | Ns Fl p
+.Op Ns Fl e | Ns Fl p | Fl s Ns | Fl C Ns
 .Op Fl w
-.Op Fl e
 .Ar pool Ns …
 .
 .Sh DESCRIPTION
@@ -114,6 +113,10 @@ The pool must have been scrubbed at least once with the
 feature enabled to use this option.
 Error scrubbing cannot be run simultaneously with regular scrubbing or
 resilvering, nor can it be run when a regular scrub is paused.
+.It Fl C
+Continue scrub from last saved txg (see zpool
+.Sy last_scrubbed_txg
+property).
 .El
 .Sh EXAMPLES
 .Ss Example 1

module/zcommon/zpool_prop.c

@@ -128,6 +128,9 @@ zpool_prop_init(void)
 	zprop_register_number(ZPOOL_PROP_DEDUP_TABLE_SIZE, "dedup_table_size",
 	    0, PROP_READONLY, ZFS_TYPE_POOL, "<size>", "DDTSIZE", B_FALSE,
 	    sfeatures);
+	zprop_register_number(ZPOOL_PROP_LAST_SCRUBBED_TXG,
+	    "last_scrubbed_txg", 0, PROP_READONLY, ZFS_TYPE_POOL, "<txg>",
+	    "LAST_SCRUBBED_TXG", B_FALSE, sfeatures);
 
 	/* default number properties */
 	zprop_register_number(ZPOOL_PROP_VERSION, "version", SPA_VERSION,

module/zfs/dsl_scan.c

@@ -231,6 +231,9 @@ static uint_t zfs_resilver_defer_percent = 10;
 	((scn)->scn_phys.scn_func == POOL_SCAN_SCRUB || \
 	(scn)->scn_phys.scn_func == POOL_SCAN_RESILVER)
 
+#define	DSL_SCAN_IS_SCRUB(scn)		\
+	((scn)->scn_phys.scn_func == POOL_SCAN_SCRUB)
+
 /*
  * Enable/disable the processing of the free_bpobj object.
  */
@@ -1131,15 +1134,24 @@ dsl_scan_done(dsl_scan_t *scn, boolean_t complete, dmu_tx_t *tx)
 
 	spa_notify_waiters(spa);
 
-	if (dsl_scan_restarting(scn, tx))
+	if (dsl_scan_restarting(scn, tx)) {
 		spa_history_log_internal(spa, "scan aborted, restarting", tx,
 		    "errors=%llu", (u_longlong_t)spa_approx_errlog_size(spa));
-	else if (!complete)
+	} else if (!complete) {
 		spa_history_log_internal(spa, "scan cancelled", tx,
 		    "errors=%llu", (u_longlong_t)spa_approx_errlog_size(spa));
-	else
+	} else {
 		spa_history_log_internal(spa, "scan done", tx,
 		    "errors=%llu", (u_longlong_t)spa_approx_errlog_size(spa));
+		if (DSL_SCAN_IS_SCRUB(scn)) {
+			VERIFY0(zap_update(dp->dp_meta_objset,
+			    DMU_POOL_DIRECTORY_OBJECT,
+			    DMU_POOL_LAST_SCRUBBED_TXG,
+			    sizeof (uint64_t), 1,
+			    &scn->scn_phys.scn_max_txg, tx));
+			spa->spa_scrubbed_last_txg = scn->scn_phys.scn_max_txg;
+		}
+	}
 
 	if (DSL_SCAN_IS_SCRUB_RESILVER(scn)) {
 		spa->spa_scrub_active = B_FALSE;

module/zfs/spa.c

@@ -451,9 +451,10 @@ spa_prop_get_config(spa_t *spa, nvlist_t *nv)
 
 		spa_prop_add_list(nv, ZPOOL_PROP_DEDUP_TABLE_SIZE, NULL,
 		    ddt_get_ddt_dsize(spa), src);
-
 		spa_prop_add_list(nv, ZPOOL_PROP_HEALTH, NULL,
 		    rvd->vdev_state, src);
+		spa_prop_add_list(nv, ZPOOL_PROP_LAST_SCRUBBED_TXG, NULL,
+		    spa_get_last_scrubbed_txg(spa), src);
 
 		version = spa_version(spa);
 		if (version == zpool_prop_default_numeric(ZPOOL_PROP_VERSION)) {
@@ -4727,6 +4728,12 @@ spa_ld_get_props(spa_t *spa)
 	if (error != 0 && error != ENOENT)
 		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
 
+	/* Load the last scrubbed txg. */
+	error = spa_dir_prop(spa, DMU_POOL_LAST_SCRUBBED_TXG,
+	    &spa->spa_scrubbed_last_txg, B_FALSE);
+	if (error != 0 && error != ENOENT)
+		return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
+
 	/*
 	 * Load the livelist deletion field. If a livelist is queued for
 	 * deletion, indicate that in the spa

module/zfs/spa_misc.c

@@ -2676,6 +2676,12 @@ spa_mode(spa_t *spa)
 	return (spa->spa_mode);
 }
 
+uint64_t
+spa_get_last_scrubbed_txg(spa_t *spa)
+{
+	return (spa->spa_scrubbed_last_txg);
+}
+
 uint64_t
 spa_bootfs(spa_t *spa)
 {

module/zfs/zfs_ioctl.c

@@ -1718,6 +1718,9 @@ zfs_ioc_pool_scrub(const char *poolname, nvlist_t *innvl, nvlist_t *outnvl)
 		error = spa_scrub_pause_resume(spa, POOL_SCRUB_PAUSE);
 	} else if (scan_type == POOL_SCAN_NONE) {
 		error = spa_scan_stop(spa);
+	} else if (scan_cmd == POOL_SCRUB_FROM_LAST_TXG) {
+		error = spa_scan_range(spa, scan_type,
+		    spa_get_last_scrubbed_txg(spa), 0);
 	} else {
 		error = spa_scan(spa, scan_type);
 	}

tests/zfs-tests/tests/Makefile.am

@@ -1225,6 +1225,7 @@ nobase_dist_datadir_zfs_tests_tests_SCRIPTS += \
 	functional/cli_root/zpool_scrub/zpool_scrub_multiple_copies.ksh \
 	functional/cli_root/zpool_scrub/zpool_scrub_offline_device.ksh \
 	functional/cli_root/zpool_scrub/zpool_scrub_print_repairing.ksh \
+	functional/cli_root/zpool_scrub/zpool_scrub_txg_continue_from_last.ksh \
 	functional/cli_root/zpool_scrub/zpool_error_scrub_001_pos.ksh \
 	functional/cli_root/zpool_scrub/zpool_error_scrub_002_pos.ksh \
 	functional/cli_root/zpool_scrub/zpool_error_scrub_003_pos.ksh \

tests/zfs-tests/tests/functional/cli_root/zpool_get/zpool_get.cfg

@@ -63,6 +63,7 @@ typeset -a properties=(
     "bcloneused"
     "bclonesaved"
     "bcloneratio"
+    "last_scrubbed_txg"
     "feature@async_destroy"
     "feature@empty_bpobj"
     "feature@lz4_compress"