Fix "Detach spare vdev in case if resilvering does not happen"

Spare vdev should detach from the pool when a disk is reinserted.
However, spare detachment depends on the completion of resilvering,
and if resilver does not schedule, the spare vdev keeps attached to
the pool until the next resilvering. When a zfs pool contains
several disks (25+ mirror), resilvering does not always happen when
a disk is reinserted. In this patch, spare vdev is manually detached
from the pool when resilvering does not occur and it has been tested
on both Linux and FreeBSD.

Signed-off-by: Ameer Hamza <[email protected]>

Author: ixhamza

cmd/ztest.c

@@ -3456,7 +3456,7 @@ ztest_vdev_aux_add_remove(ztest_ds_t *zd, uint64_t id)
 		 * of devices that have pending state changes.
 		 */
 		if (ztest_random(2) == 0)
-			(void) vdev_online(spa, guid, 0, NULL);
+			(void) vdev_online(spa, guid, 0, NULL, NULL);
 
 		error = spa_vdev_remove(spa, guid, B_FALSE);
 
@@ -3935,7 +3935,7 @@ online_vdev(vdev_t *vd, void *arg)
 
 	/* Calling vdev_online will initialize the new metaslabs */
 	spa_config_exit(spa, SCL_STATE, spa);
-	error = vdev_online(spa, guid, ZFS_ONLINE_EXPAND, &newstate);
+	error = vdev_online(spa, guid, ZFS_ONLINE_EXPAND, &newstate, NULL);
 	spa_config_enter(spa, SCL_STATE, spa, RW_READER);
 
 	/*
@@ -6215,7 +6215,7 @@ ztest_fault_inject(ztest_ds_t *zd, uint64_t id)
 			 * aux_vdev removal.
 			 */
 			mutex_enter(&ztest_vdev_lock);
-			(void) vdev_online(spa, guid0, 0, NULL);
+			(void) vdev_online(spa, guid0, 0, NULL, NULL);
 			mutex_exit(&ztest_vdev_lock);
 		}
 	}

include/sys/vdev.h

@@ -146,7 +146,7 @@ vdev_gang_header_asize(vdev_t *vd)
 extern int vdev_fault(spa_t *spa, uint64_t guid, vdev_aux_t aux);
 extern int vdev_degrade(spa_t *spa, uint64_t guid, vdev_aux_t aux);
 extern int vdev_online(spa_t *spa, uint64_t guid, uint64_t flags,
-    vdev_state_t *);
+    vdev_state_t *, uint64_t *spare_guid);
 extern int vdev_offline(spa_t *spa, uint64_t guid, uint64_t flags);
 extern int vdev_remove_wanted(spa_t *spa, uint64_t guid);
 extern void vdev_clear(spa_t *spa, vdev_t *vd);

module/os/freebsd/zfs/vdev_geom.c

@@ -166,7 +166,7 @@ vdev_geom_resize(struct g_consumer *cp)
 		spa = vd->vdev_spa;
 		if (!spa->spa_autoexpand)
 			continue;
-		vdev_online(spa, vd->vdev_guid, ZFS_ONLINE_EXPAND, NULL);
+		vdev_online(spa, vd->vdev_guid, ZFS_ONLINE_EXPAND, NULL, NULL);
 	}
 }
 

module/zfs/spa.c

@@ -7058,7 +7058,7 @@ spa_vdev_attach(spa_t *spa, uint64_t guid, nvlist_t *nvroot, int replacing,
  * Detach a device from a mirror or replacing vdev.
  *
  * If 'replace_done' is specified, only detach if the parent
- * is a replacing vdev.
+ * is a replacing or a spare vdev.
  */
 int
 spa_vdev_detach(spa_t *spa, uint64_t guid, uint64_t pguid, int replace_done)

module/zfs/vdev.c

@@ -4102,11 +4102,13 @@ vdev_remove_wanted(spa_t *spa, uint64_t guid)
  * events are generated if the device fails to open.
  */
 int
-vdev_online(spa_t *spa, uint64_t guid, uint64_t flags, vdev_state_t *newstate)
+vdev_online(spa_t *spa, uint64_t guid, uint64_t flags, vdev_state_t *newstate,
+    uint64_t *spare_guid)
 {
 	vdev_t *vd, *tvd, *pvd, *rvd = spa->spa_root_vdev;
 	boolean_t wasoffline;
 	vdev_state_t oldstate;
+	boolean_t parent_is_spare = B_FALSE;
 
 	spa_vdev_state_enter(spa, SCL_NONE);
 
@@ -4146,8 +4148,10 @@ vdev_online(spa_t *spa, uint64_t guid, uint64_t flags, vdev_state_t *newstate)
 	if ((flags & ZFS_ONLINE_UNSPARE) &&
 	    !vdev_is_dead(vd) && vd->vdev_parent &&
 	    vd->vdev_parent->vdev_ops == &vdev_spare_ops &&
-	    vd->vdev_parent->vdev_child[0] == vd)
+	    vd->vdev_parent->vdev_child[0] == vd) {
 		vd->vdev_unspare = B_TRUE;
+		parent_is_spare = B_TRUE;
+	}
 
 	if ((flags & ZFS_ONLINE_EXPAND) || spa->spa_autoexpand) {
 
@@ -4183,9 +4187,23 @@ vdev_online(spa_t *spa, uint64_t guid, uint64_t flags, vdev_state_t *newstate)
 
 	if (wasoffline ||
 	    (oldstate < VDEV_STATE_DEGRADED &&
-	    vd->vdev_state >= VDEV_STATE_DEGRADED))
+	    vd->vdev_state >= VDEV_STATE_DEGRADED)) {
 		spa_event_notify(spa, vd, NULL, ESC_ZFS_VDEV_ONLINE);
 
+		/*
+		 * If resilver or rebuild could not get schedule, report
+		 * to detach the spare vdev
+		 */
+		if (spare_guid && !dsl_scan_resilvering(spa->spa_dsl_pool) &&
+		    !dsl_scan_resilver_scheduled(spa->spa_dsl_pool) &&
+		    !vdev_rebuild_active(tvd) && parent_is_spare &&
+		    vd->vdev_parent->vdev_child[1] &&
+		    vd->vdev_parent->vdev_child[1]->vdev_isspare) {
+			*spare_guid =
+			    vd->vdev_parent->vdev_child[1]->vdev_guid;
+		}
+	}
+
 	return (spa_vdev_state_exit(spa, vd, 0));
 }
 

module/zfs/zfs_ioctl.c

@@ -1882,12 +1882,16 @@ zfs_ioc_vdev_set_state(zfs_cmd_t *zc)
 	spa_t *spa;
 	int error;
 	vdev_state_t newstate = VDEV_STATE_UNKNOWN;
+	uint64_t spare_guid = 0;
 
 	if ((error = spa_open(zc->zc_name, &spa, FTAG)) != 0)
 		return (error);
 	switch (zc->zc_cookie) {
 	case VDEV_STATE_ONLINE:
-		error = vdev_online(spa, zc->zc_guid, zc->zc_obj, &newstate);
+		error = vdev_online(spa, zc->zc_guid, zc->zc_obj, &newstate,
+		    &spare_guid);
+		if (error == 0 && spare_guid != 0)
+			spa_vdev_detach(spa, spare_guid, 0, B_FALSE);
 		break;
 
 	case VDEV_STATE_OFFLINE: