arc_evict: rework evict thread tuneables

This tries to simplify the relationship between the
zfs_arc_evict_threads and zfs_arc_evict_threads_max tuneables.

zfs_arc_evict_threads works largely as it did before: use this explicit
value, or a compute a reasonable value based on the number of CPUs.

zfs_arc_evict_threads_max, if set, will become the hard limit for all
zfs_arc_evict_threads, computed or explicit. If not set, it will be set
at 4, 16, or the initial explicit zfs_arc_evict_threads, whichever is
smaller.

Taken together, we try to balance given the operator exactly what they
ask for, without creating lots of unused threads that we won't use, and
still allowing a little room to increase the number of threads at
runtime in many scenarios.

Finally, this fixes up handling for an initial thread count of 1. If we
start at 1, but the max is higher, we delay creating the taskq until the
thread count goes to two or more. This is to retain the "single-thread"
behaviour on small systems by default, while still allowing them to
switch to parallel at runtime by raising the thread count.

Sponsored-by: Expensify, Inc.
Sponsored-by: Klara, Inc.
Signed-off-by: Rob Norris <[email protected]>

Author: robn

man/man4/zfs.4

@@ -734,29 +734,53 @@ This batch-style operation prevents entire sub-lists from being evicted at once
 but comes at a cost of additional unlocking and locking.
 .
 .It Sy zfs_arc_evict_threads Ns = Ns Sy 0 Pq int
-Controls the number of ARC eviction threads to be used.
+Sets the number of ARC eviction threads to be used.
 .Pp
-When set to 0, ZFS will compute the number of required eviction threads
-depending on the number of CPU cores (ncpu_max).
-The minimum number of threads is 1 and applies to systems from 1 to 5 CPU cores.
-Systems with 6 CPU cores get 2 eviction threads.
-ZFS on systems larger than that uses log2 of the CPU count
-plus one for each 64 CPUs.
-This way the number of eviction threads scales up more on high CPU counts.
-Currently, ZFS will not scale automatically beyond 16 threads.
+If set greater than 0, ZFS will dedicate up to that many threads to ARC
+eviction.
+Each thread will process one sub-list at a time,
+until the eviction target is reached or all sub-lists have been processed.
+When set to 0, ZFS will compute a reasonable number of eviction threads based
+on the number of CPUs.
 .Pp
-When set to 1, the parallel arc eviction is disabled.
-Only one thread will be used to evict from ARC.
+.TS
+box;
+lb l l .
+	CPUs	Threads
+_
+	1-4	1
+	5-8	2
+	9-15	3
+	16-31	4
+	32-63	6
+	64-95	8
+	96-127	9
+	128-160	11
+	160-191	12
+	192-223	13
+	224-255	14
+	256+	16
+.TE
 .Pp
-When set to a value greater than 1, the value will be used as an exact number
-of eviction threads.
-If changed live, it will be limited by number of threads allocated on module
-load.
+.
+The value can be changed while ZFS is running, but will always be limited to
+a maximum of
+.Sy zfs_arc_evict_threads_max
+set at module load time.
 .Pp
 More threads may improve the responsiveness of ZFS to memory pressure.
 This can be important for performance when eviction from the ARC becomes
 a bottleneck for reads and writes.
 .
+.It Sy zfs_arc_evict_threads_max Ns = Ns Sy 0 Pq int
+Sets the maximum possible number of ARC eviction threads.
+.Pp
+This parameter is only settable at module load time,
+and sets a hard upper limit on
+.Sy zfs_arc_evict_threads .
+If 0, a max will be chosen large enough to accomodate
+.Sy zfs_arc_evict_threads , either computed or provided by the operator.
+.
 .It Sy zfs_arc_grow_retry Ns = Ns Sy 0 Ns s Pq uint
 If set to a non zero value, it will replace the
 .Sy arc_grow_retry

module/zfs/arc.c

@@ -470,22 +470,23 @@ static int zfs_arc_prune_task_threads = 1;
 static taskq_t *arc_flush_taskq;
 
 /*
- * Controls the number of ARC eviction threads.
+ * Controls the number of ARC eviction threads to dispatch sublists to.
+ *
  * Possible values:
  * 0  (auto) compute the number of threads using a logarithmic formula.
  * 1  (disabled) one thread - parallel eviction is disabled.
- * 2+ (manual) set the number manually, limited by zfs_arc_evict_threads_max.
+ * 2+ (manual) set the number manually.
+ *
+ * See arc_evict_thread_init() for how "auto" is computed.
  */
 static uint_t zfs_arc_evict_threads = 0;
 
 /*
- * The number of allocated ARC eviction threads. This limits the maximum value
- * of zfs_arc_evict_threads.
- * The number is set up at module load time and depends on the initial value of
- * zfs_arc_evict_threads. If zfs_arc_evict_threads is set to auto, a logarithmic
- * function is used to compute this value. Otherwise, it is set to max_ncpus.
+ * The max number of ARC eviction threads to use. All provided or computed
+ * values for zfs_arc_evict_threads will be clamped to this. If not set at
+ * module load time, will be computed. See arc_evict_thread_init().
  */
-static uint_t zfs_arc_evict_threads_max;
+static uint_t zfs_arc_evict_threads_max = 0;
 
 /* The 7 states: */
 arc_state_t ARC_anon;
@@ -4086,6 +4087,104 @@ arc_evict_task(void *arg)
 	    eva->eva_marker, eva->eva_spa, eva->eva_bytes);
 }
 
+static uint_t arc_evict_threads = 0;
+static uint_t arc_evict_threads_auto = 0;
+
+static void
+arc_evict_thread_recalc(void)
+{
+	/* Reload from tuneable */
+	uint_t want_threads = zfs_arc_evict_threads;
+	if (want_threads == 0)
+		want_threads = arc_evict_threads_auto;
+
+	/* Clamp to configured maximum */
+	if (want_threads > zfs_arc_evict_threads_max)
+		want_threads = zfs_arc_evict_threads_max;
+
+	arc_evict_threads = want_threads;
+
+	if (arc_evict_threads > 1 && arc_evict_taskq == NULL)
+		/*
+		 * First time the thread count goes over 1, ensure the taskq
+		 * is created. By default, this will happen in the call from
+		 * arc_init() on any system with 6+ CPUs. This is here to
+		 * support a system that started out with 1 thread (either
+		 * explicitly configured or because the system has <6 CPUs),
+		 * which was then raised. This should be a rare case, and we
+		 * do this to avoid precreating a taskq that we won't ever use.
+		 */
+		arc_evict_taskq = taskq_create("arc_evict",
+		    zfs_arc_evict_threads_max, defclsyspri, 0, INT_MAX,
+		    TASKQ_PREPOPULATE);
+}
+
+static void
+arc_evict_thread_init(void)
+{
+	/*
+	 * Compute number of threads we want to use for eviction.
+	 *
+	 * Normally, it's log2(ncpus) + ncpus/32, which gets us to the
+	 * default max of 16 threads at ~256 CPUs.
+	 *
+	 * However, this formula goes to two threads at 4 CPUs, which
+	 * is still rather to low to be really useful, so we just go
+	 * with 1 thread at fewer than 6 cores.
+	 */
+	if (max_ncpus < 6)
+		arc_evict_threads_auto = 1;
+	else
+		arc_evict_threads_auto = 
+		    (highbit64(max_ncpus) - 1) + max_ncpus / 32;
+
+	/*
+	 * If not set, compute an appropriate max possible threads.
+	 */
+	if (zfs_arc_evict_threads_max == 0) {
+		/*
+		 * If they want the computed number of threads, then then
+		 * set the max such that the computer value falls in the
+		 * range of 4 to 16 threads.
+		 *
+		 * 4 threads in the minimum, because this allows machines
+		 * with <16 CPUs to still have room to raise
+		 * zfs_arc_evict_threads at runtime, giving operators of
+		 * smaller machines some room to move.
+		 *
+		 * Above that, we set it to the computed value, so it can
+		 * only be reduced at runtime.
+		 *
+		 * 16 is the hard upper limit for a computed maximum,
+		 * regardless of the number of CPUs. This is the computed value
+		 * for 256-287 cores, at which point you almost certainly
+		 * should be setting these tuneables yourself.
+		 *
+		 * If they have supplied a wanted number of threads, but
+		 * have not explicitly set a maximum, follow the same rules,
+		 * but if they go over 16, set the max there, so they don't
+		 * have to set a separate tuneable just to get their choice
+		 * through.
+		 */
+		if (zfs_arc_evict_threads > 16)
+			zfs_arc_evict_threads_max = zfs_arc_evict_threads;
+		else {
+			uint_t nthreads = zfs_arc_evict_threads ?
+			    zfs_arc_evict_threads : arc_evict_threads_auto;
+
+			zfs_arc_evict_threads_max = MAX(MIN(nthreads, 16), 4);
+		}
+	}
+
+	/* Clamp computed number of threads to maximum */
+	if (arc_evict_threads_auto > zfs_arc_evict_threads_max)
+		arc_evict_threads_auto = zfs_arc_evict_threads_max;
+
+	arc_evict_thread_recalc();
+
+	cmn_err(CE_NOTE, "max %u auto %u threads %u", zfs_arc_evict_threads_max, arc_evict_threads_auto, arc_evict_threads);
+}
+
 /*
  * The minimum number of bytes we can evict at once is a block size.
  * So, SPA_MAXBLOCKSIZE is a reasonable minimal value per an eviction task.
@@ -4118,21 +4217,7 @@ arc_evict_state(arc_state_t *state, arc_buf_contents_t type, uint64_t spa,
 
 	num_sublists = multilist_get_num_sublists(ml);
 
-	/* Compute how many sublists we can evict in parallel */
-	uint_t nthreads = 1;
-	if (arc_evict_taskq) {
-		/* One sublist per thread */
-		nthreads = num_sublists;
-
-		/* Operator has set an ideal number of threads, honour it */
-		if (zfs_arc_evict_threads > 0)
-			nthreads = MIN(nthreads, zfs_arc_evict_threads);
-
-		/* Never more than the max threads */
-		nthreads = MIN(nthreads, zfs_arc_evict_threads_max);
-	}
-
-	boolean_t use_evcttq = nthreads > 1;
+	boolean_t use_evcttq = arc_evict_threads > 1;
 
 	/*
 	 * If we've tried to evict from each sublist, made some
@@ -4156,9 +4241,9 @@ arc_evict_state(arc_state_t *state, arc_buf_contents_t type, uint64_t spa,
 	}
 
 	if (use_evcttq) {
-		eva = kmem_alloc(sizeof (*eva) * nthreads, KM_NOSLEEP);
+		eva = kmem_alloc(sizeof (*eva) * arc_evict_threads, KM_NOSLEEP);
 		if (eva) {
-			for (int i = 0; i < nthreads; i++) {
+			for (int i = 0; i < arc_evict_threads; i++) {
 				taskq_init_ent(&eva[i].eva_tqent);
 				eva[i].eva_ml = ml;
 				eva[i].eva_spa = spa;
@@ -4181,7 +4266,7 @@ arc_evict_state(arc_state_t *state, arc_buf_contents_t type, uint64_t spa,
 	while (total_evicted < bytes) {
 		uint64_t scan_evicted = 0;
 		uint64_t evict = MIN_EVICT_SIZE;
-		uint_t ntasks = nthreads;
+		uint_t ntasks = arc_evict_threads;
 
 		if (use_evcttq) {
 			if (sublists_left < ntasks)
@@ -4283,7 +4368,7 @@ arc_evict_state(arc_state_t *state, arc_buf_contents_t type, uint64_t spa,
 	}
 
 	if (eva)
-		kmem_free(eva, sizeof (*eva) * nthreads);
+		kmem_free(eva, sizeof (*eva) * arc_evict_threads);
 
 	for (int i = 0; i < num_sublists; i++) {
 		multilist_sublist_t *mls = multilist_sublist_lock_idx(ml, i);
@@ -4806,6 +4891,12 @@ arc_evict_cb(void *arg, zthr_t *zthr)
 {
 	(void) arg;
 
+	/*
+	 * Recompute how many sublists we can evict in parallel, as it may
+	 * have been changed by the operator since the last eviction call.
+	 */
+	arc_evict_thread_recalc();
+
 	uint64_t evicted = 0;
 	fstrans_cookie_t cookie = spl_fstrans_mark();
 
@@ -8002,34 +8093,7 @@ arc_init(void)
 	arc_prune_taskq = taskq_create("arc_prune", zfs_arc_prune_task_threads,
 	    defclsyspri, 100, INT_MAX, TASKQ_PREPOPULATE | TASKQ_DYNAMIC);
 
-	if (max_ncpus > 1) {
-		if (zfs_arc_evict_threads == 0) {
-			/*
-			 * Limit the maximum number of threads by 16.
-			 * We reach the limit when max_ncpu == 256.
-			 */
-			uint_t nthreads = MIN((highbit64(max_ncpus) - 1) +
-			    max_ncpus / 32, 16);
-
-			/*
-			 * If there's less then four cores in the system,
-			 * disable parallel eviction entirely. This is somewhat
-			 * arbitrary but at least, this is quite a small
-			 * computer that should probably use those cores for
-			 * something else.
-			 */
-			zfs_arc_evict_threads_max = max_ncpus < 4 ? 1 :
-			    nthreads;
-		} else {
-			zfs_arc_evict_threads_max = max_ncpus / 2;
-		}
-
-		if (zfs_arc_evict_threads_max > 1) {
-			arc_evict_taskq = taskq_create("arc_evict",
-			    zfs_arc_evict_threads_max,
-			    defclsyspri, 0, INT_MAX, TASKQ_PREPOPULATE);
-		}
-	}
+	arc_evict_thread_init();
 
 	list_create(&arc_async_flush_list, sizeof (arc_async_flush_t),
 	    offsetof(arc_async_flush_t, af_node));
@@ -11266,7 +11330,7 @@ ZFS_MODULE_PARAM(zfs_arc, zfs_arc_, prune_task_threads, INT, ZMOD_RW,
 	"Number of arc_prune threads");
 
 ZFS_MODULE_PARAM(zfs_arc, zfs_arc_, evict_threads, UINT, ZMOD_RW,
-	"Controls the number of ARC eviction threads");
+	"Number of threads to use for ARC eviction.");
 
 ZFS_MODULE_PARAM(zfs_arc, zfs_arc_, evict_threads_max, UINT, ZMOD_RD,
-	"The number of allocated ARC eviction threads");
+	"The max number of threads that can be used for ARC eviction");