Use wmsum for arc, abd, dbuf and zfetch statistics.

wmsum was designed exactly for cases like these with many updates
and rare reads.  It allows to completely avoid atomic operations on
congested global variables.

Signed-off-by: Alexander Motin <[email protected]>
Sponsored-By: iXsystems, Inc.

Author: amotin

include/sys/abd_impl.h

@@ -27,6 +27,7 @@
 #define	_ABD_IMPL_H
 
 #include <sys/abd.h>
+#include <sys/wmsum.h>
 
 #ifdef __cplusplus
 extern "C" {
@@ -82,9 +83,8 @@ void abd_iter_unmap(struct abd_iter *);
 /*
  * Helper macros
  */
-#define	ABDSTAT(stat)		(abd_stats.stat.value.ui64)
 #define	ABDSTAT_INCR(stat, val) \
-	atomic_add_64(&abd_stats.stat.value.ui64, (val))
+	wmsum_add(&abd_wmsums.stat, (val))
 #define	ABDSTAT_BUMP(stat)	ABDSTAT_INCR(stat, 1)
 #define	ABDSTAT_BUMPDOWN(stat)	ABDSTAT_INCR(stat, -1)
 

include/sys/arc_impl.h

@@ -33,6 +33,7 @@
 #include <sys/zio_crypt.h>
 #include <sys/zthr.h>
 #include <sys/aggsum.h>
+#include <sys/wmsum.h>
 
 #ifdef __cplusplus
 extern "C" {
@@ -563,7 +564,6 @@ typedef struct arc_stats {
 	kstat_named_t arcstat_c;
 	kstat_named_t arcstat_c_min;
 	kstat_named_t arcstat_c_max;
-	/* Not updated directly; only synced in arc_kstat_update. */
 	kstat_named_t arcstat_size;
 	/*
 	 * Number of compressed bytes stored in the arc_buf_hdr_t's b_pabd.
@@ -592,37 +592,31 @@ typedef struct arc_stats {
 	 * (allocated via arc_buf_hdr_t_full and arc_buf_hdr_t_l2only
 	 * caches), and arc_buf_t structures (allocated via arc_buf_t
 	 * cache).
-	 * Not updated directly; only synced in arc_kstat_update.
 	 */
 	kstat_named_t arcstat_hdr_size;
 	/*
 	 * Number of bytes consumed by ARC buffers of type equal to
 	 * ARC_BUFC_DATA. This is generally consumed by buffers backing
 	 * on disk user data (e.g. plain file contents).
-	 * Not updated directly; only synced in arc_kstat_update.
 	 */
 	kstat_named_t arcstat_data_size;
 	/*
 	 * Number of bytes consumed by ARC buffers of type equal to
 	 * ARC_BUFC_METADATA. This is generally consumed by buffers
 	 * backing on disk data that is used for internal ZFS
 	 * structures (e.g. ZAP, dnode, indirect blocks, etc).
-	 * Not updated directly; only synced in arc_kstat_update.
 	 */
 	kstat_named_t arcstat_metadata_size;
 	/*
 	 * Number of bytes consumed by dmu_buf_impl_t objects.
-	 * Not updated directly; only synced in arc_kstat_update.
 	 */
 	kstat_named_t arcstat_dbuf_size;
 	/*
 	 * Number of bytes consumed by dnode_t objects.
-	 * Not updated directly; only synced in arc_kstat_update.
 	 */
 	kstat_named_t arcstat_dnode_size;
 	/*
 	 * Number of bytes consumed by bonus buffers.
-	 * Not updated directly; only synced in arc_kstat_update.
 	 */
 	kstat_named_t arcstat_bonus_size;
 #if defined(COMPAT_FREEBSD11)
@@ -637,47 +631,41 @@ typedef struct arc_stats {
 	 * arc_anon state. This includes *all* buffers in the arc_anon
 	 * state; e.g. data, metadata, evictable, and unevictable buffers
 	 * are all included in this value.
-	 * Not updated directly; only synced in arc_kstat_update.
 	 */
 	kstat_named_t arcstat_anon_size;
 	/*
 	 * Number of bytes consumed by ARC buffers that meet the
 	 * following criteria: backing buffers of type ARC_BUFC_DATA,
 	 * residing in the arc_anon state, and are eligible for eviction
 	 * (e.g. have no outstanding holds on the buffer).
-	 * Not updated directly; only synced in arc_kstat_update.
 	 */
 	kstat_named_t arcstat_anon_evictable_data;
 	/*
 	 * Number of bytes consumed by ARC buffers that meet the
 	 * following criteria: backing buffers of type ARC_BUFC_METADATA,
 	 * residing in the arc_anon state, and are eligible for eviction
 	 * (e.g. have no outstanding holds on the buffer).
-	 * Not updated directly; only synced in arc_kstat_update.
 	 */
 	kstat_named_t arcstat_anon_evictable_metadata;
 	/*
 	 * Total number of bytes consumed by ARC buffers residing in the
 	 * arc_mru state. This includes *all* buffers in the arc_mru
 	 * state; e.g. data, metadata, evictable, and unevictable buffers
 	 * are all included in this value.
-	 * Not updated directly; only synced in arc_kstat_update.
 	 */
 	kstat_named_t arcstat_mru_size;
 	/*
 	 * Number of bytes consumed by ARC buffers that meet the
 	 * following criteria: backing buffers of type ARC_BUFC_DATA,
 	 * residing in the arc_mru state, and are eligible for eviction
 	 * (e.g. have no outstanding holds on the buffer).
-	 * Not updated directly; only synced in arc_kstat_update.
 	 */
 	kstat_named_t arcstat_mru_evictable_data;
 	/*
 	 * Number of bytes consumed by ARC buffers that meet the
 	 * following criteria: backing buffers of type ARC_BUFC_METADATA,
 	 * residing in the arc_mru state, and are eligible for eviction
 	 * (e.g. have no outstanding holds on the buffer).
-	 * Not updated directly; only synced in arc_kstat_update.
 	 */
 	kstat_named_t arcstat_mru_evictable_metadata;
 	/*
@@ -688,64 +676,55 @@ typedef struct arc_stats {
 	 * don't actually have ARC buffers linked off of these headers.
 	 * Thus, *if* the headers had associated ARC buffers, these
 	 * buffers *would have* consumed this number of bytes.
-	 * Not updated directly; only synced in arc_kstat_update.
 	 */
 	kstat_named_t arcstat_mru_ghost_size;
 	/*
 	 * Number of bytes that *would have been* consumed by ARC
 	 * buffers that are eligible for eviction, of type
 	 * ARC_BUFC_DATA, and linked off the arc_mru_ghost state.
-	 * Not updated directly; only synced in arc_kstat_update.
 	 */
 	kstat_named_t arcstat_mru_ghost_evictable_data;
 	/*
 	 * Number of bytes that *would have been* consumed by ARC
 	 * buffers that are eligible for eviction, of type
 	 * ARC_BUFC_METADATA, and linked off the arc_mru_ghost state.
-	 * Not updated directly; only synced in arc_kstat_update.
 	 */
 	kstat_named_t arcstat_mru_ghost_evictable_metadata;
 	/*
 	 * Total number of bytes consumed by ARC buffers residing in the
 	 * arc_mfu state. This includes *all* buffers in the arc_mfu
 	 * state; e.g. data, metadata, evictable, and unevictable buffers
 	 * are all included in this value.
-	 * Not updated directly; only synced in arc_kstat_update.
 	 */
 	kstat_named_t arcstat_mfu_size;
 	/*
 	 * Number of bytes consumed by ARC buffers that are eligible for
 	 * eviction, of type ARC_BUFC_DATA, and reside in the arc_mfu
 	 * state.
-	 * Not updated directly; only synced in arc_kstat_update.
 	 */
 	kstat_named_t arcstat_mfu_evictable_data;
 	/*
 	 * Number of bytes consumed by ARC buffers that are eligible for
 	 * eviction, of type ARC_BUFC_METADATA, and reside in the
 	 * arc_mfu state.
-	 * Not updated directly; only synced in arc_kstat_update.
 	 */
 	kstat_named_t arcstat_mfu_evictable_metadata;
 	/*
 	 * Total number of bytes that *would have been* consumed by ARC
 	 * buffers in the arc_mfu_ghost state. See the comment above
 	 * arcstat_mru_ghost_size for more details.
-	 * Not updated directly; only synced in arc_kstat_update.
 	 */
 	kstat_named_t arcstat_mfu_ghost_size;
 	/*
 	 * Number of bytes that *would have been* consumed by ARC
 	 * buffers that are eligible for eviction, of type
 	 * ARC_BUFC_DATA, and linked off the arc_mfu_ghost state.
-	 * Not updated directly; only synced in arc_kstat_update.
 	 */
 	kstat_named_t arcstat_mfu_ghost_evictable_data;
 	/*
 	 * Number of bytes that *would have been* consumed by ARC
 	 * buffers that are eligible for eviction, of type
 	 * ARC_BUFC_METADATA, and linked off the arc_mru_ghost state.
-	 * Not updated directly; only synced in arc_kstat_update.
 	 */
 	kstat_named_t arcstat_mfu_ghost_evictable_metadata;
 	kstat_named_t arcstat_l2_hits;
@@ -779,7 +758,6 @@ typedef struct arc_stats {
 	kstat_named_t arcstat_l2_io_error;
 	kstat_named_t arcstat_l2_lsize;
 	kstat_named_t arcstat_l2_psize;
-	/* Not updated directly; only synced in arc_kstat_update. */
 	kstat_named_t arcstat_l2_hdr_size;
 	/*
 	 * Number of L2ARC log blocks written. These are used for restoring the
@@ -860,7 +838,6 @@ typedef struct arc_stats {
 	kstat_named_t arcstat_tempreserve;
 	kstat_named_t arcstat_loaned_bytes;
 	kstat_named_t arcstat_prune;
-	/* Not updated directly; only synced in arc_kstat_update. */
 	kstat_named_t arcstat_meta_used;
 	kstat_named_t arcstat_meta_limit;
 	kstat_named_t arcstat_dnode_limit;
@@ -876,6 +853,96 @@ typedef struct arc_stats {
 	kstat_named_t arcstat_abd_chunk_waste_size;
 } arc_stats_t;
 
+typedef struct arc_wmsums {
+	wmsum_t arcstat_hits;
+	wmsum_t arcstat_misses;
+	wmsum_t arcstat_demand_data_hits;
+	wmsum_t arcstat_demand_data_misses;
+	wmsum_t arcstat_demand_metadata_hits;
+	wmsum_t arcstat_demand_metadata_misses;
+	wmsum_t arcstat_prefetch_data_hits;
+	wmsum_t arcstat_prefetch_data_misses;
+	wmsum_t arcstat_prefetch_metadata_hits;
+	wmsum_t arcstat_prefetch_metadata_misses;
+	wmsum_t arcstat_mru_hits;
+	wmsum_t arcstat_mru_ghost_hits;
+	wmsum_t arcstat_mfu_hits;
+	wmsum_t arcstat_mfu_ghost_hits;
+	wmsum_t arcstat_deleted;
+	wmsum_t arcstat_mutex_miss;
+	wmsum_t arcstat_access_skip;
+	wmsum_t arcstat_evict_skip;
+	wmsum_t arcstat_evict_not_enough;
+	wmsum_t arcstat_evict_l2_cached;
+	wmsum_t arcstat_evict_l2_eligible;
+	wmsum_t arcstat_evict_l2_eligible_mfu;
+	wmsum_t arcstat_evict_l2_eligible_mru;
+	wmsum_t arcstat_evict_l2_ineligible;
+	wmsum_t arcstat_evict_l2_skip;
+	wmsum_t arcstat_hash_collisions;
+	wmsum_t arcstat_hash_chains;
+	aggsum_t arcstat_size;
+	wmsum_t arcstat_compressed_size;
+	wmsum_t arcstat_uncompressed_size;
+	wmsum_t arcstat_overhead_size;
+	wmsum_t arcstat_hdr_size;
+	wmsum_t arcstat_data_size;
+	wmsum_t arcstat_metadata_size;
+	wmsum_t arcstat_dbuf_size;
+	aggsum_t arcstat_dnode_size;
+	wmsum_t arcstat_bonus_size;
+	wmsum_t arcstat_l2_hits;
+	wmsum_t arcstat_l2_misses;
+	wmsum_t arcstat_l2_prefetch_asize;
+	wmsum_t arcstat_l2_mru_asize;
+	wmsum_t arcstat_l2_mfu_asize;
+	wmsum_t arcstat_l2_bufc_data_asize;
+	wmsum_t arcstat_l2_bufc_metadata_asize;
+	wmsum_t arcstat_l2_feeds;
+	wmsum_t arcstat_l2_rw_clash;
+	wmsum_t arcstat_l2_read_bytes;
+	wmsum_t arcstat_l2_write_bytes;
+	wmsum_t arcstat_l2_writes_sent;
+	wmsum_t arcstat_l2_writes_done;
+	wmsum_t arcstat_l2_writes_error;
+	wmsum_t arcstat_l2_writes_lock_retry;
+	wmsum_t arcstat_l2_evict_lock_retry;
+	wmsum_t arcstat_l2_evict_reading;
+	wmsum_t arcstat_l2_evict_l1cached;
+	wmsum_t arcstat_l2_free_on_write;
+	wmsum_t arcstat_l2_abort_lowmem;
+	wmsum_t arcstat_l2_cksum_bad;
+	wmsum_t arcstat_l2_io_error;
+	wmsum_t arcstat_l2_lsize;
+	wmsum_t arcstat_l2_psize;
+	aggsum_t arcstat_l2_hdr_size;
+	wmsum_t arcstat_l2_log_blk_writes;
+	wmsum_t arcstat_l2_log_blk_asize;
+	wmsum_t arcstat_l2_log_blk_count;
+	wmsum_t arcstat_l2_rebuild_success;
+	wmsum_t arcstat_l2_rebuild_abort_unsupported;
+	wmsum_t arcstat_l2_rebuild_abort_io_errors;
+	wmsum_t arcstat_l2_rebuild_abort_dh_errors;
+	wmsum_t arcstat_l2_rebuild_abort_cksum_lb_errors;
+	wmsum_t arcstat_l2_rebuild_abort_lowmem;
+	wmsum_t arcstat_l2_rebuild_size;
+	wmsum_t arcstat_l2_rebuild_asize;
+	wmsum_t arcstat_l2_rebuild_bufs;
+	wmsum_t arcstat_l2_rebuild_bufs_precached;
+	wmsum_t arcstat_l2_rebuild_log_blks;
+	wmsum_t arcstat_memory_throttle_count;
+	wmsum_t arcstat_memory_direct_count;
+	wmsum_t arcstat_memory_indirect_count;
+	wmsum_t arcstat_prune;
+	aggsum_t arcstat_meta_used;
+	wmsum_t arcstat_async_upgrade_sync;
+	wmsum_t arcstat_demand_hit_predictive_prefetch;
+	wmsum_t arcstat_demand_hit_prescient_prefetch;
+	wmsum_t arcstat_raw_size;
+	wmsum_t arcstat_cached_only_in_progress;
+	wmsum_t arcstat_abd_chunk_waste_size;
+} arc_wmsums_t;
+
 typedef struct arc_evict_waiter {
 	list_node_t aew_node;
 	kcondvar_t aew_cv;
@@ -885,7 +952,7 @@ typedef struct arc_evict_waiter {
 #define	ARCSTAT(stat)	(arc_stats.stat.value.ui64)
 
 #define	ARCSTAT_INCR(stat, val) \
-	atomic_add_64(&arc_stats.stat.value.ui64, (val))
+	wmsum_add(&arc_wmsums.stat, (val))
 
 #define	ARCSTAT_BUMP(stat)	ARCSTAT_INCR(stat, 1)
 #define	ARCSTAT_BUMPDOWN(stat)	ARCSTAT_INCR(stat, -1)
@@ -899,14 +966,14 @@ typedef struct arc_evict_waiter {
 
 extern taskq_t *arc_prune_taskq;
 extern arc_stats_t arc_stats;
+extern arc_wmsums_t arc_wmsums;
 extern hrtime_t arc_growtime;
 extern boolean_t arc_warm;
 extern int arc_grow_retry;
 extern int arc_no_grow_shift;
 extern int arc_shrink_shift;
 extern kmutex_t arc_prune_mtx;
 extern list_t arc_prune_list;
-extern aggsum_t arc_size;
 extern arc_state_t	*arc_mfu;
 extern arc_state_t	*arc_mru;
 extern uint_t zfs_arc_pc_percent;