Skip to content

Commit d21252e

Browse files
amotinamotin
committed
Re-embed multilist_t storage.
Before "7968 multi-threaded spa_sync()" commit multilists were stored in caller's memory same as regular lists. It is not clear why it was changed, considering that the structure is still public, but I found it reduced performance by additional uncached pointer dereference. This commit partially reverts the mentioned one and adds some cache line alignments to separate read-only multilists and heavily modified refcount's to different cache lines. Signed-off-by: Alexander Motin <[email protected]> Sponsored-By: iXsystems, Inc.
1 parent 45516b4 commit d21252e

File tree

12 files changed

+99
-104
lines changed

12 files changed

+99
-104
lines changed

‎include/sys/arc_impl.h

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -74,20 +74,20 @@ typedef struct arc_state {
7474
/*
7575
* list of evictable buffers
7676
*/
77-
multilist_t *arcs_list[ARC_BUFC_NUMTYPES];
77+
multilist_t arcs_list[ARC_BUFC_NUMTYPES];
78+
/*
79+
* supports the "dbufs" kstat
80+
*/
81+
arc_state_type_t arcs_state;
7882
/*
7983
* total amount of evictable data in this state
8084
*/
81-
zfs_refcount_t arcs_esize[ARC_BUFC_NUMTYPES];
85+
zfs_refcount_t arcs_esize[ARC_BUFC_NUMTYPES] ____cacheline_aligned;
8286
/*
8387
* total amount of data in this state; this includes: evictable,
8488
* non-evictable, ARC_BUFC_DATA, and ARC_BUFC_METADATA.
8589
*/
8690
zfs_refcount_t arcs_size;
87-
/*
88-
* supports the "dbufs" kstat
89-
*/
90-
arc_state_type_t arcs_state;
9191
} arc_state_t;
9292

9393
typedef struct arc_callback arc_callback_t;

‎include/sys/dmu_objset.h

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -153,7 +153,7 @@ struct objset {
153153
/* no lock needed: */
154154
struct dmu_tx *os_synctx; /* XXX sketchy */
155155
zil_header_t os_zil_header;
156-
multilist_t *os_synced_dnodes;
156+
multilist_t os_synced_dnodes;
157157
uint64_t os_flags;
158158
uint64_t os_freed_dnodes;
159159
boolean_t os_rescan_dnodes;
@@ -172,7 +172,7 @@ struct objset {
172172

173173
/* Protected by os_lock */
174174
kmutex_t os_lock;
175-
multilist_t *os_dirty_dnodes[TXG_SIZE];
175+
multilist_t os_dirty_dnodes[TXG_SIZE];
176176
list_t os_dnodes;
177177
list_t os_downgraded_dbufs;
178178

‎include/sys/metaslab_impl.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -206,7 +206,7 @@ struct metaslab_class {
206206
* List of all loaded metaslabs in the class, sorted in order of most
207207
* recent use.
208208
*/
209-
multilist_t *mc_metaslab_txg_list;
209+
multilist_t mc_metaslab_txg_list;
210210

211211
metaslab_class_allocator_t mc_allocator[];
212212
};

‎include/sys/multilist.h

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -71,8 +71,9 @@ struct multilist {
7171
multilist_sublist_index_func_t *ml_index_func;
7272
};
7373

74+
void multilist_create(multilist_t *, size_t, size_t,
75+
multilist_sublist_index_func_t *);
7476
void multilist_destroy(multilist_t *);
75-
multilist_t *multilist_create(size_t, size_t, multilist_sublist_index_func_t *);
7677

7778
void multilist_insert(multilist_t *, void *);
7879
void multilist_remove(multilist_t *, void *);

‎module/zfs/arc.c

Lines changed: 43 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -2327,7 +2327,7 @@ add_reference(arc_buf_hdr_t *hdr, void *tag)
23272327
(state != arc_anon)) {
23282328
/* We don't use the L2-only state list. */
23292329
if (state != arc_l2c_only) {
2330-
multilist_remove(state->arcs_list[arc_buf_type(hdr)],
2330+
multilist_remove(&state->arcs_list[arc_buf_type(hdr)],
23312331
hdr);
23322332
arc_evictable_space_decrement(hdr, state);
23332333
}
@@ -2361,7 +2361,7 @@ remove_reference(arc_buf_hdr_t *hdr, kmutex_t *hash_lock, void *tag)
23612361
*/
23622362
if (((cnt = zfs_refcount_remove(&hdr->b_l1hdr.b_refcnt, tag)) == 0) &&
23632363
(state != arc_anon)) {
2364-
multilist_insert(state->arcs_list[arc_buf_type(hdr)], hdr);
2364+
multilist_insert(&state->arcs_list[arc_buf_type(hdr)], hdr);
23652365
ASSERT3U(hdr->b_l1hdr.b_bufcnt, >, 0);
23662366
arc_evictable_space_increment(hdr, state);
23672367
}
@@ -2464,7 +2464,7 @@ arc_change_state(arc_state_t *new_state, arc_buf_hdr_t *hdr,
24642464
if (refcnt == 0) {
24652465
if (old_state != arc_anon && old_state != arc_l2c_only) {
24662466
ASSERT(HDR_HAS_L1HDR(hdr));
2467-
multilist_remove(old_state->arcs_list[buftype], hdr);
2467+
multilist_remove(&old_state->arcs_list[buftype], hdr);
24682468

24692469
if (GHOST_STATE(old_state)) {
24702470
ASSERT0(bufcnt);
@@ -2481,7 +2481,7 @@ arc_change_state(arc_state_t *new_state, arc_buf_hdr_t *hdr,
24812481
* beforehand.
24822482
*/
24832483
ASSERT(HDR_HAS_L1HDR(hdr));
2484-
multilist_insert(new_state->arcs_list[buftype], hdr);
2484+
multilist_insert(&new_state->arcs_list[buftype], hdr);
24852485

24862486
if (GHOST_STATE(new_state)) {
24872487
ASSERT0(bufcnt);
@@ -2633,8 +2633,8 @@ arc_change_state(arc_state_t *new_state, arc_buf_hdr_t *hdr,
26332633
* L2 headers should never be on the L2 state list since they don't
26342634
* have L1 headers allocated.
26352635
*/
2636-
ASSERT(multilist_is_empty(arc_l2c_only->arcs_list[ARC_BUFC_DATA]) &&
2637-
multilist_is_empty(arc_l2c_only->arcs_list[ARC_BUFC_METADATA]));
2636+
ASSERT(multilist_is_empty(&arc_l2c_only->arcs_list[ARC_BUFC_DATA]) &&
2637+
multilist_is_empty(&arc_l2c_only->arcs_list[ARC_BUFC_METADATA]));
26382638
}
26392639

26402640
void
@@ -4205,7 +4205,7 @@ arc_evict_state(arc_state_t *state, uint64_t spa, int64_t bytes,
42054205
arc_buf_contents_t type)
42064206
{
42074207
uint64_t total_evicted = 0;
4208-
multilist_t *ml = state->arcs_list[type];
4208+
multilist_t *ml = &state->arcs_list[type];
42094209
int num_sublists;
42104210
arc_buf_hdr_t **markers;
42114211

@@ -4539,8 +4539,8 @@ arc_evict_meta(uint64_t meta_used)
45394539
static arc_buf_contents_t
45404540
arc_evict_type(arc_state_t *state)
45414541
{
4542-
multilist_t *data_ml = state->arcs_list[ARC_BUFC_DATA];
4543-
multilist_t *meta_ml = state->arcs_list[ARC_BUFC_METADATA];
4542+
multilist_t *data_ml = &state->arcs_list[ARC_BUFC_DATA];
4543+
multilist_t *meta_ml = &state->arcs_list[ARC_BUFC_METADATA];
45444544
int data_idx = multilist_get_random_index(data_ml);
45454545
int meta_idx = multilist_get_random_index(meta_ml);
45464546
multilist_sublist_t *data_mls;
@@ -7460,44 +7460,44 @@ arc_state_init(void)
74607460
arc_mfu_ghost = &ARC_mfu_ghost;
74617461
arc_l2c_only = &ARC_l2c_only;
74627462

7463-
arc_mru->arcs_list[ARC_BUFC_METADATA] =
7464-
multilist_create(sizeof (arc_buf_hdr_t),
7463+
multilist_create(&arc_mru->arcs_list[ARC_BUFC_METADATA],
7464+
sizeof (arc_buf_hdr_t),
74657465
offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
74667466
arc_state_multilist_index_func);
7467-
arc_mru->arcs_list[ARC_BUFC_DATA] =
7468-
multilist_create(sizeof (arc_buf_hdr_t),
7467+
multilist_create(&arc_mru->arcs_list[ARC_BUFC_DATA],
7468+
sizeof (arc_buf_hdr_t),
74697469
offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
74707470
arc_state_multilist_index_func);
7471-
arc_mru_ghost->arcs_list[ARC_BUFC_METADATA] =
7472-
multilist_create(sizeof (arc_buf_hdr_t),
7471+
multilist_create(&arc_mru_ghost->arcs_list[ARC_BUFC_METADATA],
7472+
sizeof (arc_buf_hdr_t),
74737473
offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
74747474
arc_state_multilist_index_func);
7475-
arc_mru_ghost->arcs_list[ARC_BUFC_DATA] =
7476-
multilist_create(sizeof (arc_buf_hdr_t),
7475+
multilist_create(&arc_mru_ghost->arcs_list[ARC_BUFC_DATA],
7476+
sizeof (arc_buf_hdr_t),
74777477
offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
74787478
arc_state_multilist_index_func);
7479-
arc_mfu->arcs_list[ARC_BUFC_METADATA] =
7480-
multilist_create(sizeof (arc_buf_hdr_t),
7479+
multilist_create(&arc_mfu->arcs_list[ARC_BUFC_METADATA],
7480+
sizeof (arc_buf_hdr_t),
74817481
offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
74827482
arc_state_multilist_index_func);
7483-
arc_mfu->arcs_list[ARC_BUFC_DATA] =
7484-
multilist_create(sizeof (arc_buf_hdr_t),
7483+
multilist_create(&arc_mfu->arcs_list[ARC_BUFC_DATA],
7484+
sizeof (arc_buf_hdr_t),
74857485
offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
74867486
arc_state_multilist_index_func);
7487-
arc_mfu_ghost->arcs_list[ARC_BUFC_METADATA] =
7488-
multilist_create(sizeof (arc_buf_hdr_t),
7487+
multilist_create(&arc_mfu_ghost->arcs_list[ARC_BUFC_METADATA],
7488+
sizeof (arc_buf_hdr_t),
74897489
offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
74907490
arc_state_multilist_index_func);
7491-
arc_mfu_ghost->arcs_list[ARC_BUFC_DATA] =
7492-
multilist_create(sizeof (arc_buf_hdr_t),
7491+
multilist_create(&arc_mfu_ghost->arcs_list[ARC_BUFC_DATA],
7492+
sizeof (arc_buf_hdr_t),
74937493
offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
74947494
arc_state_multilist_index_func);
7495-
arc_l2c_only->arcs_list[ARC_BUFC_METADATA] =
7496-
multilist_create(sizeof (arc_buf_hdr_t),
7495+
multilist_create(&arc_l2c_only->arcs_list[ARC_BUFC_METADATA],
7496+
sizeof (arc_buf_hdr_t),
74977497
offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
74987498
arc_state_multilist_index_func);
7499-
arc_l2c_only->arcs_list[ARC_BUFC_DATA] =
7500-
multilist_create(sizeof (arc_buf_hdr_t),
7499+
multilist_create(&arc_l2c_only->arcs_list[ARC_BUFC_DATA],
7500+
sizeof (arc_buf_hdr_t),
75017501
offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
75027502
arc_state_multilist_index_func);
75037503

@@ -7563,16 +7563,16 @@ arc_state_fini(void)
75637563
zfs_refcount_destroy(&arc_mfu_ghost->arcs_size);
75647564
zfs_refcount_destroy(&arc_l2c_only->arcs_size);
75657565

7566-
multilist_destroy(arc_mru->arcs_list[ARC_BUFC_METADATA]);
7567-
multilist_destroy(arc_mru_ghost->arcs_list[ARC_BUFC_METADATA]);
7568-
multilist_destroy(arc_mfu->arcs_list[ARC_BUFC_METADATA]);
7569-
multilist_destroy(arc_mfu_ghost->arcs_list[ARC_BUFC_METADATA]);
7570-
multilist_destroy(arc_mru->arcs_list[ARC_BUFC_DATA]);
7571-
multilist_destroy(arc_mru_ghost->arcs_list[ARC_BUFC_DATA]);
7572-
multilist_destroy(arc_mfu->arcs_list[ARC_BUFC_DATA]);
7573-
multilist_destroy(arc_mfu_ghost->arcs_list[ARC_BUFC_DATA]);
7574-
multilist_destroy(arc_l2c_only->arcs_list[ARC_BUFC_METADATA]);
7575-
multilist_destroy(arc_l2c_only->arcs_list[ARC_BUFC_DATA]);
7566+
multilist_destroy(&arc_mru->arcs_list[ARC_BUFC_METADATA]);
7567+
multilist_destroy(&arc_mru_ghost->arcs_list[ARC_BUFC_METADATA]);
7568+
multilist_destroy(&arc_mfu->arcs_list[ARC_BUFC_METADATA]);
7569+
multilist_destroy(&arc_mfu_ghost->arcs_list[ARC_BUFC_METADATA]);
7570+
multilist_destroy(&arc_mru->arcs_list[ARC_BUFC_DATA]);
7571+
multilist_destroy(&arc_mru_ghost->arcs_list[ARC_BUFC_DATA]);
7572+
multilist_destroy(&arc_mfu->arcs_list[ARC_BUFC_DATA]);
7573+
multilist_destroy(&arc_mfu_ghost->arcs_list[ARC_BUFC_DATA]);
7574+
multilist_destroy(&arc_l2c_only->arcs_list[ARC_BUFC_METADATA]);
7575+
multilist_destroy(&arc_l2c_only->arcs_list[ARC_BUFC_DATA]);
75767576

75777577
aggsum_fini(&arc_meta_used);
75787578
aggsum_fini(&arc_size);
@@ -8629,16 +8629,16 @@ l2arc_sublist_lock(int list_num)
86298629

86308630
switch (list_num) {
86318631
case 0:
8632-
ml = arc_mfu->arcs_list[ARC_BUFC_METADATA];
8632+
ml = &arc_mfu->arcs_list[ARC_BUFC_METADATA];
86338633
break;
86348634
case 1:
8635-
ml = arc_mru->arcs_list[ARC_BUFC_METADATA];
8635+
ml = &arc_mru->arcs_list[ARC_BUFC_METADATA];
86368636
break;
86378637
case 2:
8638-
ml = arc_mfu->arcs_list[ARC_BUFC_DATA];
8638+
ml = &arc_mfu->arcs_list[ARC_BUFC_DATA];
86398639
break;
86408640
case 3:
8641-
ml = arc_mru->arcs_list[ARC_BUFC_DATA];
8641+
ml = &arc_mru->arcs_list[ARC_BUFC_DATA];
86428642
break;
86438643
default:
86448644
return (NULL);

‎module/zfs/dbuf.c

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -203,8 +203,8 @@ static boolean_t dbuf_evict_thread_exit;
203203
* by those caches' matching enum values (from dbuf_cached_state_t).
204204
*/
205205
typedef struct dbuf_cache {
206-
multilist_t *cache;
207-
zfs_refcount_t size;
206+
multilist_t cache;
207+
zfs_refcount_t size ____cacheline_aligned;
208208
} dbuf_cache_t;
209209
dbuf_cache_t dbuf_caches[DB_CACHE_MAX];
210210

@@ -667,9 +667,9 @@ dbuf_cache_above_lowater(void)
667667
static void
668668
dbuf_evict_one(void)
669669
{
670-
int idx = multilist_get_random_index(dbuf_caches[DB_DBUF_CACHE].cache);
670+
int idx = multilist_get_random_index(&dbuf_caches[DB_DBUF_CACHE].cache);
671671
multilist_sublist_t *mls = multilist_sublist_lock(
672-
dbuf_caches[DB_DBUF_CACHE].cache, idx);
672+
&dbuf_caches[DB_DBUF_CACHE].cache, idx);
673673

674674
ASSERT(!MUTEX_HELD(&dbuf_evict_lock));
675675

@@ -833,8 +833,8 @@ dbuf_init(void)
833833
dbu_evict_taskq = taskq_create("dbu_evict", 1, defclsyspri, 0, 0, 0);
834834

835835
for (dbuf_cached_state_t dcs = 0; dcs < DB_CACHE_MAX; dcs++) {
836-
dbuf_caches[dcs].cache =
837-
multilist_create(sizeof (dmu_buf_impl_t),
836+
multilist_create(&dbuf_caches[dcs].cache,
837+
sizeof (dmu_buf_impl_t),
838838
offsetof(dmu_buf_impl_t, db_cache_link),
839839
dbuf_cache_multilist_index_func);
840840
zfs_refcount_create(&dbuf_caches[dcs].size);
@@ -901,7 +901,7 @@ dbuf_fini(void)
901901

902902
for (dbuf_cached_state_t dcs = 0; dcs < DB_CACHE_MAX; dcs++) {
903903
zfs_refcount_destroy(&dbuf_caches[dcs].size);
904-
multilist_destroy(dbuf_caches[dcs].cache);
904+
multilist_destroy(&dbuf_caches[dcs].cache);
905905
}
906906

907907
if (dbuf_ksp != NULL) {
@@ -2757,7 +2757,7 @@ dbuf_destroy(dmu_buf_impl_t *db)
27572757
ASSERT(db->db_caching_status == DB_DBUF_CACHE ||
27582758
db->db_caching_status == DB_DBUF_METADATA_CACHE);
27592759

2760-
multilist_remove(dbuf_caches[db->db_caching_status].cache, db);
2760+
multilist_remove(&dbuf_caches[db->db_caching_status].cache, db);
27612761
(void) zfs_refcount_remove_many(
27622762
&dbuf_caches[db->db_caching_status].size,
27632763
db->db.db_size, db);
@@ -3467,7 +3467,7 @@ dbuf_hold_impl(dnode_t *dn, uint8_t level, uint64_t blkid,
34673467
ASSERT(db->db_caching_status == DB_DBUF_CACHE ||
34683468
db->db_caching_status == DB_DBUF_METADATA_CACHE);
34693469

3470-
multilist_remove(dbuf_caches[db->db_caching_status].cache, db);
3470+
multilist_remove(&dbuf_caches[db->db_caching_status].cache, db);
34713471
(void) zfs_refcount_remove_many(
34723472
&dbuf_caches[db->db_caching_status].size,
34733473
db->db.db_size, db);
@@ -3709,7 +3709,7 @@ dbuf_rele_and_unlock(dmu_buf_impl_t *db, void *tag, boolean_t evicting)
37093709
DB_DBUF_METADATA_CACHE : DB_DBUF_CACHE;
37103710
db->db_caching_status = dcs;
37113711

3712-
multilist_insert(dbuf_caches[dcs].cache, db);
3712+
multilist_insert(&dbuf_caches[dcs].cache, db);
37133713
size = zfs_refcount_add_many(
37143714
&dbuf_caches[dcs].size,
37153715
db->db.db_size, db);

0 commit comments

Comments
 (0)