Skip to content

Commit e089098

Browse files
jwk404behlendorf
jwk404
authored and
behlendorf
committed
Assorted fixes for the performance tests
- Bail out early if we're running the perf tests and forget to specify disks. - Allow perf tests to run with any number of disks. - Remove weekly vs. nightly settings - Move variables with common values to perf.shlib - Use zinject to clear the ARC over export/import - Fix dbuf cache size calculation When the meaning of `dbuf_cache_max_bytes` changed, the performance test that covers the dbuf cache started to fail. The test would try to write files for the test using the max possible size of the cache, inevitably filling the pool and failing. This change uses `dbuf_cache_shift` to correctly calculate the dbuf cache size. Reviewed-by: Brian Behlendorf <[email protected]> Reviewed-by: Tony Nguyen <[email protected]> Reviewed-by: George Melikov <[email protected]> Signed-off-by: John Kennedy <[email protected]> Closes #12408
1 parent aa3b9bd commit e089098

14 files changed

+128
-269
lines changed

‎scripts/zfs-tests.sh

Lines changed: 19 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -567,18 +567,17 @@ fi
567567

568568
. "$STF_SUITE/include/default.cfg"
569569

570-
msg
571-
msg "--- Configuration ---"
572-
msg "Runfiles: $RUNFILES"
573-
msg "STF_TOOLS: $STF_TOOLS"
574-
msg "STF_SUITE: $STF_SUITE"
575-
msg "STF_PATH: $STF_PATH"
576-
577570
#
578571
# No DISKS have been provided so a basic file or loopback based devices
579572
# must be created for the test suite to use.
580573
#
581574
if [ -z "${DISKS}" ]; then
575+
#
576+
# If this is a performance run, prevent accidental use of
577+
# loopback devices.
578+
#
579+
[ "$TAGS" = "perf" ] && fail "Running perf tests without disks."
580+
582581
#
583582
# Create sparse files for the test suite. These may be used
584583
# directory or have loopback devices layered on them.
@@ -619,8 +618,14 @@ if [ -z "${DISKS}" ]; then
619618
fi
620619
fi
621620

621+
#
622+
# It may be desirable to test with fewer disks than the default when running
623+
# the performance tests, but the functional tests require at least three.
624+
#
622625
NUM_DISKS=$(echo "${DISKS}" | awk '{print NF}')
623-
[ "$NUM_DISKS" -lt 3 ] && fail "Not enough disks ($NUM_DISKS/3 minimum)"
626+
if [ "$TAGS" != "perf" ]; then
627+
[ "$NUM_DISKS" -lt 3 ] && fail "Not enough disks ($NUM_DISKS/3 minimum)"
628+
fi
624629

625630
#
626631
# Disable SELinux until the ZFS Test Suite has been updated accordingly.
@@ -637,6 +642,12 @@ if [ -e /sys/module/zfs/parameters/zfs_dbgmsg_enable ]; then
637642
sudo /bin/sh -c "echo 0 >/proc/spl/kstat/zfs/dbgmsg"
638643
fi
639644

645+
msg
646+
msg "--- Configuration ---"
647+
msg "Runfiles: $RUNFILES"
648+
msg "STF_TOOLS: $STF_TOOLS"
649+
msg "STF_SUITE: $STF_SUITE"
650+
msg "STF_PATH: $STF_PATH"
640651
msg "FILEDIR: $FILEDIR"
641652
msg "FILES: $FILES"
642653
msg "LOOPBACKS: $LOOPBACKS"

‎tests/zfs-tests/include/tunables.cfg

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ COMPRESSED_ARC_ENABLED compressed_arc_enabled zfs_compressed_arc_enabled
2727
CONDENSE_INDIRECT_COMMIT_ENTRY_DELAY_MS condense.indirect_commit_entry_delay_ms zfs_condense_indirect_commit_entry_delay_ms
2828
CONDENSE_INDIRECT_OBSOLETE_PCT condense.indirect_obsolete_pct zfs_condense_indirect_obsolete_pct
2929
CONDENSE_MIN_MAPPING_BYTES condense.min_mapping_bytes zfs_condense_min_mapping_bytes
30-
DBUF_CACHE_MAX_BYTES dbuf_cache.max_bytes dbuf_cache_max_bytes
30+
DBUF_CACHE_SHIFT dbuf.cache_shift dbuf_cache_shift
3131
DEADMAN_CHECKTIME_MS deadman.checktime_ms zfs_deadman_checktime_ms
3232
DEADMAN_FAILMODE deadman.failmode zfs_deadman_failmode
3333
DEADMAN_SYNCTIME_MS deadman.synctime_ms zfs_deadman_synctime_ms

‎tests/zfs-tests/tests/perf/perf.shlib

Lines changed: 44 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -10,18 +10,18 @@
1010
#
1111

1212
#
13-
# Copyright (c) 2015, 2016 by Delphix. All rights reserved.
13+
# Copyright (c) 2015, 2021 by Delphix. All rights reserved.
1414
# Copyright (c) 2016, Intel Corporation.
1515
#
1616

1717
. $STF_SUITE/include/libtest.shlib
1818

19-
# If neither is specified, do a nightly run.
20-
[[ -z $PERF_REGRESSION_WEEKLY ]] && export PERF_REGRESSION_NIGHTLY=1
21-
22-
# Default runtime for each type of test run.
23-
export PERF_RUNTIME_WEEKLY=$((30 * 60))
24-
export PERF_RUNTIME_NIGHTLY=$((10 * 60))
19+
# Defaults common to all the tests in the regression group
20+
export PERF_RUNTIME=${PERF_RUNTIME:-'180'}
21+
export PERF_RANDSEED=${PERF_RANDSEED:-'1234'}
22+
export PERF_COMPPERCENT=${PERF_COMPPERCENT:-'66'}
23+
export PERF_COMPCHUNK=${PERF_COMPCHUNK:-'4096'}
24+
export PERF_SYNC_TYPES=${PERF_SYNC_TYPES:-'1'}
2525

2626
# Default to JSON for fio output
2727
export PERF_FIO_FORMAT=${PERF_FIO_FORMAT:-'json'}
@@ -90,8 +90,7 @@ function do_fio_run_impl
9090

9191
if $clear_cache; then
9292
# Clear the ARC
93-
zpool export $PERFPOOL
94-
zpool import $PERFPOOL
93+
log_must zinject -a
9594
fi
9695

9796
if [[ -n $ZINJECT_DELAYS ]]; then
@@ -159,8 +158,6 @@ function do_fio_run_impl
159158
# set before launching zfstest to override the defaults.
160159
#
161160
# PERF_RUNTIME: The time in seconds each fio invocation should run.
162-
# PERF_RUNTYPE: A human readable tag that appears in logs. The defaults are
163-
# nightly and weekly.
164161
# PERF_NTHREADS: A list of how many threads each fio invocation will use.
165162
# PERF_SYNC_TYPES: Whether to use (O_SYNC) or not. 1 is sync IO, 0 is async IO.
166163
# PERF_IOSIZES: A list of blocksizes in which each fio invocation will do IO.
@@ -424,22 +421,44 @@ function get_max_arc_size
424421
echo $max_arc_size
425422
}
426423

427-
function get_max_dbuf_cache_size
424+
function get_arc_target
428425
{
429-
typeset -l max_dbuf_cache_size
426+
typeset -l arc_c
427+
428+
if is_freebsd; then
429+
arc_c=$(sysctl -n kstat.zfs.misc.arcstats.c)
430+
elif is_illumos; then
431+
arc_c=$(dtrace -qn 'BEGIN {
432+
printf("%u\n", `arc_stats.arcstat_c.value.ui64);
433+
exit(0);
434+
}')
435+
elif is_linux; then
436+
arc_c=`awk '$1 == "c" { print $3 }' \
437+
/proc/spl/kstat/zfs/arcstats`
438+
fi
439+
440+
[[ $? -eq 0 ]] || log_fail "get_arc_target failed"
441+
442+
echo $arc_c
443+
}
444+
445+
function get_dbuf_cache_size
446+
{
447+
typeset -l dbuf_cache_size dbuf_cache_shift
430448

431449
if is_illumos; then
432-
max_dbuf_cache_size=$(dtrace -qn 'BEGIN {
450+
dbuf_cache_size=$(dtrace -qn 'BEGIN {
433451
printf("%u\n", `dbuf_cache_max_bytes);
434452
exit(0);
435453
}')
436454
else
437-
max_dbuf_cache_size=$(get_tunable DBUF_CACHE_MAX_BYTES)
455+
dbuf_cache_shift=$(get_tunable DBUF_CACHE_SHIFT)
456+
dbuf_cache_size=$(($(get_arc_target) / 2**dbuf_cache_shift))
438457
fi
439458

440-
[[ $? -eq 0 ]] || log_fail "get_max_dbuf_cache_size failed"
459+
[[ $? -eq 0 ]] || log_fail "get_dbuf_cache_size failed"
441460

442-
echo $max_dbuf_cache_size
461+
echo $dbuf_cache_size
443462
}
444463

445464
# Create a file with some information about how this system is configured.
@@ -569,6 +588,14 @@ function pool_to_lun_list
569588
echo $lun_list
570589
}
571590

591+
function print_perf_settings
592+
{
593+
echo "PERF_NTHREADS: $PERF_NTHREADS"
594+
echo "PERF_NTHREADS_PER_FS: $PERF_NTHREADS_PER_FS"
595+
echo "PERF_SYNC_TYPES: $PERF_SYNC_TYPES"
596+
echo "PERF_IOSIZES: $PERF_IOSIZES"
597+
}
598+
572599
# Create a perf_data directory to hold performance statistics and
573600
# configuration information.
574601
export PERF_DATA_DIR=$(get_perf_output_dir)

‎tests/zfs-tests/tests/perf/regression/random_reads.ksh

Lines changed: 6 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
#
1313

1414
#
15-
# Copyright (c) 2015, 2020 by Delphix. All rights reserved.
15+
# Copyright (c) 2015, 2021 by Delphix. All rights reserved.
1616
#
1717

1818
#
@@ -55,28 +55,10 @@ populate_perf_filesystems
5555
# Aim to fill the pool to 50% capacity while accounting for a 3x compressratio.
5656
export TOTAL_SIZE=$(($(get_prop avail $PERFPOOL) * 3 / 2))
5757

58-
# Variables for use by fio.
59-
if [[ -n $PERF_REGRESSION_WEEKLY ]]; then
60-
export PERF_RUNTIME=${PERF_RUNTIME:-$PERF_RUNTIME_WEEKLY}
61-
export PERF_RANDSEED=${PERF_RANDSEED:-'1234'}
62-
export PERF_COMPPERCENT=${PERF_COMPPERCENT:-'66'}
63-
export PERF_COMPCHUNK=${PERF_COMPCHUNK:-'4096'}
64-
export PERF_RUNTYPE=${PERF_RUNTYPE:-'weekly'}
65-
export PERF_NTHREADS=${PERF_NTHREADS:-'8 16 32 64'}
66-
export PERF_NTHREADS_PER_FS=${PERF_NTHREADS_PER_FS:-'0'}
67-
export PERF_SYNC_TYPES=${PERF_SYNC_TYPES:-'1'}
68-
export PERF_IOSIZES=${PERF_IOSIZES:-'8k 64k 128k'}
69-
elif [[ -n $PERF_REGRESSION_NIGHTLY ]]; then
70-
export PERF_RUNTIME=${PERF_RUNTIME:-$PERF_RUNTIME_NIGHTLY}
71-
export PERF_RANDSEED=${PERF_RANDSEED:-'1234'}
72-
export PERF_COMPPERCENT=${PERF_COMPPERCENT:-'66'}
73-
export PERF_COMPCHUNK=${PERF_COMPCHUNK:-'4096'}
74-
export PERF_RUNTYPE=${PERF_RUNTYPE:-'nightly'}
75-
export PERF_NTHREADS=${PERF_NTHREADS:-'16 32'}
76-
export PERF_NTHREADS_PER_FS=${PERF_NTHREADS_PER_FS:-'0'}
77-
export PERF_SYNC_TYPES=${PERF_SYNC_TYPES:-'1'}
78-
export PERF_IOSIZES=${PERF_IOSIZES:-'8k'}
79-
fi
58+
# Variables specific to this test for use by fio.
59+
export PERF_NTHREADS=${PERF_NTHREADS:-'16 32'}
60+
export PERF_NTHREADS_PER_FS=${PERF_NTHREADS_PER_FS:-'0'}
61+
export PERF_IOSIZES=${PERF_IOSIZES:-'8k'}
8062

8163
# Layout the files to be used by the read tests. Create as many files as the
8264
# largest number of threads. An fio run with fewer threads will use a subset
@@ -109,6 +91,6 @@ else
10991
)
11092
fi
11193

112-
log_note "Random reads with $PERF_RUNTYPE settings"
94+
log_note "Random reads with settings: $(print_perf_settings)"
11395
do_fio_run random_reads.fio false true
11496
log_pass "Measure IO stats during random read load"

‎tests/zfs-tests/tests/perf/regression/random_readwrite.ksh

Lines changed: 6 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
#
1313

1414
#
15-
# Copyright (c) 2015, 2020 by Delphix. All rights reserved.
15+
# Copyright (c) 2015, 2021 by Delphix. All rights reserved.
1616
#
1717

1818
#
@@ -55,28 +55,10 @@ populate_perf_filesystems
5555
# Aim to fill the pool to 50% capacity while accounting for a 3x compressratio.
5656
export TOTAL_SIZE=$(($(get_prop avail $PERFPOOL) * 3 / 2))
5757

58-
# Variables for use by fio.
59-
if [[ -n $PERF_REGRESSION_WEEKLY ]]; then
60-
export PERF_RUNTIME=${PERF_RUNTIME:-$PERF_RUNTIME_WEEKLY}
61-
export PERF_RANDSEED=${PERF_RANDSEED:-'1234'}
62-
export PERF_COMPPERCENT=${PERF_COMPPERCENT:-'66'}
63-
export PERF_COMPCHUNK=${PERF_COMPCHUNK:-'4096'}
64-
export PERF_RUNTYPE=${PERF_RUNTYPE:-'weekly'}
65-
export PERF_NTHREADS=${PERF_NTHREADS:-'4 8 16 64'}
66-
export PERF_NTHREADS_PER_FS=${PERF_NTHREADS_PER_FS:-'0'}
67-
export PERF_SYNC_TYPES=${PERF_SYNC_TYPES:-'0 1'}
68-
export PERF_IOSIZES='' # bssplit used instead
69-
elif [[ -n $PERF_REGRESSION_NIGHTLY ]]; then
70-
export PERF_RUNTIME=${PERF_RUNTIME:-$PERF_RUNTIME_NIGHTLY}
71-
export PERF_RANDSEED=${PERF_RANDSEED:-'1234'}
72-
export PERF_COMPPERCENT=${PERF_COMPPERCENT:-'66'}
73-
export PERF_COMPCHUNK=${PERF_COMPCHUNK:-'4096'}
74-
export PERF_RUNTYPE=${PERF_RUNTYPE:-'nightly'}
75-
export PERF_NTHREADS=${PERF_NTHREADS:-'32 64'}
76-
export PERF_NTHREADS_PER_FS=${PERF_NTHREADS_PER_FS:-'0'}
77-
export PERF_SYNC_TYPES=${PERF_SYNC_TYPES:-'1'}
78-
export PERF_IOSIZES='' # bssplit used instead
79-
fi
58+
# Variables specific to this test for use by fio.
59+
export PERF_NTHREADS=${PERF_NTHREADS:-'32 64'}
60+
export PERF_NTHREADS_PER_FS=${PERF_NTHREADS_PER_FS:-'0'}
61+
export PERF_IOSIZES='' # bssplit used instead
8062

8163
# Layout the files to be used by the readwrite tests. Create as many files
8264
# as the largest number of threads. An fio run with fewer threads will use
@@ -109,6 +91,6 @@ else
10991
)
11092
fi
11193

112-
log_note "Random reads and writes with $PERF_RUNTYPE settings"
94+
log_note "Random reads and writes with settings: $(print_perf_settings)"
11395
do_fio_run random_readwrite.fio false true
11496
log_pass "Measure IO stats during random read and write load"

‎tests/zfs-tests/tests/perf/regression/random_readwrite_fixed.ksh

Lines changed: 6 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111
#
1212

1313
#
14-
# Copyright (c) 2017, 2020 by Delphix. All rights reserved.
14+
# Copyright (c) 2017, 2021 by Delphix. All rights reserved.
1515
#
1616

1717
#
@@ -45,28 +45,10 @@ populate_perf_filesystems
4545
# Aim to fill the pool to 50% capacity while accounting for a 3x compressratio.
4646
export TOTAL_SIZE=$(($(get_prop avail $PERFPOOL) * 3 / 2))
4747

48-
# Variables for use by fio.
49-
if [[ -n $PERF_REGRESSION_WEEKLY ]]; then
50-
export PERF_RUNTIME=${PERF_RUNTIME:-$PERF_RUNTIME_WEEKLY}
51-
export PERF_RANDSEED=${PERF_RANDSEED:-'1234'}
52-
export PERF_COMPPERCENT=${PERF_COMPPERCENT:-'66'}
53-
export PERF_COMPCHUNK=${PERF_COMPCHUNK:-'4096'}
54-
export PERF_RUNTYPE=${PERF_RUNTYPE:-'weekly'}
55-
export PERF_NTHREADS=${PERF_NTHREADS:-'8 16 32 64'}
56-
export PERF_NTHREADS_PER_FS=${PERF_NTHREADS_PER_FS:-'0'}
57-
export PERF_SYNC_TYPES=${PERF_SYNC_TYPES:-'0 1'}
58-
export PERF_IOSIZES='8k 64k'
59-
elif [[ -n $PERF_REGRESSION_NIGHTLY ]]; then
60-
export PERF_RUNTIME=${PERF_RUNTIME:-$PERF_RUNTIME_NIGHTLY}
61-
export PERF_RANDSEED=${PERF_RANDSEED:-'1234'}
62-
export PERF_COMPPERCENT=${PERF_COMPPERCENT:-'66'}
63-
export PERF_COMPCHUNK=${PERF_COMPCHUNK:-'4096'}
64-
export PERF_RUNTYPE=${PERF_RUNTYPE:-'nightly'}
65-
export PERF_NTHREADS=${PERF_NTHREADS:-'64 128'}
66-
export PERF_NTHREADS_PER_FS=${PERF_NTHREADS_PER_FS:-'0'}
67-
export PERF_SYNC_TYPES=${PERF_SYNC_TYPES:-'1'}
68-
export PERF_IOSIZES='8k'
69-
fi
48+
# Variables specific to this test for use by fio.
49+
export PERF_NTHREADS=${PERF_NTHREADS:-'64 128'}
50+
export PERF_NTHREADS_PER_FS=${PERF_NTHREADS_PER_FS:-'0'}
51+
export PERF_IOSIZES=${PERF_IOSIZES:-'8k'}
7052

7153
# Layout the files to be used by the readwrite tests. Create as many files
7254
# as the largest number of threads. An fio run with fewer threads will use
@@ -101,6 +83,6 @@ else
10183
)
10284
fi
10385

104-
log_note "Random reads and writes with $PERF_RUNTYPE settings"
86+
log_note "Random reads and writes with settings: $(print_perf_settings)"
10587
do_fio_run random_readwrite_fixed.fio false true
10688
log_pass "Measure IO stats during random read and write load"

‎tests/zfs-tests/tests/perf/regression/random_writes.ksh

Lines changed: 6 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
#
1313

1414
#
15-
# Copyright (c) 2015, 2020 by Delphix. All rights reserved.
15+
# Copyright (c) 2015, 2021 by Delphix. All rights reserved.
1616
#
1717

1818
#
@@ -54,28 +54,10 @@ populate_perf_filesystems
5454
# Aim to fill the pool to 50% capacity while accounting for a 3x compressratio.
5555
export TOTAL_SIZE=$(($(get_prop avail $PERFPOOL) * 3 / 2))
5656

57-
# Variables for use by fio.
58-
if [[ -n $PERF_REGRESSION_WEEKLY ]]; then
59-
export PERF_RUNTIME=${PERF_RUNTIME:-$PERF_RUNTIME_WEEKLY}
60-
export PERF_RANDSEED=${PERF_RANDSEED:-'1234'}
61-
export PERF_COMPPERCENT=${PERF_COMPPERCENT:-'66'}
62-
export PERF_COMPCHUNK=${PERF_COMPCHUNK:-'4096'}
63-
export PERF_RUNTYPE=${PERF_RUNTYPE:-'weekly'}
64-
export PERF_NTHREADS=${PERF_NTHREADS:-'1 4 8 16 32 64 128'}
65-
export PERF_NTHREADS_PER_FS=${PERF_NTHREADS_PER_FS:-'0'}
66-
export PERF_SYNC_TYPES=${PERF_SYNC_TYPES:-'0 1'}
67-
export PERF_IOSIZES=${PERF_IOSIZES:-'8k 64k 256k'}
68-
elif [[ -n $PERF_REGRESSION_NIGHTLY ]]; then
69-
export PERF_RUNTIME=${PERF_RUNTIME:-$PERF_RUNTIME_NIGHTLY}
70-
export PERF_RANDSEED=${PERF_RANDSEED:-'1234'}
71-
export PERF_COMPPERCENT=${PERF_COMPPERCENT:-'66'}
72-
export PERF_COMPCHUNK=${PERF_COMPCHUNK:-'4096'}
73-
export PERF_RUNTYPE=${PERF_RUNTYPE:-'nightly'}
74-
export PERF_NTHREADS=${PERF_NTHREADS:-'32 128'}
75-
export PERF_NTHREADS_PER_FS=${PERF_NTHREADS_PER_FS:-'0'}
76-
export PERF_SYNC_TYPES=${PERF_SYNC_TYPES:-'1'}
77-
export PERF_IOSIZES=${PERF_IOSIZES:-'8k'}
78-
fi
57+
# Variables specific to this test for use by fio.
58+
export PERF_NTHREADS=${PERF_NTHREADS:-'32 128'}
59+
export PERF_NTHREADS_PER_FS=${PERF_NTHREADS_PER_FS:-'0'}
60+
export PERF_IOSIZES=${PERF_IOSIZES:-'8k'}
7961

8062
# Set up the scripts and output files that will log performance data.
8163
lun_list=$(pool_to_lun_list $PERFPOOL)
@@ -100,6 +82,6 @@ else
10082
)
10183
fi
10284

103-
log_note "Random writes with $PERF_RUNTYPE settings"
85+
log_note "Random writes with settings: $(print_perf_settings)"
10486
do_fio_run random_writes.fio true false
10587
log_pass "Measure IO stats during random write load"

0 commit comments

Comments
 (0)