Skip to content

Commit 6a9e406

Browse files
szubersktonyhutter
szubersk
authored and
tonyhutter
committed
Add Linux kmemleak support to ZTS
- Kmemleak `clear` is invoked right before every test case run. - Kmemleak `scan` is requested right after each test case is finished. - Kmemleak instrumentation is not used for setup/cleanup/pretest/posttest/failsafe stages to shorten the test case execution time. - Kmemleak periodic scan is disabled (`scan=0`) before the test suite run to avoid interfering with the on-demand scan results. - There are unavoidable potential false positives coming from kernel areas other than OpenZFS module. - The ZTS with kmemleak enabled duration is increased by ~50%. Example run ``` Running Time: 07:12:13 Percent passed: 98.3% unreferenced object 0xffff9da82aea5410 (size 80): comm "kworker/u32:10", pid 942206, jiffies 4296749716 (age 2615.516s) hex dump (first 32 bytes): 00 30 30 00 00 00 00 00 ff 8f 30 00 00 00 00 00 .00.......0..... 51 e6 77 05 a8 9d ff ff 00 00 00 00 00 00 00 00 Q.w............. backtrace: [<000000005cf1fea2>] alloc_extent_state+0x1d/0xb0 [btrfs] [<0000000083f78ae5>] set_extent_bit+0x2ff/0x670 [btrfs] [<00000000de29249e>] lock_extent_bits+0x6b/0xa0 [btrfs] [<00000000b241f424>] lock_and_cleanup_extent_if_need+0xaf/0x1c0 [btrfs] [<0000000093ca72b5>] btrfs_buffered_write+0x297/0x7d0 [btrfs] [<000000002c2938c8>] btrfs_file_write_iter+0x127/0x390 [btrfs] [<00000000b888f720>] do_iter_readv_writev+0x152/0x1b0 [<00000000320f0bcc>] do_iter_write+0x7c/0x1c0 [<000000000b5a8fe0>] lo_write_bvec+0x62/0x150 [loop] [<000000009aa03c73>] loop_process_work+0x250/0xbd0 [loop] [<00000000c7487d8a>] process_one_work+0x1f1/0x390 [<000000000b236831>] worker_thread+0x53/0x3e0 [<0000000023cb3e57>] kthread+0x127/0x150 [<000000002d48676a>] ret_from_fork+0x22/0x30 ``` Reviewed-by: Brian Behlendorf <[email protected]> Reviewed-by: Ryan Moeller <[email protected]> Signed-off-by: szubersk <[email protected]> Closes openzfs#13084
1 parent 1925025 commit 6a9e406

File tree

3 files changed

+58
-13
lines changed

3 files changed

+58
-13
lines changed

‎scripts/zfs-tests.sh

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@ ZFS_DBGMSG="$STF_SUITE/callbacks/zfs_dbgmsg.ksh"
5353
ZFS_DMESG="$STF_SUITE/callbacks/zfs_dmesg.ksh"
5454
UNAME=$(uname -s)
5555
RERUN=""
56+
KMEMLEAK=""
5657

5758
# Override some defaults if on FreeBSD
5859
if [ "$UNAME" = "FreeBSD" ] ; then
@@ -328,6 +329,7 @@ OPTIONS:
328329
-S Enable stack tracer (negative performance impact)
329330
-c Only create and populate constrained path
330331
-R Automatically rerun failing tests
332+
-m Enable kmemleak reporting (Linux only)
331333
-n NFSFILE Use the nfsfile to determine the NFS configuration
332334
-I NUM Number of iterations
333335
-d DIR Use DIR for files and loopback devices
@@ -354,7 +356,7 @@ $0 -x
354356
EOF
355357
}
356358

357-
while getopts 'hvqxkfScRn:d:s:r:?t:T:u:I:' OPTION; do
359+
while getopts 'hvqxkfScRmn:d:s:r:?t:T:u:I:' OPTION; do
358360
case $OPTION in
359361
h)
360362
usage
@@ -385,6 +387,9 @@ while getopts 'hvqxkfScRn:d:s:r:?t:T:u:I:' OPTION; do
385387
R)
386388
RERUN="yes"
387389
;;
390+
m)
391+
KMEMLEAK="yes"
392+
;;
388393
n)
389394
nfsfile=$OPTARG
390395
[ -f "$nfsfile" ] || fail "Cannot read file: $nfsfile"
@@ -694,12 +699,14 @@ REPORT_FILE=$(mktemp_file zts-report)
694699
#
695700
# Run all the tests as specified.
696701
#
697-
msg "${TEST_RUNNER} ${QUIET:+-q}" \
702+
msg "${TEST_RUNNER}" \
703+
"${QUIET:+-q}" \
704+
"${KMEMLEAK:+-m}" \
698705
"-c \"${RUNFILES}\"" \
699706
"-T \"${TAGS}\"" \
700707
"-i \"${STF_SUITE}\"" \
701708
"-I \"${ITERATIONS}\""
702-
${TEST_RUNNER} ${QUIET:+-q} \
709+
${TEST_RUNNER} ${QUIET:+-q} ${KMEMLEAK:+-m} \
703710
-c "${RUNFILES}" \
704711
-T "${TAGS}" \
705712
-i "${STF_SUITE}" \
@@ -719,7 +726,7 @@ if [ "$RESULT" -eq "2" ] && [ -n "$RERUN" ]; then
719726
for test_name in $MAYBES; do
720727
grep "$test_name " "$TEMP_RESULTS_FILE" >>"$TEST_LIST"
721728
done
722-
${TEST_RUNNER} ${QUIET:+-q} \
729+
${TEST_RUNNER} ${QUIET:+-q} ${KMEMLEAK:+-m} \
723730
-c "${RUNFILES}" \
724731
-T "${TAGS}" \
725732
-i "${STF_SUITE}" \

‎tests/test-runner/bin/test-runner.py.in

Lines changed: 45 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -36,11 +36,13 @@ from pwd import getpwuid
3636
from select import select
3737
from subprocess import PIPE
3838
from subprocess import Popen
39+
from subprocess import check_output
3940
from threading import Timer
4041
from time import time
4142

4243
BASEDIR = '/var/tmp/test_results'
4344
TESTDIR = '/usr/share/zfs/'
45+
KMEMLEAK_FILE = '/sys/kernel/debug/kmemleak'
4446
KILL = 'kill'
4547
TRUE = 'true'
4648
SUDO = 'sudo'
@@ -83,6 +85,7 @@ class Result(object):
8385
self.runtime = ''
8486
self.stdout = []
8587
self.stderr = []
88+
self.kmemleak = ''
8689
self.result = ''
8790

8891
def done(self, proc, killed, reran):
@@ -98,6 +101,9 @@ class Result(object):
98101
if killed:
99102
self.result = 'KILLED'
100103
Result.runresults['KILLED'] += 1
104+
elif len(self.kmemleak) > 0:
105+
self.result = 'FAIL'
106+
Result.runresults['FAIL'] += 1
101107
elif self.returncode == 0:
102108
self.result = 'PASS'
103109
Result.runresults['PASS'] += 1
@@ -258,7 +264,7 @@ User: %s
258264

259265
return out.lines, err.lines
260266

261-
def run(self, dryrun):
267+
def run(self, dryrun, kmemleak):
262268
"""
263269
This is the main function that runs each individual test.
264270
Determine whether or not the command requires sudo, and modify it
@@ -278,6 +284,11 @@ User: %s
278284
fail('%s' % e)
279285

280286
self.result.starttime = monotonic_time()
287+
288+
if kmemleak:
289+
cmd = f'echo clear | {SUDO} tee {KMEMLEAK_FILE}'
290+
check_output(cmd, shell=True)
291+
281292
proc = Popen(privcmd, stdout=PIPE, stderr=PIPE)
282293
# Allow a special timeout value of 0 to mean infinity
283294
if int(self.timeout) == 0:
@@ -287,6 +298,12 @@ User: %s
287298
try:
288299
t.start()
289300
self.result.stdout, self.result.stderr = self.collect_output(proc)
301+
302+
if kmemleak:
303+
cmd = f'echo scan | {SUDO} tee {KMEMLEAK_FILE}'
304+
check_output(cmd, shell=True)
305+
cmd = f'{SUDO} cat {KMEMLEAK_FILE}'
306+
self.result.kmemleak = check_output(cmd, shell=True)
290307
except KeyboardInterrupt:
291308
self.kill_cmd(proc, True)
292309
fail('\nRun terminated at user request.')
@@ -363,6 +380,9 @@ User: %s
363380
with open(os.path.join(self.outputdir, 'merged'), 'wb') as merged:
364381
for _, line in lines:
365382
os.write(merged.fileno(), b'%s\n' % line)
383+
if len(self.result.kmemleak):
384+
with open(os.path.join(self.outputdir, 'kmemleak'), 'wb') as kmem:
385+
kmem.write(self.result.kmemleak)
366386

367387

368388
class Test(Cmd):
@@ -447,22 +467,22 @@ Tags: %s
447467

448468
cont = True
449469
if len(pretest.pathname):
450-
pretest.run(options.dryrun)
470+
pretest.run(options.dryrun, False)
451471
cont = pretest.result.result == 'PASS'
452472
pretest.log(options)
453473

454474
if cont:
455-
test.run(options.dryrun)
475+
test.run(options.dryrun, options.kmemleak)
456476
if test.result.result == 'KILLED' and len(failsafe.pathname):
457-
failsafe.run(options.dryrun)
477+
failsafe.run(options.dryrun, False)
458478
failsafe.log(options, suppress_console=True)
459479
else:
460480
test.skip()
461481

462482
test.log(options)
463483

464484
if len(posttest.pathname):
465-
posttest.run(options.dryrun)
485+
posttest.run(options.dryrun, False)
466486
posttest.log(options)
467487

468488

@@ -565,7 +585,7 @@ Tags: %s
565585

566586
cont = True
567587
if len(pretest.pathname):
568-
pretest.run(options.dryrun)
588+
pretest.run(options.dryrun, False)
569589
cont = pretest.result.result == 'PASS'
570590
pretest.log(options)
571591

@@ -578,17 +598,17 @@ Tags: %s
578598
failsafe = Cmd(self.failsafe, outputdir=odir, timeout=self.timeout,
579599
user=self.failsafe_user, identifier=self.identifier)
580600
if cont:
581-
test.run(options.dryrun)
601+
test.run(options.dryrun, options.kmemleak)
582602
if test.result.result == 'KILLED' and len(failsafe.pathname):
583-
failsafe.run(options.dryrun)
603+
failsafe.run(options.dryrun, False)
584604
failsafe.log(options, suppress_console=True)
585605
else:
586606
test.skip()
587607

588608
test.log(options)
589609

590610
if len(posttest.pathname):
591-
posttest.run(options.dryrun)
611+
posttest.run(options.dryrun, False)
592612
posttest.log(options)
593613

594614

@@ -853,6 +873,11 @@ class TestRun(object):
853873
else:
854874
write_log('Could not make a symlink to directory %s\n' %
855875
self.outputdir, LOG_ERR)
876+
877+
if options.kmemleak:
878+
cmd = f'echo scan=0 | {SUDO} tee {KMEMLEAK_FILE}'
879+
check_output(cmd, shell=True)
880+
856881
iteration = 0
857882
while iteration < options.iterations:
858883
for test in sorted(self.tests.keys()):
@@ -998,6 +1023,14 @@ def fail(retstr, ret=1):
9981023
exit(ret)
9991024

10001025

1026+
def kmemleak_cb(option, opt_str, value, parser):
1027+
if not os.path.exists(KMEMLEAK_FILE):
1028+
fail(f"File '{KMEMLEAK_FILE}' doesn't exist. " +
1029+
"Enable CONFIG_DEBUG_KMEMLEAK in kernel configuration.")
1030+
1031+
setattr(parser.values, option.dest, True)
1032+
1033+
10011034
def options_cb(option, opt_str, value, parser):
10021035
path_options = ['outputdir', 'template', 'testdir', 'logfile']
10031036

@@ -1035,6 +1068,9 @@ def parse_args():
10351068
parser.add_option('-i', action='callback', callback=options_cb,
10361069
default=TESTDIR, dest='testdir', type='string',
10371070
metavar='testdir', help='Specify a test directory.')
1071+
parser.add_option('-m', action='callback', callback=kmemleak_cb,
1072+
default=False, dest='kmemleak',
1073+
help='Enable kmemleak reporting (Linux only)')
10381074
parser.add_option('-p', action='callback', callback=options_cb,
10391075
default='', dest='pre', metavar='script',
10401076
type='string', help='Specify a pre script.')

‎tests/test-runner/man/test-runner.1

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -210,6 +210,8 @@ to be consumed by the run command.
210210
.It Fl d
211211
Dry run mode.
212212
Execute no tests, but print a description of each test that would have been run.
213+
.It Fl m
214+
Enable kmemleak reporting (Linux only)
213215
.It Fl g
214216
Create test groups from any directories found while searching for tests.
215217
.It Fl o Ar outputdir

0 commit comments

Comments
 (0)