Skip to content

Commit 92c6475

Browse files
szuberskandrewc12
szubersk
authored and
andrewc12
committed
Add Linux kmemleak support to ZTS
- Kmemleak `clear` is invoked right before every test case run. - Kmemleak `scan` is requested right after each test case is finished. - Kmemleak instrumentation is not used for setup/cleanup/pretest/posttest/failsafe stages to shorten the test case execution time. - Kmemleak periodic scan is disabled (`scan=0`) before the test suite run to avoid interfering with the on-demand scan results. - There are unavoidable potential false positives coming from kernel areas other than OpenZFS module. - The ZTS with kmemleak enabled duration is increased by ~50%. Example run ``` Running Time: 07:12:13 Percent passed: 98.3% unreferenced object 0xffff9da82aea5410 (size 80): comm "kworker/u32:10", pid 942206, jiffies 4296749716 (age 2615.516s) hex dump (first 32 bytes): 00 30 30 00 00 00 00 00 ff 8f 30 00 00 00 00 00 .00.......0..... 51 e6 77 05 a8 9d ff ff 00 00 00 00 00 00 00 00 Q.w............. backtrace: [<000000005cf1fea2>] alloc_extent_state+0x1d/0xb0 [btrfs] [<0000000083f78ae5>] set_extent_bit+0x2ff/0x670 [btrfs] [<00000000de29249e>] lock_extent_bits+0x6b/0xa0 [btrfs] [<00000000b241f424>] lock_and_cleanup_extent_if_need+0xaf/0x1c0 [btrfs] [<0000000093ca72b5>] btrfs_buffered_write+0x297/0x7d0 [btrfs] [<000000002c2938c8>] btrfs_file_write_iter+0x127/0x390 [btrfs] [<00000000b888f720>] do_iter_readv_writev+0x152/0x1b0 [<00000000320f0bcc>] do_iter_write+0x7c/0x1c0 [<000000000b5a8fe0>] lo_write_bvec+0x62/0x150 [loop] [<000000009aa03c73>] loop_process_work+0x250/0xbd0 [loop] [<00000000c7487d8a>] process_one_work+0x1f1/0x390 [<000000000b236831>] worker_thread+0x53/0x3e0 [<0000000023cb3e57>] kthread+0x127/0x150 [<000000002d48676a>] ret_from_fork+0x22/0x30 ``` Reviewed-by: Brian Behlendorf <[email protected]> Reviewed-by: Ryan Moeller <[email protected]> Signed-off-by: szubersk <[email protected]> Closes openzfs#13084
1 parent 03bd520 commit 92c6475

File tree

3 files changed

+62
-13
lines changed

3 files changed

+62
-13
lines changed

‎scripts/zfs-tests.sh

Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@ ZFS_DBGMSG="$STF_SUITE/callbacks/zfs_dbgmsg.ksh"
5454
ZFS_DMESG="$STF_SUITE/callbacks/zfs_dmesg.ksh"
5555
UNAME=$(uname -s)
5656
RERUN=""
57+
KMEMLEAK=""
5758

5859
# Override some defaults if on FreeBSD
5960
if [ "$UNAME" = "FreeBSD" ] ; then
@@ -329,6 +330,7 @@ OPTIONS:
329330
-S Enable stack tracer (negative performance impact)
330331
-c Only create and populate constrained path
331332
-R Automatically rerun failing tests
333+
-m Enable kmemleak reporting (Linux only)
332334
-n NFSFILE Use the nfsfile to determine the NFS configuration
333335
-I NUM Number of iterations
334336
-d DIR Use DIR for files and loopback devices
@@ -355,7 +357,7 @@ $0 -x
355357
EOF
356358
}
357359

358-
while getopts 'hvqxkfScRn:d:s:r:?t:T:u:I:' OPTION; do
360+
while getopts 'hvqxkfScRmn:d:s:r:?t:T:u:I:' OPTION; do
359361
case $OPTION in
360362
h)
361363
usage
@@ -386,6 +388,9 @@ while getopts 'hvqxkfScRn:d:s:r:?t:T:u:I:' OPTION; do
386388
R)
387389
RERUN="yes"
388390
;;
391+
m)
392+
KMEMLEAK="yes"
393+
;;
389394
n)
390395
nfsfile=$OPTARG
391396
[ -f "$nfsfile" ] || fail "Cannot read file: $nfsfile"
@@ -695,12 +700,16 @@ fi
695700
#
696701
# Run all the tests as specified.
697702
#
698-
msg "${TEST_RUNNER} ${QUIET:+-q}" \
703+
msg "${TEST_RUNNER}" \
704+
"${QUIET:+-q}" \
705+
"${KMEMLEAK:+-m}" \
699706
"-c \"${RUNFILES}\"" \
700707
"-T \"${TAGS}\"" \
701708
"-i \"${STF_SUITE}\"" \
702709
"-I \"${ITERATIONS}\""
703-
{ ${TEST_RUNNER} ${QUIET:+-q} \
710+
{ ${TEST_RUNNER} \
711+
${QUIET:+-q} \
712+
${KMEMLEAK:+-m} \
704713
-c "${RUNFILES}" \
705714
-T "${TAGS}" \
706715
-i "${STF_SUITE}" \
@@ -722,7 +731,9 @@ if [ "$RESULT" -eq "2" ] && [ -n "$RERUN" ]; then
722731
for test_name in $MAYBES; do
723732
grep "$test_name " "$TEMP_RESULTS_FILE" >>"$TEST_LIST"
724733
done
725-
{ ${TEST_RUNNER} ${QUIET:+-q} \
734+
{ ${TEST_RUNNER} \
735+
${QUIET:+-q} \
736+
${KMEMLEAK:+-m} \
726737
-c "${RUNFILES}" \
727738
-T "${TAGS}" \
728739
-i "${STF_SUITE}" \

‎tests/test-runner/bin/test-runner.py.in

Lines changed: 45 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -31,11 +31,13 @@ from pwd import getpwuid
3131
from select import select
3232
from subprocess import PIPE
3333
from subprocess import Popen
34+
from subprocess import check_output
3435
from threading import Timer
3536
from time import time, CLOCK_MONOTONIC
3637

3738
BASEDIR = '/var/tmp/test_results'
3839
TESTDIR = '/usr/share/zfs/'
40+
KMEMLEAK_FILE = '/sys/kernel/debug/kmemleak'
3941
KILL = 'kill'
4042
TRUE = 'true'
4143
SUDO = 'sudo'
@@ -75,6 +77,7 @@ class Result(object):
7577
self.runtime = ''
7678
self.stdout = []
7779
self.stderr = []
80+
self.kmemleak = ''
7881
self.result = ''
7982

8083
def done(self, proc, killed, reran):
@@ -90,6 +93,9 @@ class Result(object):
9093
if killed:
9194
self.result = 'KILLED'
9295
Result.runresults['KILLED'] += 1
96+
elif len(self.kmemleak) > 0:
97+
self.result = 'FAIL'
98+
Result.runresults['FAIL'] += 1
9399
elif self.returncode == 0:
94100
self.result = 'PASS'
95101
Result.runresults['PASS'] += 1
@@ -250,7 +256,7 @@ User: %s
250256

251257
return out.lines, err.lines
252258

253-
def run(self, dryrun):
259+
def run(self, dryrun, kmemleak):
254260
"""
255261
This is the main function that runs each individual test.
256262
Determine whether or not the command requires sudo, and modify it
@@ -270,6 +276,11 @@ User: %s
270276
fail('%s' % e)
271277

272278
self.result.starttime = monotonic_time()
279+
280+
if kmemleak:
281+
cmd = f'echo clear | {SUDO} tee {KMEMLEAK_FILE}'
282+
check_output(cmd, shell=True)
283+
273284
proc = Popen(privcmd, stdout=PIPE, stderr=PIPE)
274285
# Allow a special timeout value of 0 to mean infinity
275286
if int(self.timeout) == 0:
@@ -279,6 +290,12 @@ User: %s
279290
try:
280291
t.start()
281292
self.result.stdout, self.result.stderr = self.collect_output(proc)
293+
294+
if kmemleak:
295+
cmd = f'echo scan | {SUDO} tee {KMEMLEAK_FILE}'
296+
check_output(cmd, shell=True)
297+
cmd = f'{SUDO} cat {KMEMLEAK_FILE}'
298+
self.result.kmemleak = check_output(cmd, shell=True)
282299
except KeyboardInterrupt:
283300
self.kill_cmd(proc, True)
284301
fail('\nRun terminated at user request.')
@@ -355,6 +372,9 @@ User: %s
355372
with open(os.path.join(self.outputdir, 'merged'), 'wb') as merged:
356373
for _, line in lines:
357374
os.write(merged.fileno(), b'%s\n' % line)
375+
if len(self.result.kmemleak):
376+
with open(os.path.join(self.outputdir, 'kmemleak'), 'wb') as kmem:
377+
kmem.write(self.result.kmemleak)
358378

359379

360380
class Test(Cmd):
@@ -439,22 +459,22 @@ Tags: %s
439459

440460
cont = True
441461
if len(pretest.pathname):
442-
pretest.run(options.dryrun)
462+
pretest.run(options.dryrun, False)
443463
cont = pretest.result.result == 'PASS'
444464
pretest.log(options)
445465

446466
if cont:
447-
test.run(options.dryrun)
467+
test.run(options.dryrun, options.kmemleak)
448468
if test.result.result == 'KILLED' and len(failsafe.pathname):
449-
failsafe.run(options.dryrun)
469+
failsafe.run(options.dryrun, False)
450470
failsafe.log(options, suppress_console=True)
451471
else:
452472
test.skip()
453473

454474
test.log(options)
455475

456476
if len(posttest.pathname):
457-
posttest.run(options.dryrun)
477+
posttest.run(options.dryrun, False)
458478
posttest.log(options)
459479

460480

@@ -557,7 +577,7 @@ Tags: %s
557577

558578
cont = True
559579
if len(pretest.pathname):
560-
pretest.run(options.dryrun)
580+
pretest.run(options.dryrun, False)
561581
cont = pretest.result.result == 'PASS'
562582
pretest.log(options)
563583

@@ -570,17 +590,17 @@ Tags: %s
570590
failsafe = Cmd(self.failsafe, outputdir=odir, timeout=self.timeout,
571591
user=self.failsafe_user, identifier=self.identifier)
572592
if cont:
573-
test.run(options.dryrun)
593+
test.run(options.dryrun, options.kmemleak)
574594
if test.result.result == 'KILLED' and len(failsafe.pathname):
575-
failsafe.run(options.dryrun)
595+
failsafe.run(options.dryrun, False)
576596
failsafe.log(options, suppress_console=True)
577597
else:
578598
test.skip()
579599

580600
test.log(options)
581601

582602
if len(posttest.pathname):
583-
posttest.run(options.dryrun)
603+
posttest.run(options.dryrun, False)
584604
posttest.log(options)
585605

586606

@@ -845,6 +865,11 @@ class TestRun(object):
845865
else:
846866
write_log('Could not make a symlink to directory %s\n' %
847867
self.outputdir, LOG_ERR)
868+
869+
if options.kmemleak:
870+
cmd = f'echo scan=0 | {SUDO} tee {KMEMLEAK_FILE}'
871+
check_output(cmd, shell=True)
872+
848873
iteration = 0
849874
while iteration < options.iterations:
850875
for test in sorted(self.tests.keys()):
@@ -990,6 +1015,14 @@ def fail(retstr, ret=1):
9901015
exit(ret)
9911016

9921017

1018+
def kmemleak_cb(option, opt_str, value, parser):
1019+
if not os.path.exists(KMEMLEAK_FILE):
1020+
fail(f"File '{KMEMLEAK_FILE}' doesn't exist. " +
1021+
"Enable CONFIG_DEBUG_KMEMLEAK in kernel configuration.")
1022+
1023+
setattr(parser.values, option.dest, True)
1024+
1025+
9931026
def options_cb(option, opt_str, value, parser):
9941027
path_options = ['outputdir', 'template', 'testdir', 'logfile']
9951028

@@ -1027,6 +1060,9 @@ def parse_args():
10271060
parser.add_option('-i', action='callback', callback=options_cb,
10281061
default=TESTDIR, dest='testdir', type='string',
10291062
metavar='testdir', help='Specify a test directory.')
1063+
parser.add_option('-m', action='callback', callback=kmemleak_cb,
1064+
default=False, dest='kmemleak',
1065+
help='Enable kmemleak reporting (Linux only)')
10301066
parser.add_option('-p', action='callback', callback=options_cb,
10311067
default='', dest='pre', metavar='script',
10321068
type='string', help='Specify a pre script.')

‎tests/test-runner/man/test-runner.1

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -210,6 +210,8 @@ to be consumed by the run command.
210210
.It Fl d
211211
Dry run mode.
212212
Execute no tests, but print a description of each test that would have been run.
213+
.It Fl m
214+
Enable kmemleak reporting (Linux only)
213215
.It Fl g
214216
Create test groups from any directories found while searching for tests.
215217
.It Fl o Ar outputdir

0 commit comments

Comments
 (0)