Block cloning tests.

The test mostly focus on testing various corner cases.
The tests take a long time to run, so for the common.run runfile
we randomly select a hundred tests.
To run all the bclone tests, bclone.run runfile should be used.

Signed-off-by: Pawel Jakub Dawidek <[email protected]>

Author: pjd

tests/Makefile.am

@@ -16,6 +16,7 @@ dist_scripts_test_runner_include_DATA = \
 
 scripts_runfilesdir = $(datadir)/$(PACKAGE)/runfiles
 dist_scripts_runfiles_DATA = \
+	%D%/runfiles/bclone.run \
 	%D%/runfiles/common.run \
 	%D%/runfiles/freebsd.run \
 	%D%/runfiles/linux.run \

tests/runfiles/bclone.run

@@ -0,0 +1,46 @@
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+# This run file contains all of the common functional tests.  When
+# adding a new test consider also adding it to the sanity.run file
+# if the new test runs to completion in only a few seconds.
+#
+# Approximate run time: 5 hours
+#
+
+[DEFAULT]
+pre = setup
+quiet = False
+pre_user = root
+user = root
+timeout = 28800
+post_user = root
+post = cleanup
+failsafe_user = root
+failsafe = callbacks/zfs_failsafe
+outputdir = /var/tmp/test_results
+tags = ['bclone']
+
+[tests/functional/bclone]
+tests = ['bclone_crossfs_corner_cases',
+    'bclone_crossfs_data',
+    'bclone_crossfs_embedded',
+    'bclone_crossfs_hole',
+    'bclone_diffprops_all',
+    'bclone_diffprops_checksum',
+    'bclone_diffprops_compress',
+    'bclone_diffprops_copies',
+    'bclone_diffprops_recordsize',
+    'bclone_prop_sync',
+    'bclone_samefs_corner_cases',
+    'bclone_samefs_data',
+    'bclone_samefs_embedded',
+    'bclone_samefs_hole']
+tags = ['bclone']

tests/runfiles/common.run

@@ -53,6 +53,24 @@ tags = ['functional', 'arc']
 tests = ['atime_001_pos', 'atime_002_neg', 'root_atime_off', 'root_atime_on']
 tags = ['functional', 'atime']
 
+[tests/functional/bclone]
+tests = ['bclone_crossfs_corner_cases_limited',
+    'bclone_crossfs_data',
+    'bclone_crossfs_embedded',
+    'bclone_crossfs_hole',
+    'bclone_diffprops_all',
+    'bclone_diffprops_checksum',
+    'bclone_diffprops_compress',
+    'bclone_diffprops_copies',
+    'bclone_diffprops_recordsize',
+    'bclone_prop_sync',
+    'bclone_samefs_corner_cases_limited',
+    'bclone_samefs_data',
+    'bclone_samefs_embedded',
+    'bclone_samefs_hole']
+tags = ['functional', 'bclone']
+timeout = 7200
+
 [tests/functional/bootfs]
 tests = ['bootfs_001_pos', 'bootfs_002_neg', 'bootfs_003_pos',
     'bootfs_004_neg', 'bootfs_005_neg', 'bootfs_006_pos', 'bootfs_007_pos',

tests/zfs-tests/cmd/Makefile.am

@@ -2,6 +2,7 @@ scripts_zfs_tests_bindir = $(datadir)/$(PACKAGE)/zfs-tests/bin
 
 
 scripts_zfs_tests_bin_PROGRAMS  = %D%/chg_usr_exec
+scripts_zfs_tests_bin_PROGRAMS += %D%/clonefile
 scripts_zfs_tests_bin_PROGRAMS += %D%/cp_files
 scripts_zfs_tests_bin_PROGRAMS += %D%/ctime
 scripts_zfs_tests_bin_PROGRAMS += %D%/dir_rd_update
@@ -119,7 +120,6 @@ scripts_zfs_tests_bin_PROGRAMS += %D%/renameat2
 scripts_zfs_tests_bin_PROGRAMS += %D%/xattrtest
 scripts_zfs_tests_bin_PROGRAMS += %D%/zed_fd_spill-zedlet
 scripts_zfs_tests_bin_PROGRAMS += %D%/idmap_util
-scripts_zfs_tests_bin_PROGRAMS += %D%/clonefile
 
 %C%_idmap_util_LDADD = libspl.la
 

tests/zfs-tests/cmd/clonefile.c

@@ -59,6 +59,10 @@
 #endif
 #endif /* __NR_copy_file_range */
 
+#ifdef __FreeBSD__
+#define	loff_t	off_t
+#endif
+
 ssize_t
 copy_file_range(int, loff_t *, int, loff_t *, size_t, unsigned int)
     __attribute__((weak));
@@ -140,7 +144,7 @@ usage(void)
 	    "  FICLONERANGE:\n"
 	    "    clonefile -r <src> <dst> <soff> <doff> <len>\n"
 	    "  copy_file_range:\n"
-	    "    clonefile -f <src> <dst> <soff> <doff> <len>\n"
+	    "    clonefile -f <src> <dst> [<soff> <doff> <len | \"all\">]\n"
 	    "  FIDEDUPERANGE:\n"
 	    "    clonefile -d <src> <dst> <soff> <doff> <len>\n");
 	return (1);
@@ -179,13 +183,29 @@ main(int argc, char **argv)
 		}
 	}
 
-	if (mode == CF_MODE_NONE || (argc-optind) < 2 ||
-	    (mode != CF_MODE_CLONE && (argc-optind) < 5))
-		return (usage());
+	switch (mode) {
+		case CF_MODE_NONE:
+			return (usage());
+		case CF_MODE_CLONE:
+			if ((argc-optind) != 2)
+				return (usage());
+			break;
+		case CF_MODE_CLONERANGE:
+		case CF_MODE_DEDUPERANGE:
+			if ((argc-optind) != 5)
+				return (usage());
+			break;
+		case CF_MODE_COPYFILERANGE:
+			if ((argc-optind) != 2 && (argc-optind) != 5)
+				return (usage());
+			break;
+		default:
+			abort();
+	}
 
 	loff_t soff = 0, doff = 0;
-	size_t len = 0;
-	if (mode != CF_MODE_CLONE) {
+	size_t len = SSIZE_MAX;
+	if ((argc-optind) == 5) {
 		soff = strtoull(argv[optind+2], NULL, 10);
 		if (soff == ULLONG_MAX) {
 			fprintf(stderr, "invalid source offset");
@@ -196,10 +216,15 @@ main(int argc, char **argv)
 			fprintf(stderr, "invalid dest offset");
 			return (1);
 		}
-		len = strtoull(argv[optind+4], NULL, 10);
-		if (len == ULLONG_MAX) {
-			fprintf(stderr, "invalid length");
-			return (1);
+		if (mode == CF_MODE_COPYFILERANGE &&
+		    strcmp(argv[optind+4], "all") == 0) {
+			len = SSIZE_MAX;
+		} else {
+			len = strtoull(argv[optind+4], NULL, 10);
+			if (len == ULLONG_MAX) {
+				fprintf(stderr, "invalid length");
+				return (1);
+			}
 		}
 	}
 
@@ -237,13 +262,15 @@ main(int argc, char **argv)
 			abort();
 	}
 
-	off_t spos = lseek(sfd, 0, SEEK_CUR);
-	off_t slen = lseek(sfd, 0, SEEK_END);
-	off_t dpos = lseek(dfd, 0, SEEK_CUR);
-	off_t dlen = lseek(dfd, 0, SEEK_END);
+	if (!quiet) {
+		off_t spos = lseek(sfd, 0, SEEK_CUR);
+		off_t slen = lseek(sfd, 0, SEEK_END);
+		off_t dpos = lseek(dfd, 0, SEEK_CUR);
+		off_t dlen = lseek(dfd, 0, SEEK_END);
 
-	fprintf(stderr, "file offsets: src=%lu/%lu; dst=%lu/%lu\n", spos, slen,
-	    dpos, dlen);
+		fprintf(stderr, "file offsets: src=%lu/%lu; dst=%lu/%lu\n",
+		    spos, slen, dpos, dlen);
+	}
 
 	close(dfd);
 	close(sfd);
@@ -254,7 +281,8 @@ main(int argc, char **argv)
 int
 do_clone(int sfd, int dfd)
 {
-	fprintf(stderr, "using FICLONE\n");
+	if (!quiet)
+		fprintf(stderr, "using FICLONE\n");
 	int err = ioctl(dfd, CF_FICLONE, sfd);
 	if (err < 0) {
 		fprintf(stderr, "ioctl(FICLONE): %s\n", strerror(errno));
@@ -266,7 +294,8 @@ do_clone(int sfd, int dfd)
 int
 do_clonerange(int sfd, int dfd, loff_t soff, loff_t doff, size_t len)
 {
-	fprintf(stderr, "using FICLONERANGE\n");
+	if (!quiet)
+		fprintf(stderr, "using FICLONERANGE\n");
 	cf_file_clone_range_t fcr = {
 		.src_fd = sfd,
 		.src_offset = soff,
@@ -284,12 +313,22 @@ do_clonerange(int sfd, int dfd, loff_t soff, loff_t doff, size_t len)
 int
 do_copyfilerange(int sfd, int dfd, loff_t soff, loff_t doff, size_t len)
 {
-	fprintf(stderr, "using copy_file_range\n");
+	if (!quiet)
+		fprintf(stderr, "using copy_file_range\n");
 	ssize_t copied = cf_copy_file_range(sfd, &soff, dfd, &doff, len, 0);
 	if (copied < 0) {
 		fprintf(stderr, "copy_file_range: %s\n", strerror(errno));
 		return (1);
 	}
+	if (len == SSIZE_MAX) {
+		struct stat sb;
+
+		if (fstat(sfd, &sb) < 0) {
+			fprintf(stderr, "fstat(sfd): %s\n", strerror(errno));
+			return (1);
+		}
+		len = sb.st_size;
+	}
 	if (copied != len) {
 		fprintf(stderr, "copy_file_range: copied less than requested: "
 		    "requested=%lu; copied=%lu\n", len, copied);
@@ -301,7 +340,8 @@ do_copyfilerange(int sfd, int dfd, loff_t soff, loff_t doff, size_t len)
 int
 do_deduperange(int sfd, int dfd, loff_t soff, loff_t doff, size_t len)
 {
-	fprintf(stderr, "using FIDEDUPERANGE\n");
+	if (!quiet)
+		fprintf(stderr, "using FIDEDUPERANGE\n");
 
 	char buf[sizeof (cf_file_dedupe_range_t)+
 	    sizeof (cf_file_dedupe_range_info_t)] = {0};

tests/zfs-tests/include/commands.cfg

@@ -98,7 +98,8 @@ export SYSTEM_FILES_COMMON='awk
     uname
     uniq
     vmstat
-    wc'
+    wc
+    xargs'
 
 export SYSTEM_FILES_FREEBSD='chflags
     compress

tests/zfs-tests/include/libtest.shlib

@@ -2629,6 +2629,38 @@ function get_config
 	fi | awk -F: -v cfg="$config:" '$0 ~ cfg {sub(/^'\''/, $2); sub(/'\''$/, $2); print $2}'
 }
 
+#
+# Get random number between min and max number.
+#
+# $1 Minimal value
+# $2 Maximal value
+# [$3 How many]
+#
+function random
+{
+	typeset -ri min=$1
+	typeset -ri max=$2
+	typeset -i value
+	typeset -i count
+	typeset -i i
+
+	if [[ -z "$3" ]]; then
+		count=1
+	else
+		count=$3
+	fi
+
+	for (( i = 0; i < $count; i++ )); do
+		while true; do
+			((value = RANDOM % (max + 1)))
+			if ((value >= min)); then
+				break
+			fi
+		done
+		echo $value
+	done
+}
+
 #
 # Privated function. Random select one of items from arguments.
 #

tests/zfs-tests/tests/Makefile.am

@@ -90,6 +90,9 @@ nobase_dist_datadir_zfs_tests_tests_DATA += \
 	functional/alloc_class/alloc_class.kshlib \
 	functional/atime/atime.cfg \
 	functional/atime/atime_common.kshlib \
+	functional/bclone/bclone.cfg \
+	functional/bclone/bclone_common.kshlib \
+	functional/bclone/bclone_corner_cases.kshlib \
 	functional/block_cloning/block_cloning.kshlib \
 	functional/cache/cache.cfg \
 	functional/cache/cache.kshlib \
@@ -438,6 +441,24 @@ nobase_dist_datadir_zfs_tests_tests_SCRIPTS += \
 	functional/atime/root_atime_on.ksh \
 	functional/atime/root_relatime_on.ksh \
 	functional/atime/setup.ksh \
+	functional/bclone/bclone_crossfs_corner_cases.ksh \
+	functional/bclone/bclone_crossfs_corner_cases_limited.ksh \
+	functional/bclone/bclone_crossfs_data.ksh \
+	functional/bclone/bclone_crossfs_embedded.ksh \
+	functional/bclone/bclone_crossfs_hole.ksh \
+	functional/bclone/bclone_diffprops_all.ksh \
+	functional/bclone/bclone_diffprops_checksum.ksh \
+	functional/bclone/bclone_diffprops_compress.ksh \
+	functional/bclone/bclone_diffprops_copies.ksh \
+	functional/bclone/bclone_diffprops_recordsize.ksh \
+	functional/bclone/bclone_prop_sync.ksh \
+	functional/bclone/bclone_samefs_corner_cases.ksh \
+	functional/bclone/bclone_samefs_corner_cases_limited.ksh \
+	functional/bclone/bclone_samefs_data.ksh \
+	functional/bclone/bclone_samefs_embedded.ksh \
+	functional/bclone/bclone_samefs_hole.ksh \
+	functional/bclone/cleanup.ksh \
+	functional/bclone/setup.ksh \
 	functional/block_cloning/cleanup.ksh \
 	functional/block_cloning/setup.ksh \
 	functional/block_cloning/block_cloning_copyfilerange_cross_dataset.ksh \

tests/zfs-tests/tests/functional/bclone/TODO

@@ -0,0 +1,5 @@
+- If dedup enabled, block_cloning uses dedup.
+- block_cloning works only with the same encryption key
+- check when block cloning doesn't suppose to work
+- check block cloning between two different pools
+- block cloning from a snapshot

tests/zfs-tests/tests/functional/bclone/bclone.cfg

@@ -0,0 +1,32 @@
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or https://opensource.org/licenses/CDDL-1.0.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2023 by Pawel Jakub Dawidek
+#
+
+# TODO: We should calculate that based on ashift.
+export MINBLOCKSIZE=512
+
+export TESTSRCFS="$TESTPOOL/$TESTFS/src"
+export TESTDSTFS="$TESTPOOL/$TESTFS/dst"
+export TESTSRCDIR="$TESTDIR/src"
+export TESTDSTDIR="$TESTDIR/dst"