libzfs: add key provider machinery and a keylocation=exec:// impl

This acquires the key material by running the file specified by the URI
with [path, op, fsname], and reading back data from the pipe
located at fd 3 after the child exits (this means, that, i.a.,
children that write Too Much get an I/O error instead of hanging,
exec:// providers can be written in any language that can
write(3, [buf]), and they can be as interactive (or non-interactive)
and as verbose (or terse) as they want)

See zfs-change-key(8) for example statesome providers, or the
abomination below for a trivial stateless one
	#!/bin/sh -x
	echo "$0" "$@"

	[ -z "$2" ] && {
		echo "No dataset name (zfs-create?)" >&2
		exit 1
	}

	if command -v sha256 >/dev/null; then
		sha256 -qs "$2"
	else
		echo -n "$2" | sha256sum | awk '{print $1}'
	fi | tee /dev/stderr >&3

See zfs-change-key(8) for a user-level description of key-providers,
or below for state machines

load:
  * [_ _] => error
  * [_ x] => [x _], unseal(x)
  * [o x] =>
    + show error
    + let user choose to try either one or the other state
    + instruct what to invoke in either case
  * [o _] => unseal(o)

new: into staging area
  * fresh       : [_ _] => [_ x]
  * regenerating: [o _] => [o x]
  * dirty:        [? x] =>

shift (on success): mark new state as current, free old state
  * [_ _] => how?
  * [_ x] => [x _]
  * [o x] => [x _], free(o)
  * [o _] => [_ _], free(o)
i.e.
  [a b] => [b _], free(a)

unshift (on deletion): move current state to new
  * [_ _] => how?
  * [_ x] => wrong
  * [o x] => wrong
  * [o _] => [_ o]
i.e.
  [a b] => [_ a] (technically free(b), i guess, but shouldn't happen)

cancel (on error): free new state if present
  * how?                         : [_ _] =>
  * from new                     : [_ x] => [_ _] free(x)
  * from new                     : [o x] => [o _] free(x)
  * from inherit/other executable: [o _] => [o _]
i.e.
  [a, b] => [a, _], free(b)

two stable states:
[_ _] -> new: [_ n]     --ok----> shift : [n _]
[_ _] -> new: [_ n]     --error-> cancel: [n _]

[o _] -> new: [o n]     --ok----> shift : [n _], free(o)
[o _] -> new: [o n]     --error-> cancel: [o _], free(n)

[o _] (inheriting)      --ok----> shift : [_ _], free(o)
[o _] (inheriting)      --error-> cancel: [o _]
inheriting homomorphic to switching to something else

[o _] -> unshift: [_ o] --ok----> cancel: [_ _], free(o)
[o _] -> unshift: [_ o] --error-> shift : [o _]

if shift or cancel wasn't called:
[_ o] -> load -> [o, _], unseal
[ó o] -> load -> pick a resolution. should allow loading either
                 to check with zfs load-key -n and instruct
                 what to do in either case

[_ o] -> new -> error? try to load first, and pick a resolution
[ó o] -> new -> error? try to load first, and pick a resolution

i.e. somehow libzfs failed to do the shift after committing

Signed-off-by: Ahelenia Ziemiańska <[email protected]>

Author: nabijaczleweli

config/user-libexec.m4

@@ -1,9 +1,15 @@
 AC_DEFUN([ZFS_AC_CONFIG_USER_ZFSEXEC], [
+	AC_REQUIRE([AC_LIB_PREPARE_PREFIX])
+
 	AC_ARG_WITH(zfsexecdir,
 		AS_HELP_STRING([--with-zfsexecdir=DIR],
 		[install scripts [[@<:@libexecdir@:>@/zfs]]]),
 		[zfsexecdir=$withval],
 		[zfsexecdir="${libexecdir}/zfs"])
 
 	AC_SUBST([zfsexecdir])
+	AC_LIB_WITH_FINAL_PREFIX([
+		eval true_zfsexecdir=\"$zfsexecdir\"
+		AC_DEFINE_UNQUOTED([ZFSEXECDIR], ["$true_zfsexecdir"], [location of non-user-runnable executables])
+	])
 ])

include/libzfs_impl.h

@@ -258,6 +258,17 @@ extern int find_shares_object(differ_info_t *di);
 extern void libzfs_set_pipe_max(int infd);
 extern void zfs_commit_proto(zfs_share_proto_t *);
 
+typedef enum {
+	BACK_OP_LOAD,
+	BACK_OP_NEW,
+	BACK_OP_SHIFT,
+	BACK_OP_UNSHIFT,
+	BACK_OP_CANCEL,
+} encryption_backend_op_t;
+
+extern int notify_encryption_backend(zfs_handle_t *zhp,
+    const char *keylocation, encryption_backend_op_t what_of);
+
 #ifdef	__cplusplus
 }
 #endif

lib/libzfs/libzfs_crypto.c

@@ -21,6 +21,7 @@
 #include <sys/zfs_context.h>
 #include <sys/fs/zfs.h>
 #include <sys/dsl_crypt.h>
+#include <sys/wait.h>
 #include <libintl.h>
 #include <termios.h>
 #include <signal.h>
@@ -71,14 +72,25 @@ static int get_key_material_file(libzfs_handle_t *, const char *, const char *,
     zfs_keyformat_t, boolean_t, uint8_t **, size_t *);
 static int get_key_material_https(libzfs_handle_t *, const char *, const char *,
     zfs_keyformat_t, boolean_t, uint8_t **, size_t *);
+static int get_key_material_exec(libzfs_handle_t *, const char *, const char *,
+    zfs_keyformat_t, boolean_t, uint8_t **, size_t *);
 
 static zfs_uri_handler_t uri_handlers[] = {
 	{ "file", get_key_material_file },
 	{ "https", get_key_material_https },
 	{ "http", get_key_material_https },
+	{ "exec", get_key_material_exec },
 	{ NULL, NULL }
 };
 
+static const char *backend_ops_lcase[] = {
+	[BACK_OP_LOAD] = "load",
+	[BACK_OP_NEW] = "new",
+	[BACK_OP_SHIFT] = "shift",
+	[BACK_OP_UNSHIFT] = "unshift",
+	[BACK_OP_CANCEL] = "cancel",
+};
+
 static int
 pkcs11_get_urandom(uint8_t *buf, size_t bytes)
 {
@@ -669,6 +681,148 @@ get_key_material_https(libzfs_handle_t *hdl, const char *uri,
 	return (ret);
 }
 
+static int
+execute_key_provider_exec(libzfs_handle_t *hdl, const char *name,
+    encryption_backend_op_t op, const char *fsname, int *outfd)
+{
+	int ret = 0, status, compipe[2];
+	char *setpath = getenv("ZFS_PROVIDER_DIR");
+	char * const argv[] =
+	    {(char *)name, (char *)backend_ops_lcase[op], (char *)fsname, NULL};
+	pid_t child;
+
+	if (pipe2(compipe, O_NONBLOCK | O_CLOEXEC) != 0) {
+		ret = errno;
+		zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
+		    "Failed to create key provider pipes: %s"), strerror(ret));
+		return (ret);
+	}
+
+	switch ((child = fork())) {
+	case -1:
+		ret = errno;
+		zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
+		    "Failed to start key provider %s: %s"),
+		    name, strerror(ret));
+		(void) close(compipe[1]);
+		goto end;
+	case 0: /* child */
+		(void) dup2(compipe[1], 3);
+
+		char fullpath[PATH_MAX];
+		if (setpath != NULL) {
+			strlcpy(fullpath, setpath, sizeof (fullpath));
+			strlcat(fullpath, "/", sizeof (fullpath));
+			if (strlcat(fullpath, name, sizeof (fullpath)) <
+			    sizeof (fullpath))
+				(void) execv(fullpath, argv);
+		}
+
+		strcpy(fullpath, SYSCONFDIR "/zfs/providers.d/");
+		if (strlcat(fullpath, name, sizeof (fullpath)) <
+		    sizeof (fullpath))
+			(void) execv(fullpath, argv);
+
+		strcpy(fullpath, ZFSEXECDIR "/providers.d/");
+		if (strlcat(fullpath, name, sizeof (fullpath)) <
+		    sizeof (fullpath))
+			(void) execv(fullpath, argv);
+
+		status = write(3, strerror(errno), strlen(strerror(errno)));
+		_exit(-1);
+	}
+
+	/* parent */
+	(void) close(compipe[1]);
+
+	while (waitpid(child, &status, 0) == -1)
+		if (errno != EINTR) {
+			ret = errno;
+			zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
+			    "Failed to wait for key provider %s (%d): %s"),
+			    name, child, strerror(ret));
+
+			goto end;
+		}
+
+	if (WIFSIGNALED(status)) {
+		ret = EZFS_INTR;
+		zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
+		    "Key provider %s killed by %s"),
+		    name, strsignal(WTERMSIG(status)));
+	} else if (WEXITSTATUS(status) == 0xff) {
+		char errbuf[128] = {0};
+		status = read(compipe[0], errbuf, sizeof (errbuf) - 1);
+
+		ret = ENOEXEC;
+		zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
+		    "Failed to start key provider %s: %s"),
+		    name, strlen(errbuf) ? errbuf : "(?)");
+	} else if (WEXITSTATUS(status) != 0) {
+		ret = ECANCELED;
+		zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
+		    "Key provider %s failed with %d"),
+		    name, WEXITSTATUS(status));
+	}
+
+end:
+	if (ret == 0)
+		*outfd = compipe[0];
+	else
+		(void) close(compipe[0]);
+	errno = 0;
+	return (ret);
+}
+
+static int
+get_key_material_exec(libzfs_handle_t *hdl, const char *uri,
+    const char *fsname, zfs_keyformat_t keyformat, boolean_t newkey,
+    uint8_t **restrict buf, size_t *restrict len_out)
+{
+	int ret = 0, rdpipe = -1;
+	FILE *f;
+
+	if (strlen(uri) < 7)
+		return (EINVAL);
+
+	if ((ret = execute_key_provider_exec(hdl, uri + 7,
+	    newkey ? BACK_OP_NEW : BACK_OP_LOAD, fsname, &rdpipe)) != 0)
+		return (ret);
+
+	if ((f = fdopen(rdpipe, "r")) == NULL) {
+		ret = errno;
+		errno = 0;
+		zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
+		    "Failed to fdopen key provider pipe"));
+
+		(void) close(rdpipe);
+		return (ret);
+	}
+
+	ret = get_key_material_raw(f, keyformat, buf, len_out);
+	(void) fclose(f);
+	return (ret);
+}
+
+__attribute__((visibility("hidden"))) int
+notify_encryption_backend(zfs_handle_t *zhp, const char *keylocation,
+    encryption_backend_op_t what_of)
+{
+	int ret = 0;
+	int rdpipe = -1;
+
+	if (keylocation == NULL)
+		return (0);
+
+	if (strncmp(keylocation, "exec://", 7) == 0) {
+		if ((ret = execute_key_provider_exec(zhp->zfs_hdl,
+		    keylocation + 7, what_of, zfs_get_name(zhp), &rdpipe)) == 0)
+			(void) close(rdpipe);
+	}
+
+	return (ret);
+}
+
 /*
  * Attempts to fetch key material, no matter where it might live. The key
  * material is allocated and returned in km_out. *can_retry_out will be set
@@ -694,7 +848,6 @@ get_key_material(libzfs_handle_t *hdl, boolean_t do_verify, boolean_t newkey,
 	if (ret != 0)
 		goto error;
 
-	/* open the appropriate file descriptor */
 	switch (keyloc) {
 	case ZFS_KEYLOCATION_PROMPT:
 		if (isatty(fileno(stdin))) {
@@ -1574,7 +1727,7 @@ zfs_crypto_rewrap(zfs_handle_t *zhp, nvlist_t *raw_props, boolean_t inheritkey)
 {
 	int ret;
 	char errbuf[1024];
-	boolean_t is_encroot;
+	boolean_t is_encroot, requested_new_key = B_FALSE;
 	nvlist_t *props = NULL;
 	uint8_t *wkeydata = NULL;
 	uint_t wkeylen = 0;
@@ -1626,6 +1779,18 @@ zfs_crypto_rewrap(zfs_handle_t *zhp, nvlist_t *raw_props, boolean_t inheritkey)
 		goto error;
 	}
 
+	/* Get current location to send the right transition to backend */
+	ret = zfs_prop_get(zhp, ZFS_PROP_KEYLOCATION,
+	    prop_keylocation, sizeof (prop_keylocation),
+	    NULL, NULL, 0, B_TRUE);
+	if (ret != 0) {
+		zfs_error_aux(zhp->zfs_hdl,
+		    dgettext(TEXT_DOMAIN, "Failed to "
+		    "get existing keylocation "
+		    "property."));
+		goto error;
+	}
+
 	/*
 	 * If the user wants to use the inheritkey variant of this function
 	 * we don't need to collect any crypto arguments.
@@ -1660,26 +1825,14 @@ zfs_crypto_rewrap(zfs_handle_t *zhp, nvlist_t *raw_props, boolean_t inheritkey)
 				if (ret != 0) {
 					zfs_error_aux(zhp->zfs_hdl,
 					    dgettext(TEXT_DOMAIN, "Failed to "
-					    "get existing keyformat "
+					    "insert existing keyformat "
 					    "property."));
 					goto error;
 				}
 			}
 
-			if (keylocation == NULL) {
-				ret = zfs_prop_get(zhp, ZFS_PROP_KEYLOCATION,
-				    prop_keylocation, sizeof (prop_keylocation),
-				    NULL, NULL, 0, B_TRUE);
-				if (ret != 0) {
-					zfs_error_aux(zhp->zfs_hdl,
-					    dgettext(TEXT_DOMAIN, "Failed to "
-					    "get existing keylocation "
-					    "property."));
-					goto error;
-				}
-
+			if (keylocation == NULL)
 				keylocation = prop_keylocation;
-			}
 		} else {
 			/* need a new key for non-encryption roots */
 			if (keyformat == ZFS_KEYFORMAT_NONE) {
@@ -1705,6 +1858,7 @@ zfs_crypto_rewrap(zfs_handle_t *zhp, nvlist_t *raw_props, boolean_t inheritkey)
 		ret = populate_create_encryption_params_nvlists(zhp->zfs_hdl,
 		    zhp, B_TRUE, keyformat, keylocation, props, &wkeydata,
 		    &wkeylen);
+		requested_new_key = B_TRUE;
 		if (ret != 0)
 			goto error;
 	} else {
@@ -1783,6 +1937,18 @@ zfs_crypto_rewrap(zfs_handle_t *zhp, nvlist_t *raw_props, boolean_t inheritkey)
 		zfs_error(zhp->zfs_hdl, EZFS_CRYPTOFAILED, errbuf);
 	}
 
+	/*
+	 * we can't roll the key back; depending on the scenario,
+	 * this will either resolve itself autimatically,
+	 * or user will have to try old/new key and remove the wrong one;
+	 * freeing the old key would be nice, but best-effort as well
+	 */
+	notify_encryption_backend(zhp,
+	    keylocation, ret == 0 ? BACK_OP_SHIFT : BACK_OP_CANCEL);
+	if (ret == 0 && strcmp(prop_keylocation, keylocation ?: "none"))
+		notify_encryption_backend(zhp,
+		    prop_keylocation, BACK_OP_SHIFT);
+
 	if (pzhp != NULL)
 		zfs_close(pzhp);
 	if (props != NULL)
@@ -1793,6 +1959,10 @@ zfs_crypto_rewrap(zfs_handle_t *zhp, nvlist_t *raw_props, boolean_t inheritkey)
 	return (ret);
 
 error:
+	if (requested_new_key)
+		notify_encryption_backend(zhp,
+		    keylocation, BACK_OP_CANCEL);
+
 	if (pzhp != NULL)
 		zfs_close(pzhp);
 	if (props != NULL)

lib/libzfs/libzfs_dataset.c

@@ -3775,6 +3775,33 @@ zfs_create(libzfs_handle_t *hdl, const char *path, zfs_type_t type,
 	return (0);
 }
 
+static int
+zfs_destroy_notification_applicable(zfs_handle_t *zhp, char *keylocation,
+    size_t keylocation_len, boolean_t *notify)
+{
+	boolean_t is_encroot;
+
+	*notify = B_FALSE;
+
+	if ((zhp->zfs_type & ZFS_TYPE_DATASET) == 0 ||
+	    zfs_prop_get_int(zhp, ZFS_PROP_KEYFORMAT) == ZFS_KEYFORMAT_NONE)
+		return (0);
+
+	zfs_crypto_get_encryption_root(zhp, &is_encroot, NULL);
+	if (!is_encroot)
+		return (0);
+
+	if (zfs_prop_get(zhp, ZFS_PROP_KEYLOCATION,
+	    keylocation, keylocation_len,
+	    NULL, NULL, 0, B_TRUE) != 0)
+		return (zfs_standard_error_fmt(zhp->zfs_hdl, EZFS_BADPROP,
+		    dgettext(TEXT_DOMAIN, "couldn't get keylocation for '%s'"),
+		    zhp->zfs_name));
+
+	*notify = B_TRUE;
+	return (0);
+}
+
 /*
  * Destroys the given dataset.  The caller must make sure that the filesystem
  * isn't mounted, and that there are no active dependents. If the file system
@@ -3784,6 +3811,8 @@ int
 zfs_destroy(zfs_handle_t *zhp, boolean_t defer)
 {
 	int error;
+	char keylocation[MAXNAMELEN];
+	boolean_t notify = B_FALSE;
 
 	if (zhp->zfs_type != ZFS_TYPE_SNAPSHOT && defer)
 		return (EINVAL);
@@ -3807,15 +3836,34 @@ zfs_destroy(zfs_handle_t *zhp, boolean_t defer)
 		error = lzc_destroy_snaps(nv, defer, NULL);
 		fnvlist_free(nv);
 	} else {
+		if ((error = zfs_destroy_notification_applicable(zhp,
+		    keylocation, sizeof (keylocation), &notify)) != 0)
+			return (error);
+
+		if (notify) {
+			error = notify_encryption_backend(zhp, keylocation,
+			    BACK_OP_UNSHIFT);
+			if (error)
+				return (zfs_standard_error_fmt(zhp->zfs_hdl,
+				    error, dgettext(TEXT_DOMAIN,
+				    "couldn't notify encryption back-end "
+				    "about destruction of '%s'"),
+				    zhp->zfs_name));
+		}
+
 		error = lzc_destroy(zhp->zfs_name);
 	}
 
 	if (error != 0 && error != ENOENT) {
-		return (zfs_standard_error_fmt(zhp->zfs_hdl, errno,
+		error = errno;
+
+		notify_encryption_backend(zhp, keylocation, BACK_OP_SHIFT);
+		return (zfs_standard_error_fmt(zhp->zfs_hdl, error,
 		    dgettext(TEXT_DOMAIN, "cannot destroy '%s'"),
 		    zhp->zfs_name));
 	}
 
+	notify_encryption_backend(zhp, keylocation, BACK_OP_CANCEL);
 	remove_mountpoint(zhp);
 
 	return (0);