lzc_ioctl_fd: add ZFS_IOC_TRACE envvar to enable ioctl tracing

When set, dumps all ZFS ioctl calls and returns and their nvlists to
STDERR, to make debugging and understanding a lot easier.

Sponsored-by: https://despairlabs.com/sponsor/
Reviewed-by: Brian Behlendorf <[email protected]>
Reviewed-by: Alexander Motin <[email protected]>
Signed-off-by: Rob Norris <[email protected]>
Closes #17344
(cherry picked from commit a387b75)

Author: robn

lib/libzfs_core/libzfs_core.c

@@ -102,6 +102,8 @@ static int g_fd = -1;
 static pthread_mutex_t g_lock = PTHREAD_MUTEX_INITIALIZER;
 static int g_refcount;
 
+static int g_ioc_trace = 0;
+
 #ifdef ZFS_DEBUG
 static zfs_ioc_t fail_ioc_cmd = ZFS_IOC_LAST;
 static zfs_errno_t fail_ioc_err;
@@ -154,6 +156,10 @@ libzfs_core_init(void)
 #ifdef ZFS_DEBUG
 	libzfs_core_debug_ioc();
 #endif
+
+	if (getenv("ZFS_IOC_TRACE"))
+		g_ioc_trace = 1;
+
 	(void) pthread_mutex_unlock(&g_lock);
 	return (0);
 }
@@ -176,7 +182,37 @@ libzfs_core_fini(void)
 int
 lzc_ioctl_fd(int fd, unsigned long ioc, zfs_cmd_t *zc)
 {
-	return (lzc_ioctl_fd_os(fd, ioc, zc));
+	if (!g_ioc_trace)
+		return (lzc_ioctl_fd_os(fd, ioc, zc));
+
+	nvlist_t *nvl;
+
+	fprintf(stderr, "=== lzc_ioctl: call: ioc=0x%lx name=%s\n",
+	    ioc, zc->zc_name[0] ? zc->zc_name : "[none]");
+	if (zc->zc_nvlist_src) {
+		nvl = fnvlist_unpack(
+		    (void *)(uintptr_t)zc->zc_nvlist_src,
+		    zc->zc_nvlist_src_size);
+		nvlist_print(stderr, nvl);
+		fnvlist_free(nvl);
+	}
+
+	int rc = lzc_ioctl_fd_os(fd, ioc, zc);
+	int err = errno;
+
+	fprintf(stderr, "=== lzc_ioctl: result: ioc=0x%lx name=%s "
+	    "rc=%d errno=%d\n", ioc, zc->zc_name[0] ? zc->zc_name : "[none]",
+	    rc, (rc < 0 ? err : 0));
+	if (rc >= 0 && zc->zc_nvlist_dst) {
+		nvl = fnvlist_unpack(
+		    (void *)(uintptr_t)zc->zc_nvlist_dst,
+		    zc->zc_nvlist_dst_size);
+		nvlist_print(stderr, nvl);
+		fnvlist_free(nvl);
+	}
+
+	errno = err;
+	return (rc);
 }
 
 static int