ZTS: Optimize ksm settings and caching

Don't use KSM on the FreeBSD VMs and optimize ksm settings for
faster ZTS test runs.

Free kernel caches on Linux VMs via cron, to get more usable
memory on them.

Signed-off-by: Tino Reichardt <[email protected]>

Author: mcmilk

.github/workflows/scripts/qemu-1-setup.sh

@@ -10,36 +10,12 @@ set -eu
 export DEBIAN_FRONTEND="noninteractive"
 sudo apt-get -y update
 sudo apt-get install -y axel cloud-image-utils daemonize guestfs-tools \
-  ksmtuned virt-manager linux-modules-extra-$(uname -r) zfsutils-linux
+  virt-manager linux-modules-extra-$(uname -r) zfsutils-linux
 
 # generate ssh keys
 rm -f ~/.ssh/id_ed25519
 ssh-keygen -t ed25519 -f ~/.ssh/id_ed25519 -q -N ""
 
-# we expect RAM shortage
-cat << EOF | sudo tee /etc/ksmtuned.conf > /dev/null
-# /etc/ksmtuned.conf - Configuration file for ksmtuned
-# https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/virtualization_tuning_and_optimization_guide/chap-ksm
-KSM_MONITOR_INTERVAL=60
-
-# Millisecond sleep between ksm scans for 16Gb server.
-# Smaller servers sleep more, bigger sleep less.
-KSM_SLEEP_MSEC=30
-
-KSM_NPAGES_BOOST=0
-KSM_NPAGES_DECAY=0
-KSM_NPAGES_MIN=1000
-KSM_NPAGES_MAX=25000
-
-KSM_THRES_COEF=80
-KSM_THRES_CONST=8192
-
-LOGFILE=/var/log/ksmtuned.log
-DEBUG=1
-EOF
-sudo systemctl restart ksm
-sudo systemctl restart ksmtuned
-
 # not needed
 sudo systemctl stop docker.socket
 sudo systemctl stop multipathd.socket
@@ -69,11 +45,11 @@ sleep 1
 sudo mkswap $DISK-part1
 sudo swapon $DISK-part1
 
-# 60GB data disk
+# disk for operating system
 SSD1="$DISK-part2"
 
-# 10GB data disk on ext4
-sudo fallocate -l 10G /test.ssd1
+# data disk on ext4
+sudo fallocate -l 12G /test.ssd1
 SSD2=$(sudo losetup -b 4096 -f /test.ssd1 --show)
 
 # adjust zfs module parameter and create pool
@@ -89,5 +65,5 @@ sudo zpool create -f -o ashift=12 zpool $SSD1 $SSD2 \
 
 # no need for some scheduler
 for i in /sys/block/s*/queue/scheduler; do
-  echo "none" | sudo tee $i > /dev/null
+  echo "none" | sudo tee $i
 done

.github/workflows/scripts/qemu-2-start.sh

@@ -150,7 +150,7 @@ echo "ENV=$ENV" >> $ENV
 # result path
 echo 'RESPATH="/var/tmp/test_results"' >> $ENV
 
-# FreeBSD 13 has problems with: e1000+virtio
+# FreeBSD 13 has problems with: e1000 and virtio
 echo "NIC=$NIC" >> $ENV
 
 # freebsd15 -> used in zfs-qemu.yml
@@ -162,6 +162,14 @@ echo "OSv=\"$OSv\"" >> $ENV
 # FreeBSD 15 (Current) -> used for summary
 echo "OSNAME=\"$OSNAME\"" >> $ENV
 
+# default vm count for testings
+VMs=2
+echo "VMs=\"$VMs\"" >> $ENV
+
+# default cpu count for testing vm's
+CPU=2
+echo "CPU=\"$CPU\"" >> $ENV
+
 sudo mkdir -p "/mnt/tests"
 sudo chown -R $(whoami) /mnt/tests
 
@@ -180,6 +188,7 @@ fi
 DISK="/dev/zvol/zpool/openzfs"
 FORMAT="raw"
 sudo zfs create -ps -b 64k -V 80g zpool/openzfs
+sudo zfs set sync=disabled redundant_metadata=none zpool/openzfs
 while true; do test -b $DISK && break; sleep 1; done
 echo "Importing VM image to zvol..."
 sudo qemu-img dd -f qcow2 -O raw if=$IMG of=$DISK bs=4M
@@ -223,12 +232,19 @@ sudo virt-install \
   --disk $DISK,bus=virtio,cache=none,format=$FORMAT,driver.discard=unmap \
   --import --noautoconsole >/dev/null
 
+# enable KSM on Linux
+if [ $(uname) = "Linux" ]; then
+  sudo virsh dommemstat --domain "openzfs" --period 5
+  sudo virsh node-memory-tune 500 100 1
+  echo 1 | sudo tee /sys/kernel/mm/ksm/run > /dev/null
+fi
+
 # Give the VMs hostnames so we don't have to refer to them with
 # hardcoded IP addresses.
 #
 # vm0:          Initial VM we install dependencies and build ZFS on.
 # vm1..2        Testing VMs
-for i in {0..9} ; do
+for ((i=0; i<=VMs; i++)); do
   echo "192.168.122.1$i vm$i" | sudo tee -a /etc/hosts
 done
 

.github/workflows/scripts/qemu-3-deps-vm.sh

@@ -32,8 +32,8 @@ function debian() {
 
   echo "##[group]Install Development Tools"
   sudo apt-get install -y \
-    acl alien attr autoconf bc cpio cryptsetup curl dbench dh-python dkms \
-    fakeroot fio gdb gdebi git ksh lcov isc-dhcp-client jq libacl1-dev \
+    acl alien attr autoconf bc cpio cron cryptsetup curl dbench dh-python \
+    dkms fakeroot fio gdb gdebi git ksh lcov isc-dhcp-client jq libacl1-dev \
     libaio-dev libattr1-dev libblkid-dev libcurl4-openssl-dev libdevmapper-dev \
     libelf-dev libffi-dev libmount-dev libpam0g-dev libselinux-dev libssl-dev \
     libtool libtool-bin libudev-dev libunwind-dev linux-headers-$(uname -r) \
@@ -77,8 +77,8 @@ function rhel() {
   fi
 
   sudo dnf install -y \
-    acl attr bc bzip2 cryptsetup curl dbench dkms elfutils-libelf-devel fio \
-    gdb git jq kernel-rpm-macros ksh libacl-devel libaio-devel \
+    acl attr bc bzip2 cronie cryptsetup curl dbench dkms elfutils-libelf-devel \
+    fio gdb git jq kernel-rpm-macros ksh libacl-devel libaio-devel \
     libargon2-devel libattr-devel libblkid-devel libcurl-devel libffi-devel \
     ncompress libselinux-devel libtirpc-devel libtool libudev-devel \
     libuuid-devel lsscsi mdadm nfs-utils openssl-devel pam-devel pamtester \

.github/workflows/scripts/qemu-5-setup.sh

@@ -14,39 +14,50 @@ PID=$(pidof /usr/bin/qemu-system-x86_64)
 tail --pid=$PID -f /dev/null
 sudo virsh undefine openzfs
 
-# default values per test vm:
-VMs=2
-CPU=2
-
 # cpu pinning
 CPUSET=("0,1" "2,3")
 
 case "$OS" in
   freebsd*)
-    # FreeBSD can't be optimized via ksmtuned
+    # FreeBSD needs only 6GiB
     RAM=6
     ;;
   *)
-    # Linux can be optimized via ksmtuned
-    RAM=8
+    # Linux needs more memory, but can be optimized to share it via KSM
+    RAM=7
     ;;
 esac
 
-# this can be different for each distro
-echo "VMs=$VMs" >> $ENV
-
 # create snapshot we can clone later
 sudo zfs snapshot zpool/openzfs@now
 
 # setup the testing vm's
 PUBKEY=$(cat ~/.ssh/id_ed25519.pub)
-for i in $(seq 1 $VMs); do
 
+# generate some memory stats
+cat <<EOF > cronjob.sh
+exec 1>>/var/tmp/stats.txt
+exec 2>&1
+echo "********************************************************************************"
+uptime
+free -m
+zfs list
+EOF
+
+sudo chmod +x cronjob.sh
+sudo mv -f cronjob.sh /root/cronjob.sh
+echo '*/5 * * * *  /root/cronjob.sh' > crontab.txt
+sudo crontab crontab.txt
+rm crontab.txt
+
+# start testing VMs
+for ((i=1; i<=VMs; i++)); do
   echo "Creating disk for vm$i..."
   DISK="/dev/zvol/zpool/vm$i"
   FORMAT="raw"
-  sudo zfs clone zpool/openzfs@now zpool/vm$i
-  sudo zfs create -ps -b 64k -V 80g zpool/vm$i-2
+  sudo zfs clone zpool/openzfs@now zpool/vm$i-system
+  sudo zfs create -ps -b 128k -V 64g zpool/vm$i-tests
+  sudo zfs set sync=disabled redundant_metadata=none zpool/vm$i-tests
 
   cat <<EOF > /tmp/user-data
 #cloud-config
@@ -83,40 +94,22 @@ EOF
     --graphics none \
     --cloud-init user-data=/tmp/user-data \
     --network bridge=virbr0,model=$NIC,mac="52:54:00:83:79:0$i" \
-    --disk $DISK,bus=virtio,cache=none,format=$FORMAT,driver.discard=unmap \
-    --disk $DISK-2,bus=virtio,cache=none,format=$FORMAT,driver.discard=unmap \
+    --disk $DISK-system,bus=virtio,cache=none,format=$FORMAT,driver.discard=unmap \
+    --disk $DISK-tests,bus=virtio,cache=none,format=$FORMAT,driver.discard=unmap \
     --import --noautoconsole >/dev/null
 done
 
-# check the memory state from time to time
-cat <<EOF > cronjob.sh
-# $OS
-exec 1>>/var/tmp/stats.txt
-exec 2>&1
-echo "*******************************************************"
-date
-uptime
-free -m
-df -h /mnt/tests
-zfs list
-EOF
-sudo chmod +x cronjob.sh
-sudo mv -f cronjob.sh /root/cronjob.sh
-echo '*/5 * * * *  /root/cronjob.sh' > crontab.txt
-sudo crontab crontab.txt
-rm crontab.txt
-
 # check if the machines are okay
 echo "Waiting for vm's to come up...  (${VMs}x CPU=$CPU RAM=$RAM)"
-for i in $(seq 1 $VMs); do
+for ((i=1; i<=VMs; i++)); do
   .github/workflows/scripts/qemu-wait-for-vm.sh vm$i
 done
 echo "All $VMs VMs are up now."
 
 # Save the VM's serial output (ttyS0) to /var/tmp/console.txt
 # - ttyS0 on the VM corresponds to a local /dev/pty/N entry
 # - use 'virsh ttyconsole' to lookup the /dev/pty/N entry
-for i in $(seq 1 $VMs); do
+for ((i=1; i<=VMs; i++)); do
   mkdir -p $RESPATH/vm$i
   read "pty" <<< $(sudo virsh ttyconsole vm$i)
   sudo nohup bash -c "cat $pty > $RESPATH/vm$i/console.txt" &

.github/workflows/scripts/qemu-6-tests.sh

@@ -45,7 +45,7 @@ if [ -z ${1:-} ]; then
   echo 0 > /tmp/ctr
   date "+%s" > /tmp/tsstart
 
-  for i in $(seq 1 $VMs); do
+  for ((i=1; i<=VMs; i++)); do
     IP="192.168.122.1$i"
     daemonize -c /var/tmp -p vm${i}.pid -o vm${i}log.txt -- \
       $SSH zfs@$IP $TESTS $OS $i $VMs $CI_TYPE
@@ -58,7 +58,7 @@ if [ -z ${1:-} ]; then
   done
 
   # wait for all vm's to finish
-  for i in $(seq 1 $VMs); do
+  for ((i=1; i<=VMs; i++)); do
     tail --pid=$(cat vm${i}.pid) -f /dev/null
     pid=$(cat vm${i}log.pid)
     rm -f vm${i}log.pid
@@ -72,19 +72,46 @@ fi
 export PATH="$PATH:/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/sbin:/usr/local/bin"
 case "$1" in
   freebsd*)
+    TDIR="/usr/local/share/zfs"
     sudo kldstat -n zfs 2>/dev/null && sudo kldunload zfs
     sudo -E ./zfs/scripts/zfs.sh
-    TDIR="/usr/local/share/zfs"
+    sudo mv -f /var/tmp/*.txt /tmp
+    sudo newfs -U -t -L tmp /dev/vtbd1 >/dev/null
+    sudo mount -o noatime /dev/vtbd1 /var/tmp
+    sudo chmod 1777 /var/tmp
+    sudo mv -f /tmp/*.txt /var/tmp
     ;;
   *)
     # use xfs @ /var/tmp for all distros
+    TDIR="/usr/share/zfs"
+    sudo -E modprobe zfs
     sudo mv -f /var/tmp/*.txt /tmp
     sudo mkfs.xfs -fq /dev/vdb
     sudo mount -o noatime /dev/vdb /var/tmp
     sudo chmod 1777 /var/tmp
     sudo mv -f /tmp/*.txt /var/tmp
-    sudo -E modprobe zfs
-    TDIR="/usr/share/zfs"
+
+    # no info messages about dropping caches in kernel log
+    echo 4 | sudo tee /proc/sys/vm/drop_caches >/dev/null
+
+    # check the memory state from time to time
+    cat <<EOF > cronjob.sh
+exec 1>>/var/tmp/stats.txt
+exec 2>&1
+echo "********************************************************************************"
+uptime
+free -m
+# don't waste memory with buffering
+echo "dropped caches:"
+sync
+echo 3 > /proc/sys/vm/drop_caches
+free -m
+EOF
+    sudo chmod +x cronjob.sh
+    sudo mv -f cronjob.sh /root/cronjob.sh
+    echo '*/2 * * * *  /root/cronjob.sh' > crontab.txt
+    sudo crontab crontab.txt
+    rm crontab.txt
     ;;
 esac
 

.github/workflows/scripts/qemu-7-prepare.sh

@@ -28,7 +28,7 @@ BASE="$HOME/work/zfs/zfs"
 MERGE="$BASE/.github/workflows/scripts/merge_summary.awk"
 
 # catch result files of testings (vm's should be there)
-for i in $(seq 1 $VMs); do
+for ((i=1; i<=VMs; i++)); do
   rsync -arL zfs@vm$i:$RESPATH/current $RESPATH/vm$i || true
   scp zfs@vm$i:"/var/tmp/*.txt" $RESPATH/vm$i || true
   scp zfs@vm$i:"/var/tmp/*.rpm" $RESPATH/vm$i || true
@@ -37,7 +37,7 @@ cp -f /var/tmp/*.txt $RESPATH || true
 cd $RESPATH
 
 # prepare result files for summary
-for i in $(seq 1 $VMs); do
+for ((i=1; i<=VMs; i++)); do
   file="vm$i/build-stderr.txt"
   test -s $file && mv -f $file build-stderr.txt
 

.github/workflows/scripts/qemu-8-summary.sh

@@ -45,7 +45,7 @@ fi
 
 echo -e "\nFull logs for download:\n    $1\n"
 
-for i in $(seq 1 $VMs); do
+for ((i=1; i<=VMs; i++)); do
   rv=$(cat vm$i/tests-exitcode.txt)
 
   if [ $rv = 0 ]; then