ZTS: Use QEMU for tests on Linux and FreeBSD

This commit adds functional tests for these systems:
- AlmaLinux 8, AlmaLinux 9
- ArchLinux
- CentOS Stream 9
- Fedora 39, Fedora 40
- Debian 11, Debian 12
- FreeBSD 13, FreeBSD 14, FreeBSD 15
- Ubuntu 22.04, Ubuntu 24.04

Workflow for each operating system:
- install QEMU on the github runner
- download current cloud image
- start and init that image via cloud-init
- install deps and poweroff system
- start system and build openzfs and then poweroff again
- clone the system and start 3 qemu machines for tests
- use trimable virtual disks (3x 2GB)
- do the functional testings in < 3h

Signed-off-by: Tino Reichardt <[email protected]>
Co-authored-by: Tony Hutter <[email protected]>

Author: mcmilk

.github/workflows/checkstyle.yaml

@@ -8,7 +8,7 @@ jobs:
   checkstyle:
     runs-on: ubuntu-22.04
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v3
       with:
         ref: ${{ github.event.pull_request.head.sha }}
     - name: Install dependencies
@@ -52,7 +52,7 @@ jobs:
       if: failure() && steps.CheckABI.outcome == 'failure'
       run: |
         find -name *.abi | tar -cf abi_files.tar -T -
-    - uses: actions/upload-artifact@v4
+    - uses: actions/upload-artifact@v3
       if: failure() && steps.CheckABI.outcome == 'failure'
       with:
         name: New ABI files (use only if you're sure about interface changes)

.github/workflows/codeql.yml

@@ -24,12 +24,11 @@ jobs:
         echo "MAKEFLAGS=-j$(nproc)" >> $GITHUB_ENV
 
     - name: Checkout repository
-      uses: actions/checkout@v4
+      uses: actions/checkout@v3
 
     - name: Initialize CodeQL
       uses: github/codeql-action/init@v2
       with:
-        config-file: .github/codeql-${{ matrix.language }}.yml
         languages: ${{ matrix.language }}
 
     - name: Autobuild

.github/workflows/scripts/README.md

@@ -0,0 +1,10 @@
+
+Workflow for each operating system:
+ - install QEMU on the github runner
+ - download current cloud image
+ - start and init that image via cloud-init
+ - install deps and poweroff system
+ - start system and build openzfs and then poweroff again
+ - clone the system and start 4 qemu workers for the testings (4x 3GB RAM)
+ - use trimable virtual disks (3x 1GB) for each testing system
+ - do the functional testings < 2h for each os

.github/workflows/scripts/generate-summary.sh

@@ -1,119 +1,148 @@
 #!/usr/bin/env bash
 
-# for runtime reasons we split functional testings into N parts
-# - use a define to check for missing tarfiles
-FUNCTIONAL_PARTS="4"
-
-ZTS_REPORT="tests/test-runner/bin/zts-report.py"
-chmod +x $ZTS_REPORT
+######################################################################
+# generate github summary page of all the testings
+######################################################################
 
 function output() {
-  echo -e $* >> Summary.md
+  echo -e $* >> "out-$logfile.md"
 }
 
-function error() {
-  output ":bangbang: $* :bangbang:\n"
+function outfile() {
+  if [ -f $1 ]; then
+    cat "$1" >> "out-$logfile.md"
+  else
+    echo "File $1 not found!" >> error.txt
+  fi
 }
 
-# this function generates the real summary
-# - expects a logfile "log" in current directory
-function generate() {
-  # we issued some error already
-  test ! -s log && return
+function send2github() {
+  test -f "$1" && dd if="$1" bs=999k count=1 >> $GITHUB_STEP_SUMMARY
+}
 
-  # for overview and zts-report
-  cat log | grep '^Test' > list
+# generate summary of one test
+function generate() {
+  test -f log || return
+
+  ######################################################
+  # input:
+  # - 3x log     -> full debug output
+  # - 3x results -> full list with summary in the end
+  ######################################################
+  # output:
+  # - info.txt   -> short summary list (zts-report)
+  # - list.txt   -> full list, but without debugging
+  # - debug.txt  -> full list with debugging info
+  ######################################################
+
+  cat log | grep '^Test[: ]' > list.txt
+  ./zts-report.py --no-maybes ./list.txt > info.txt
 
   # error details
   awk '/\[FAIL\]|\[KILLED\]/{ show=1; print; next; }
-    /\[SKIP\]|\[PASS\]/{ show=0; } show' log > err
+    /\[SKIP\]|\[PASS\]/{ show=0; } show' log > debug.txt
+
+  # headline of this summary
+  output "\n## $headline\n"
 
-  # summary of errors
-  if [ -s err ]; then
+  if [ -s uname.txt ]; then
     output "<pre>"
-    $ZTS_REPORT --no-maybes ./list >> Summary.md
+    outfile uname.txt
     output "</pre>"
+  fi
 
-    # generate seperate error logfile
-    ERRLOGS=$((ERRLOGS+1))
-    errfile="err-$ERRLOGS.md"
-    echo -e "\n## $headline (debugging)\n" >> $errfile
-    echo "<details><summary>Error Listing - with dmesg and dbgmsg</summary><pre>" >> $errfile
-    dd if=err bs=999k count=1 >> $errfile
-    echo "</pre></details>" >> $errfile
-  else
-    output "All tests passed :thumbsup:"
+  echo "VM disk usage before:"
+  cat disk-before.txt
+  echo "and afterwards:"
+  cat disk-afterwards.txt
+
+  if [ -s info.txt ]; then
+    output "<details><summary>Summary of all tests</summary><pre>"
+    outfile info.txt
+    output "</pre></details>"
   fi
 
-  output "<details><summary>Full Listing</summary><pre>"
-  cat list >> Summary.md
-  output "</pre></details>"
+  if [ -s list.txt ]; then
+    output "<details><summary>List of all tests</summary><pre>"
+    outfile list.txt
+    output "</pre></details>"
+  fi
+
+  for i in `seq 1 3`; do
+    output "<details><summary>vm$i - serial console</summary><pre>"
+    outfile vm$i/console.txt
+    output "</pre></details>"
+
+    output "<details><summary>vm$i - stderr of build</summary><pre>"
+    outfile vm$i/build-stderr.txt
+    output "</pre></details>"
+
+    output "<details><summary>vm$i - dmesg prerun</summary><pre>"
+    outfile vm$i/dmesg-prerun.txt
+    output "</pre></details>"
 
-  # remove tmp files
-  rm -f err list log
+    output "<details><summary>vm$i - dmesg module load</summary><pre>"
+    outfile vm$i/dmesg-module-load.txt
+    output "</pre></details>"
+  done
+
+  if [ -s error.txt ]; then
+    output "<pre>"
+    outfile error.txt
+    output "</pre>"
+  fi
+
+  logfile=$((logfile+1))
 }
 
 # check tarfiles and untar
-function check_tarfile() {
+function my_untar() {
   if [ -f "$1" ]; then
-    tar xf "$1" || error "Tarfile $1 returns some error"
-  else
-    error "Tarfile $1 not found"
+    tar xf "$1" || echo "Tarfile $1 returns some error" >> error.txt
   fi
 }
 
-# check logfile and concatenate test results
-function check_logfile() {
-  if [ -f "$1" ]; then
-    cat "$1" >> log
+# check file and copy
+function my_copy() {
+  if [ -f $1 ]; then
+    cat $1 >> $2
   else
-    error "Logfile $1 not found"
+    echo "File $1 not found!" >> error.txt
   fi
 }
 
-# sanity
-function summarize_s() {
-  headline="$1"
-  output "\n## $headline\n"
-  rm -rf testfiles
-  check_tarfile "$2/sanity.tar"
-  check_logfile "testfiles/log"
-  generate
-}
-
-# functional
-function summarize_f() {
-  headline="$1"
-  output "\n## $headline\n"
-  rm -rf testfiles
-  for i in $(seq 1 $FUNCTIONAL_PARTS); do
-    tarfile="$2-part$i/part$i.tar"
-    check_tarfile "$tarfile"
-    check_logfile "testfiles/log"
+# functional tests via qemu
+function summarize() {
+  for tarfile in Logs-functional-*/qemu-*.tar; do
+    rm -rf vm* *.txt log
+    my_untar "$tarfile"
+    osname=`cat osname.txt`
+    headline="Functional Tests: $osname"
+    my_copy "vm1/current/log" log
+    my_copy "vm2/current/log" log
+    my_copy "vm3/current/log" log
+    generate
   done
-  generate
 }
 
 # https://docs.github.com/en/[email protected]/actions/using-workflows/workflow-commands-for-github-actions#step-isolation-and-limits
 # Job summaries are isolated between steps and each step is restricted to a maximum size of 1MiB.
 # [ ] can not show all error findings here
 # [x] split files into smaller ones and create additional steps
 
-ERRLOGS=0
-if [ ! -f Summary/Summary.md ]; then
-  # first call, we do the default summary (~500k)
-  echo -n > Summary.md
-  summarize_s "Sanity Tests Ubuntu 20.04" Logs-20.04-sanity
-  summarize_s "Sanity Tests Ubuntu 22.04" Logs-22.04-sanity
-  summarize_f "Functional Tests Ubuntu 20.04" Logs-20.04-functional
-  summarize_f "Functional Tests Ubuntu 22.04" Logs-22.04-functional
-
-  cat Summary.md >> $GITHUB_STEP_SUMMARY
-  mkdir -p Summary
-  mv *.md Summary
+# first call, generate all summaries
+if [ ! -f out-0.md ]; then
+  # create ./zts-report.py for generate()
+  TEMPLATE="tests/test-runner/bin/zts-report.py.in"
+  cat $TEMPLATE| sed -e 's|@PYTHON_SHEBANG@|python3|' > ./zts-report.py
+  chmod +x ./zts-report.py
+
+  logfile="0"
+  summarize
+
+  send2github out-0.md
 else
-  # here we get, when errors where returned in first call
-  test -f Summary/err-$1.md && cat Summary/err-$1.md >> $GITHUB_STEP_SUMMARY
+  send2github out-$1.md
 fi
 
 exit 0

.github/workflows/scripts/qemu-1-setup.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+
+######################################################################
+# 1) setup the action runner to start some qemu instance
+######################################################################
+
+set -eu
+
+# docker isn't needed, free some memory
+sudo systemctl stop docker.socket
+sudo apt-get remove docker-ce-cli docker-ce podman
+
+# remove unneeded things
+sudo apt-get remove google-chrome-stable snapd
+
+# only install qemu
+sudo apt-get update
+sudo apt-get install \
+  axel cloud-image-utils daemonize guestfs-tools virt-manager \
+  linux-modules-extra-`uname -r`
+
+# disk usage afterwards
+sudo df -h /
+sudo df -h /mnt
+sudo fstrim -a
+
+# generate ssh keys
+ssh-keygen -t ed25519 -f ~/.ssh/id_ed25519 -q -N ""