CI: run only sanity check on limited OSes when changes are trivial

The commit uses some heuristics to determine the triviality of a PR.

It runs "quick" CI (i.e., only use sanity.run on fewer OSes)
if (explicity required by user):
- the *last* commit message contains a line 'ZFS-CI-Type: quick',
or if (by heuristics):
- the files changed are not in the list of specified directory, and
- all commit messages does not contain 'ZFS-CI-Type: full'.

It runs "full" CI otherwise.

See: #16561 (comment)
Signed-off-by: Shengqi Chen <[email protected]>

Author: Harry-Chen

.github/CONTRIBUTING.md

@@ -147,6 +147,9 @@ Any required reviews can then be finalized and the pull request merged.
 #### Tests and Benchmarks
 * Every pull request is tested using a GitHub Actions workflow on multiple platforms by running the [zfs-tests.sh and zloop.sh](
 https://openzfs.github.io/openzfs-docs/Developer%20Resources/Building%20ZFS.html#running-zloop-sh-and-zfs-tests-sh) test suites.
+`.github/workflows/scripts/generate-ci-type.py` is used to determine whether the pull request is "trivial", i.e., not initroducing behavior changes of any code, configuration or tests. If so, the CI will run on fewer platforms and only essential sanity tsets will run. You can control its behaviour by adding `ZFS-CI-Type` line to your commit message:
+  * If your last commit (or `HEAD` in git terms) contains a line `ZFS-CI-Type: quick`, quick mode is forced regardless of what files are changed.
+  * Otherwise, if any commit in a PR contains a line `ZFS-CI-Type: full`, full mode is forced.
 * To verify your changes conform to the [style guidelines](
 https://github.com/openzfs/zfs/blob/master/.github/CONTRIBUTING.md#style-guides
 ), please run `make checkstyle` and resolve any warnings.

.github/workflows/scripts/generate-ci-type.py

@@ -0,0 +1,106 @@
+#!/usr/bin/env python3
+
+"""
+Determine the CI type based on the change list and commit message.
+
+Prints "quick" if (explicity required by user):
+- the *last* commit message contains 'ZFS-CI-Type: quick'
+or if (heuristics):
+- the files changed are not in the list of specified directory, and
+- all commit messages does not contain 'ZFS-CI-Type: full'
+
+Otherwise prints "full".
+"""
+
+import sys
+import subprocess
+import re
+
+"""
+Patterns of files that are not considered to trigger full CI.
+"""
+FULL_RUN_IGNORE_REGEX = list(map(re.compile, [
+    r'.*\.md',
+    r'.*\.gitignore'
+]))
+
+"""
+Patterns of files that are considered to trigger full CI.
+"""
+FULL_RUN_REGEX = list(map(re.compile, [
+    r'cmd.*',
+    r'configs/.*',
+    r'META',
+    r'.*\.am',
+    r'.*\.m4',
+    r'autogen\.sh',
+    r'configure\.ac',
+    r'copy-builtin',
+    r'contrib',
+    r'etc',
+    r'include',
+    r'lib/.*',
+    r'module/.*',
+    r'scripts/.*',
+    r'tests/.*',
+    r'udev/.*'
+]))
+
+if __name__ == '__main__':
+
+    prog = sys.argv[0]
+
+    if len(sys.argv) != 3:
+        print(f'Usage: {prog} <head_ref> <base_ref>')
+        sys.exit(1)
+
+    head, base = sys.argv[1:3]
+
+    def output_type(type, reason):
+        print(f'{prog}: will run {type} CI: {reason}', file=sys.stderr)
+        print(type)
+        sys.exit(0)
+
+    # check last (HEAD) commit message
+    last_commit_message_raw = subprocess.run([
+        'git', 'show', '-s', '--format=%B', 'HEAD'
+    ], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+
+    for line in last_commit_message_raw.stdout.decode().splitlines():
+        if line.strip().lower() == 'zfs-ci-type: quick':
+            output_type('quick', f'explicitly requested by HEAD commit {head}')
+
+    # check all commit messages
+    all_commit_message_raw = subprocess.run([
+        'git', 'show', '-s',
+        '--format=ZFS-CI-Commit: %H%n%B', f'{head}...{base}'
+    ], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    all_commit_message = all_commit_message_raw.stdout.decode().splitlines()
+
+    commit_ref = head
+    for line in all_commit_message:
+        if line.startswith('ZFS-CI-Commit:'):
+            commit_ref = line.lstrip('ZFS-CI-Commit:').rstrip()
+        if line.strip().lower() == 'zfs-ci-type: full':
+            output_type('full', f'explicitly requested by commit {commit_ref}')
+
+    # check changed files
+    changed_files_raw = subprocess.run([
+        'git', 'diff', '--name-only', head, base
+    ], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    changed_files = changed_files_raw.stdout.decode().splitlines()
+
+    for f in changed_files:
+        for r in FULL_RUN_IGNORE_REGEX:
+            if r.match(f):
+                break
+        else:
+            for r in FULL_RUN_REGEX:
+                if r.match(f):
+                    output_type(
+                        'full',
+                        f'changed file "{f}" matches pattern "{r.pattern}"'
+                        )
+
+    # catch-all
+    output_type('quick', 'no changed file matches full CI patterns')

.github/workflows/scripts/qemu-6-tests.sh

@@ -48,7 +48,7 @@ if [ -z ${1:-} ]; then
   for i in $(seq 1 $VMs); do
     IP="192.168.122.1$i"
     daemonize -c /var/tmp -p vm${i}.pid -o vm${i}log.txt -- \
-      $SSH zfs@$IP $TESTS $OS $i $VMs
+      $SSH zfs@$IP $TESTS $OS $i $VMs $CI_TYPE
     # handly line by line and add info prefix
     stdbuf -oL tail -fq vm${i}log.txt \
       | while read -r line; do prefix "$i" "$line"; done &
@@ -91,6 +91,9 @@ esac
 # run functional testings and save exitcode
 cd /var/tmp
 TAGS=$2/$3
+if [ "$4" == "quick" ]; then
+  export RUNFILES="sanity.run"
+fi
 sudo dmesg -c > dmesg-prerun.txt
 mount > mount.txt
 df -h > df-prerun.txt

.github/workflows/zfs-qemu.yml

@@ -6,15 +6,48 @@ on:
 
 jobs:
 
+  test-config:
+    name: Generate configuration for QEMU test
+    runs-on: ubuntu-24.04
+    outputs:
+      test_os: ${{ steps.os.outputs.os }}
+      ci_type: ${{ steps.os.outputs.ci_type }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Generate OS config and CI type
+        id: os
+        run: |
+          FULL_OS='["almalinux8", "almalinux9", "centos-stream9", "debian11", "debian12", "fedora39", "fedora40", "freebsd13", "freebsd13r", "freebsd14", "freebsd14r", "ubuntu20", "ubuntu22", "ubuntu24"]'
+          QUICK_OS='["almalinux8", "almalinux9", "debian12", "freebsd13", "freebsd14", "ubuntu22"]'
+          # determine CI type when running on PR
+          ci_type="full"
+          if ${{ github.event_name == 'pull_request' }}; then
+            head=${{ github.event.pull_request.head.sha }}
+            base=${{ github.event.pull_request.base.sha }}
+            ci_type=$(python3 .github/workflows/scripts/generate-ci-type.py $head $base)
+          fi
+          if [ "$ci_type" == "quick" ]; then
+            os_selection="$QUICK_OS"
+          else
+            os_selection="$FULL_OS"
+          fi
+          os_json=$(echo ${os_selection} | jq -c)
+          echo "os=$os_json" >> $GITHUB_OUTPUT
+          echo "ci_type=$ci_type" >> $GITHUB_OUTPUT
+
   qemu-vm:
     name: qemu-x86
+    needs: [ test-config ]
     strategy:
       fail-fast: false
       matrix:
         # all:
         # os:  [almalinux8, almalinux9, archlinux, centos-stream9, fedora39, fedora40, debian11, debian12, freebsd13, freebsd13r, freebsd14, freebsd14r, freebsd15, ubuntu20, ubuntu22, ubuntu24]
         # openzfs:
-        os: [almalinux8, almalinux9, centos-stream9, debian11, debian12, fedora39, fedora40, freebsd13, freebsd13r, freebsd14, freebsd14r, ubuntu20, ubuntu22, ubuntu24]
+        # os: [almalinux8, almalinux9, centos-stream9, debian11, debian12, fedora39, fedora40, freebsd13, freebsd13r, freebsd14, freebsd14r, ubuntu20, ubuntu22, ubuntu24]
+        os: ${{ fromJson(needs.test-config.outputs.test_os) }}
     runs-on: ubuntu-24.04
     steps:
     - uses: actions/checkout@v4
@@ -64,6 +97,8 @@ jobs:
     - name: Run tests
       timeout-minutes: 270
       run: .github/workflows/scripts/qemu-6-tests.sh
+      env:
+        CI_TYPE: ${{ needs.test-config.outputs.ci_type }}
 
     - name: Prepare artifacts
       if: always()