LU-18356 tests: allow server to specify except list

Allow the installed server code to specify a lists of subtests that
should be excluded by older clients when running a particular test
script.  This allows older clients to skip tests that they would
otherwise run from their local test script, but that do not work due
to server changes.

The files for each test script are read from the mds1 and ost1 facets.
The filename(s) under lustre/tests/except/ should start with the base
test script name (e.g. sanity), followed by '.', an optional unique
string to avoid conflicts between patches, and end with ".ex".
For example, sanity.ex, sanity.test_142.ex, sanity.acl.ex are valid
"sanity.sh" except filenames, but sanity-acl.ex is not.

Lines starting with '#' are comments and ignored.  Otherwise, lines
should have whitespace-separated fields on each line, as shown in the
examples below.

  #facet op need_version             jira     space_separated_subtests
  mds1    < v2_14_55-100-g8a84c7f9c7 LU-14927 0f
  linux   < 5.12.0                   LU-18102 27J
  client  == OST1_VERSION            LU-13081 151 156

The facet may be "client", "mds1", "ost1", or "linux" (client), and
"need_version" can be any Lustre (or Linux) version number or another
version name like OST1_VERSION, MDS1_VERSION, or CLIENT_VERSION.
The "op" can be standard math/logic comparisons ">=", "<", "!=", etc.

The version comparison is handled like the below pseudo-code:

  ${FACET}_VERSION $op $need_version OR except $subtests

In other words, the version check must be true or the subtest(s) will
not be run.  Checks within a single file should be ordered by subtest
number to make it easier to see whether some subtest is being skipped.

Lustre-change: https://review.whamcloud.com/56901
Lustre-commit: c4c3a7350b55ace0c38123c4b820c713f42e1cb7

Fix tests using "sh TESTSCRIPT.sh" instead of "bash TESTSCRIPT.sh"
to start a script that is calling test-framework.sh, since that
would now run afoul of the bashism that is added in this patch.

Signed-off-by: Andreas Dilger <adilger@whamcloud.com>
Change-Id: I0216d9980147ce3409807e9d7f9759fe533ebbe5

diff --git a/lustre/tests/Makefile.am b/lustre/tests/Makefile.am
index 5092c10..04a83af 100644 (file)
--- a/lustre/tests/Makefile.am
+++ b/lustre/tests/Makefile.am
@@ -55,12 +55,9 @@ nobase_noinst_SCRIPTS += racer/file_chown.sh racer/file_delxattr.sh racer/file_g
 nobase_noinst_SCRIPTS += racer/file_mknod.sh racer/file_setxattr.sh racer/file_truncate.sh
 nobase_noinst_SCRIPTS += racer/lss_create.sh racer/lss_destroy.sh
 nobase_noinst_SCRIPTS += posix/posix.cfg
-nobase_noinst_DATA = acl/cp.test acl/getfacl-noacl.test acl/inheritance.test
-nobase_noinst_DATA += acl/misc.test acl/permissions.test acl/setfacl.test
-nobase_noinst_DATA += acl/permissions_xattr.test
-nobase_noinst_DATA += acl/974.test acl/974_remote.test
-nobase_noinst_DATA += acl/2561.test acl/2561_zfs.test acl/4924.test
+nobase_noinst_DATA = acl/*.test
 nobase_noinst_DATA += clientapi/simple_test.c
+nobase_noinst_DATA += except/*.ex
 
 EXTRA_DIST = $(noinst_SCRIPTS) $(noinst_DATA) \
              $(nobase_noinst_SCRIPTS) $(nobase_noinst_DATA)

diff --git a/lustre/tests/conf-sanity.sh b/lustre/tests/conf-sanity.sh
index 7326e7e..69e54b1 100755 (executable)
--- a/lustre/tests/conf-sanity.sh
+++ b/lustre/tests/conf-sanity.sh
@@ -3023,7 +3023,7 @@ run_test 33b "Drop cancel during umount"
 
 test_34a() {
        setup
-       do_facet client "sh runmultiop_bg_pause $DIR/file O_c"
+       do_facet client "bash runmultiop_bg_pause $DIR/file O_c"
        manual_umount_client
        rc=$?
        do_facet client killall -USR1 multiop

diff --git a/lustre/tests/except/README b/lustre/tests/except/README
new file mode 100644 (file)
index 0000000..e2347e7
--- /dev/null
+++ b/lustre/tests/except/README
@@ -0,0 +1,39 @@
+# The files in this directory contain lists of subtests that should
+# be excluded by older clients when running a particular test
+# script.  This allows older clients to skip tests that they would
+# otherwise run from their local test script, but that do not work due
+# to server changes.
+#
+# The files for each test script are read from the mds1 and ost1 facet
+# nodes.  The filename(s) under lustre/tests/except/ should start with
+# the base test script name (e.g. sanity), followed by '.', an optional
+# unique string to avoid conflicts between patches, and end with ".ex".
+# Lines are handled in build_test_filter() to check subtests to exclude.
+#
+# The files for each test script are read from the mds1 and ost1 facets.
+# The filename(s) under lustre/tests/except/ should start with the base
+# test script name (e.g. sanity), followed by '.', an optional unique
+# string to avoid conflicts between patches, and end with ".ex".
+# Lines are handled in build_test_filter() to check subtests to exclude.
+#
+# Lines starting with '#' are comments and ignored.  Otherwise, lines
+# should have whitespace-separated fields on each line, as shown in the
+# examples below.
+#
+#   facet  op need_version             jira     space_separated_subtests
+#   mds1    < v2_14_55-100-g8a84c7f9c7 LU-14927 0f
+#   linux   < 5.12.0                   LU-18102 27J
+#   ost1   == CLIENT_VERSION           LU-13081 151 156
+#
+# The facet may be "client", "mds1", "ost1", or "linux" (client), and
+# "need_version" can be any Lustre (or Linux) version number or another
+# version name like OST1_VERSION, MDS1_VERSION, or CLIENT_VERSION.
+#
+# The version comparison is something like the below pseudo-code:
+#
+#         (( ${FACET}_VERSION $op $need_version )) ||
+#                 always_except $ticket $subtests
+#
+# In other words, the version check must be true or subtest(s) will
+# not be run.  Checks within a single file should be ordered by subtest
+# number to make it easier to see whether some subtest is being skipped.

diff --git a/lustre/tests/except/sanity.ex b/lustre/tests/except/sanity.ex
new file mode 100644 (file)
index 0000000..7e13fc5
--- /dev/null
+++ b/lustre/tests/except/sanity.ex
@@ -0,0 +1,2 @@
+# facet op need_version             jira     space_separated_subtests
+mds1    <  v2_14_55-100-g8a84c7f9c7 LU-14927 0f

diff --git a/lustre/tests/functions.sh b/lustre/tests/functions.sh
index 3017fdb..27c4e94 100644 (file)
--- a/lustre/tests/functions.sh
+++ b/lustre/tests/functions.sh
@@ -575,7 +575,7 @@ run_connectathon() {
        fi
        echo "tests: $tests"
        for test in $tests; do
-               local cmd="sh ./runtests -N $cnt_NRUN $test -f $testdir"
+               local cmd="bash ./runtests -N $cnt_NRUN $test -f $testdir"
                local rc=0
 
                log "$cmd"

diff --git a/lustre/tests/runtests b/lustre/tests/runtests
index 8c9d7ab..a96ca3e 100755 (executable)
--- a/lustre/tests/runtests
+++ b/lustre/tests/runtests
@@ -11,6 +11,8 @@ RUNTESTS_SRC=${RUNTESTS_SRC:-"/etc /bin"}
 [ "$COUNT" ] || COUNT=10000
 [ "$SLOW" = "no" ] && COUNT=1000
 
+build_test_filter
+
 [ "$MKDIRMANY" ] || MKDIRMANY="createmany -d"
 [ "$STATMANY" ] || STATMANY="statmany -s"
 [ "$RMDIRMANY" ] || RMDIRMANY="unlinkmany -d"

diff --git a/lustre/tests/sanity-pcc.sh b/lustre/tests/sanity-pcc.sh
index 9ce8b47..27e5302 100644 (file)
--- a/lustre/tests/sanity-pcc.sh
+++ b/lustre/tests/sanity-pcc.sh
@@ -1579,7 +1579,7 @@ test_101a() {
        # File is owned by root, make it accessible to RUNAS user
        chmod a+rw $DIR/$tdir/$tfile.shell
        stack_trap 'rm -f $DIR/$tdir/$tfile.shell' EXIT
-       do_facet $SINGLEAGT nsenter -t $PID -U -m "sh $DIR/$tdir/$tfile.shell"
+       do_facet $SINGLEAGT nsenter -t $PID -U -m "bash $DIR/$tdir/$tfile.shell"
        do_facet $SINGLEAGT nsenter -t $PID -U -m $LFS pcc attach -i $HSM_ARCHIVE_NUMBER \
                $file || error "RW-PCC attach $file failed"
        check_lpcc_state $file "readwrite"

diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh
index 69e4309..31ebdc9 100755 (executable)
--- a/lustre/tests/test-framework.sh
+++ b/lustre/tests/test-framework.sh
@@ -6510,7 +6510,8 @@ skip_eopnotsupp() {
 # Add a list of tests to ALWAYS_EXCEPT due to an issue.
 # Usage: always_except LU-4815 23 42q ...
 #
-function always_except() {
+function \
+always_except() {
        local issue="${1:-}" # single jira style issue ("LU-4815")
        local test_num
 
@@ -6538,10 +6539,32 @@ build_test_filter() {
                fi
        done
 
-       [ "$EXCEPT$ALWAYS_EXCEPT" ] &&
-               log "excepting tests: `echo $EXCEPT $ALWAYS_EXCEPT`"
-       [ "$EXCEPT_SLOW" ] &&
-               log "skipping tests SLOW=no: `echo $EXCEPT_SLOW`"
+       local nodes=$(comma_list $(facets_nodes mds1,ost1))
+       local exceptions="$LUSTRE/tests/except/$TESTSUITE.*ex"
+
+       do_nodes --verbose $nodes "ls $exceptions || true"
+       while read facet op need_ver jira subs; do
+               local have_ver_code=${facet^^*}_VERSION
+               local need_ver_code
+
+               [[ "$facet" =~ "#" ]] && continue
+               [[ "$need_ver" =~ _VERSION ]] && need_ver_code=$need_ver ||
+                       need_ver_code=$(version_code $need_ver)
+
+               (( ${!have_ver_code} $op $need_ver_code )) &&
+                       echo "- see $facet $op $need_ver for $jira, go $subs" ||
+               {
+                       log "- need $facet $op $need_ver for $jira, skip $subs"
+                       for E in $subs; do
+                               eval EXCEPT_${E}=true
+                       done
+               }
+       done < <(do_nodes $nodes "cat $exceptions 2>/dev/null ||true" | sort -u)
+
+       [[ -z "$EXCEPT$ALWAYS_EXCEPT" ]] ||
+               log "excepting tests: $(echo $EXCEPT $ALWAYS_EXCEPT)"
+       [[ -z "$EXCEPT_SLOW" ]] ||
+               log "skipping tests SLOW=no: $(echo $EXCEPT_SLOW)"
        for E in $EXCEPT; do
                eval EXCEPT_${E}=true
        done