LU-12045 tests: honor EXCEPT tests when using ONLY list

The Lustre test framework allows a user to specify a subset
of tests to run using the ONLY parameter or --only flag.
The test framwork also allows the user to specify a list of
tests to skip using the EXCEPT or ALWAYS_EXCEPT parameters.
By default, if the ONLY parameter or --only flag is used,
the EXCEPT and ALWAYS_EXCEPT lists are ignored.

Add a flag to auster, -H, and an environment variable,
HONOR_EXCEPT, to skip the tests on the ALWAYS_EXCEPT,
EXCEPT and SLOW lists when using the ONLY/--only parameter.

Test-Parameters: trivial
Test-Parameters: envdefinitions=ONLY="40-43" testlist=sanity
Test-Parameters: envdefinitions=ONLY="40-43" austeroptions=-H testlist=sanity
Test-Parameters: envdefinitions=SLOW="no",ONLY="27" testlist=sanity
Test-Parameters: envdefinitions=SLOW="no",ONLY="27" austeroptions=-H testlist=sanity
Test-Parameters: envdefinitions=SLOW="yes",ONLY="27" testlist=sanity
Test-Parameters: envdefinitions=SLOW="yes",ONLY="27" austeroptions=-H testlist=sanity
Signed-off-by: James Nunez <jnunez@whamcloud.com>
Change-Id: I173e48e1d2dc3b404d148146639a13148bc48a3d
Reviewed-on: https://review.whamcloud.com/34938
Tested-by: Jenkins
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Wei Liu <sarah@whamcloud.com>
Reviewed-by: Jian Yu <yujian@whamcloud.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>

diff --git a/lustre/tests/auster b/lustre/tests/auster
index c8093a5..89f15b2 100755 (executable)
--- a/lustre/tests/auster
+++ b/lustre/tests/auster
@@ -14,7 +14,7 @@ export TF_FAIL=/tmp/tf.fail
 export TF_SKIP=/tmp/tf.skip
 
 usage() {
 export TF_SKIP=/tmp/tf.skip
 
 usage() {

-    less -F <<EOF
+       less -F <<EOF

 Usage ${0##*/} [options]  suite [suite options] [suite [suite options]]
 Run Lustre regression tests suites.
       -c CONFIG Test environment config file
 Usage ${0##*/} [options]  suite [suite options] [suite [suite options]]
 Run Lustre regression tests suites.
       -c CONFIG Test environment config file


@@ -23,6 +23,7 @@ Run Lustre regression tests suites.
       -f STR    Config name (cfg/<name>.sh)
       -g GROUP  Test group file (Overrides tests listed on command line)
       -S TESTSUITE First test suite to run allows for restarts
       -f STR    Config name (cfg/<name>.sh)
       -g GROUP  Test group file (Overrides tests listed on command line)
       -S TESTSUITE First test suite to run allows for restarts

+      -H        Honor the EXCEPT and ALWAYS_EXCEPT list when --only is used

       -i N      Repeat tests N times (default 1). A new directory
                 will be created under LOGDIR for each iteration.
       -k        Don't stop when subtests fail
       -i N      Repeat tests N times (default 1). A new directory
                 will be created under LOGDIR for each iteration.
       -k        Don't stop when subtests fail


@@ -57,7 +58,7 @@ Run all tests in the regression group 5 times using large config.
   auster -f large -g test-groups/regression -i 5
 
 EOF
   auster -f large -g test-groups/regression -i 5
 
 EOF

-    exit
+       exit

 }
 
 dry_run=false
 }
 
 dry_run=false


@@ -68,9 +69,10 @@ upload_logs=false
 reformat=false
 script_lang=bash
 test_logs_dir=/tmp/test_logs/$(date +%Y-%m-%d)/$(date +%H%M%S)
 reformat=false
 script_lang=bash
 test_logs_dir=/tmp/test_logs/$(date +%Y-%m-%d)/$(date +%H%M%S)

+export HONOR_EXCEPT=

 export ${SLOW:=no}
 export ${NAME:=local}
 export ${SLOW:=no}
 export ${NAME:=local}

-while getopts "c:d:D:nkf:S:g:i:rRslL:hv" opt
+while getopts "c:d:D:nkf:S:g:Hi:rRslL:hv" opt

 do
        case "$opt" in
                c) export CONFIG=$OPTARG;;
 do
        case "$opt" in
                c) export CONFIG=$OPTARG;;


@@ -81,6 +83,7 @@ do
                k) export FAIL_ON_ERROR=false;;
                n) dry_run=:;;
                v) verbose=:;;
                k) export FAIL_ON_ERROR=false;;
                n) dry_run=:;;
                v) verbose=:;;

+               H) export HONOR_EXCEPT="y";;

                i) repeat_count=$OPTARG;;
                f) NAME=$OPTARG;;
                R) do_reset=:;;
                i) repeat_count=$OPTARG;;
                f) NAME=$OPTARG;;
                R) do_reset=:;;

diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh
index 1315d8a..3a442b3 100755 (executable)
--- a/lustre/tests/test-framework.sh
+++ b/lustre/tests/test-framework.sh
@@ -6007,63 +6007,78 @@ export ALWAYS_SKIPPED=
 #
 run_test() {
        assert_DIR
 #
 run_test() {
        assert_DIR

-

        export base=$(basetest $1)
        export base=$(basetest $1)

-       if [ -n "$ONLY" ]; then
-               testname=ONLY_$1
-               if [ ${!testname}x != x ]; then
-                       [ -n "$LAST_SKIPPED" ] && echo "" && LAST_SKIPPED=
-                       run_one_logged $1 "$2"
-                       return $?
-               fi
-               testname=ONLY_$base
-               if [ ${!testname}x != x ]; then
-                       [ -n "$LAST_SKIPPED" ] && echo "" && LAST_SKIPPED=
-                       run_one_logged $1 "$2"
-                       return $?
-               fi
-               LAST_SKIPPED="y"
-               return 0
-       fi
+       TESTNAME=test_$1
+       LAST_SKIPPED=
+       ALWAYS_SKIPPED=

 
 

-       LAST_SKIPPED="y"
-       ALWAYS_SKIPPED="y"
-       testname=EXCEPT_$1
+       # Check the EXCEPT, ALWAYS_EXCEPT and SLOW lists to see if we
+       # need to skip the current test. If so, set the ALWAYS_SKIPPED flag.
+       local testname=EXCEPT_$1
+       local testname_base=EXCEPT_$base

        if [ ${!testname}x != x ]; then
        if [ ${!testname}x != x ]; then

-               TESTNAME=test_$1 skip_noexit "skipping excluded test $1"
-               return 0
-       fi
-       testname=EXCEPT_$base
-       if [ ${!testname}x != x ]; then
-               TESTNAME=test_$1 skip_noexit "skipping excluded test $1 (base $base)"
-               return 0
+               ALWAYS_SKIPPED="y"
+               skip_message="skipping excluded test $1"
+       elif [ ${!testname_base}x != x ]; then
+               ALWAYS_SKIPPED="y"
+               skip_message="skipping excluded test $1 (base $base)"

        fi
        fi

+

        testname=EXCEPT_ALWAYS_$1
        testname=EXCEPT_ALWAYS_$1

+       testname_base=EXCEPT_ALWAYS_$base

        if [ ${!testname}x != x ]; then
        if [ ${!testname}x != x ]; then

-               TESTNAME=test_$1 skip_noexit "skipping ALWAYS excluded test $1"
-               return 0
-       fi
-       testname=EXCEPT_ALWAYS_$base
-       if [ ${!testname}x != x ]; then
-               TESTNAME=test_$1 skip_noexit "skipping ALWAYS excluded test $1 (base $base)"
-               return 0
+               ALWAYS_SKIPPED="y"
+               skip_message="skipping ALWAYS excluded test $1"
+       elif [ ${!testname_base}x != x ]; then
+               ALWAYS_SKIPPED="y"
+               skip_message="skipping ALWAYS excluded test $1 (base $base)"

        fi
        fi

+

        testname=EXCEPT_SLOW_$1
        testname=EXCEPT_SLOW_$1

+       testname_base=EXCEPT_SLOW_$base

        if [ ${!testname}x != x ]; then
        if [ ${!testname}x != x ]; then

-               TESTNAME=test_$1 skip_noexit "skipping SLOW test $1"
-               return 0
+               ALWAYS_SKIPPED="y"
+               skip_message="skipping SLOW test $1"
+       elif [ ${!testname_base}x != x ]; then
+               ALWAYS_SKIPPED="y"
+               skip_message="skipping SLOW test $1 (base $base)"

        fi
        fi

-       testname=EXCEPT_SLOW_$base
-       if [ ${!testname}x != x ]; then
-               TESTNAME=test_$1 skip_noexit "skipping SLOW test $1 (base $base)"
-               return 0
+
+       # If there are tests on the ONLY list, check if the current test
+       # is on that list and, if so, check if the test is to be skipped
+       # and if we are supposed to honor the skip lists.
+       if [ -n "$ONLY" ]; then
+               testname=ONLY_$1
+               testname_base=ONLY_$base
+               if [[ ${!testname}x != x || ${!testname_base}x != x ]]; then
+
+                       if [[ -n "$ALWAYS_SKIPPED" && -n "$HONOR_EXCEPT" ]]; then
+                               LAST_SKIPPED="y"
+                               skip_noexit "$skip_message"
+                               return 0
+                       else
+                               [ -n "$LAST_SKIPPED" ] &&
+                                       echo "" && LAST_SKIPPED=
+                               ALWAYS_SKIPPED=
+                               run_one_logged $1 "$2"
+                               return $?
+                       fi
+
+               else
+                       LAST_SKIPPED="y"
+                       return 0
+               fi

        fi
 
        fi
 

-       LAST_SKIPPED=
-       ALWAYS_SKIPPED=
-       run_one_logged $1 "$2"
+       if [ -n "$ALWAYS_SKIPPED" ]; then
+               LAST_SKIPPED="y"
+               skip_noexit "$skip_message"
+               return 0
+       else
+               run_one_logged $1 "$2"
+               return $?
+       fi

 
 

-       return $?

 }
 
 log() {
 }
 
 log() {