Whamcloud - gitweb
LU-13169 tests: add ONLY_REPEAT parameter to repeat subtests
[fs/lustre-release.git] / lustre / tests / test-framework.sh
index 0169d6c..32dbc87 100755 (executable)
@@ -164,6 +164,10 @@ get_lustre_env() {
        export MDS1_VERSION=$(lustre_version_code mds1)
        export OST1_VERSION=$(lustre_version_code ost1)
        export CLIENT_VERSION=$(lustre_version_code client)
+
+       # Prefer using "mds1" directly instead of SINGLEMDS.
+       # Keep this for compat until it is removed from scripts.
+       export SINGLEMDS=${SINGLEMDS:-mds1}
 }
 
 init_test_env() {
@@ -291,8 +295,6 @@ init_test_env() {
        [ ! -f "$LCTL" ] && export LCTL=$(which lctl)
        export LFS=${LFS:-"$LUSTRE/utils/lfs"}
        [ ! -f "$LFS" ] && export LFS=$(which lfs)
-       SETSTRIPE=${SETSTRIPE:-"$LFS setstripe"}
-       GETSTRIPE=${GETSTRIPE:-"$LFS getstripe"}
 
        export PERM_CMD=${PERM_CMD:-"$LCTL conf_param"}
 
@@ -1548,34 +1550,42 @@ set_debug_size () {
 }
 
 set_default_debug () {
-    local debug=${1:-"$PTLDEBUG"}
-    local subsys=${2:-"$SUBSYSTEM"}
-    local debug_size=${3:-$DEBUG_SIZE}
+       local debug=${1:-"$PTLDEBUG"}
+       local subsys=${2:-"$SUBSYSTEM"}
+       local debug_size=${3:-$DEBUG_SIZE}
 
-    [ -n "$debug" ] && lctl set_param debug="$debug" >/dev/null
-    [ -n "$subsys" ] && lctl set_param subsystem_debug="${subsys# }" >/dev/null
+       [ -n "$debug" ] && lctl set_param debug="$debug" >/dev/null
+       [ -n "$subsys" ] && lctl set_param subsystem_debug="${subsys# }" >/dev/null
 
-    [ -n "$debug_size" ] && set_debug_size $debug_size > /dev/null
+       [ -n "$debug_size" ] && set_debug_size $debug_size > /dev/null
 }
 
 set_default_debug_nodes () {
        local nodes="$1"
+       local debug="${2:-"$PTLDEBUG"}"
+       local subsys="${3:-"$SUBSYSTEM"}"
+       local debug_size="${4:-$DEBUG_SIZE}"
 
        if [[ ,$nodes, = *,$HOSTNAME,* ]]; then
                nodes=$(exclude_items_from_list "$nodes" "$HOSTNAME")
                set_default_debug
        fi
 
-       do_rpc_nodes "$nodes" set_default_debug \
-               \\\"$PTLDEBUG\\\" \\\"$SUBSYSTEM\\\" $DEBUG_SIZE || true
+       [[ -z "$nodes" ]] ||
+               do_rpc_nodes "$nodes" set_default_debug \
+                       \\\"$debug\\\" \\\"$subsys\\\" $debug_size || true
 }
 
 set_default_debug_facet () {
-    local facet=$1
-    local node=$(facet_active_host $facet)
-    [ -z "$node" ] && echo "No host defined for facet $facet" && exit 1
+       local facet=$1
+       local debug="${2:-"$PTLDEBUG"}"
+       local subsys="${3:-"$SUBSYSTEM"}"
+       local debug_size="${4:-$DEBUG_SIZE}"
+       local node=$(facet_active_host $facet)
+
+       [ -n "$node" ] || error "No host defined for facet $facet"
 
-    set_default_debug_nodes $node
+       set_default_debug_nodes $node "$debug" "$subsys" $debug_size
 }
 
 set_hostid () {
@@ -2255,6 +2265,11 @@ zconf_mount() {
                exit 1
        fi
 
+       if $GSS_SK; then
+               # update mount option with skpath
+               opts=$(add_sk_mntflag $opts)
+       fi
+
        echo "Starting client: $client: $flags $opts $device $mnt"
        do_node $client mkdir -p $mnt
        if [ -n "$FILESET" -a -z "$SKIP_FILESET" ];then
@@ -2289,43 +2304,31 @@ zconf_mount() {
 }
 
 zconf_umount() {
-    local client=$1
-    local mnt=$2
-    local force
-    local busy
-    local need_kill
-
-    [ "$3" ] && force=-f
-    local running=$(do_node $client "grep -c $mnt' ' /proc/mounts") || true
-    if [ $running -ne 0 ]; then
-        echo "Stopping client $client $mnt (opts:$force)"
-        do_node $client lsof -t $mnt || need_kill=no
-        if [ "x$force" != "x" -a "x$need_kill" != "xno" ]; then
-            pids=$(do_node $client lsof -t $mnt | sort -u);
-            if [ -n $pids ]; then
-                do_node $client kill -9 $pids || true
-            fi
-        fi
-
-        busy=$(do_node $client "umount $force $mnt 2>&1" | grep -c "busy") || true
-        if [ $busy -ne 0 ] ; then
-            echo "$mnt is still busy, wait one second" && sleep 1
-            do_node $client umount $force $mnt
-        fi
-    fi
-}
-
-# Mount the file system on the MGS
-mount_mgs_client() {
-       do_facet mgs "mkdir -p $MOUNT"
-       zconf_mount $mgs_HOST $MOUNT $MOUNT_OPTS ||
-               error "unable to mount $MOUNT on MGS"
-}
+       local client=$1
+       local mnt=$2
+       local force
+       local busy
+       local need_kill
+       local running=$(do_node $client "grep -c $mnt' ' /proc/mounts") || true
+
+       [ "$3" ] && force=-f
+       [ $running -eq 0 ] && return 0
+
+       echo "Stopping client $client $mnt (opts:$force)"
+       do_node $client lsof -t $mnt || need_kill=no
+       if [ "x$force" != "x" ] && [ "x$need_kill" != "xno" ]; then
+               pids=$(do_node $client lsof -t $mnt | sort -u);
+               if [ -n "$pids" ]; then
+                       do_node $client kill -9 $pids || true
+               fi
+       fi
 
-# Unmount the file system on the MGS
-umount_mgs_client() {
-       zconf_umount $mgs_HOST $MOUNT
-       do_facet mgs "rm -rf $MOUNT"
+       busy=$(do_node $client "umount $force $mnt 2>&1" | grep -c "busy") ||
+               true
+       if [ $busy -ne 0 ] ; then
+               echo "$mnt is still busy, wait one second" && sleep 1
+               do_node $client umount $force $mnt
+       fi
 }
 
 # nodes is comma list
@@ -2732,16 +2735,15 @@ start_client_load() {
 }
 
 start_client_loads () {
-    local -a clients=(${1//,/ })
-    local numloads=${#CLIENT_LOADS[@]}
-    local testnum
+       local -a clients=(${1//,/ })
+       local numloads=${#CLIENT_LOADS[@]}
 
-    for ((nodenum=0; nodenum < ${#clients[@]}; nodenum++ )); do
-        testnum=$((nodenum % numloads))
-        start_client_load ${clients[nodenum]} ${CLIENT_LOADS[testnum]}
-    done
-    # bug 22169: wait the background threads to start
-    sleep 2
+       for ((nodenum=0; nodenum < ${#clients[@]}; nodenum++ )); do
+               local load=$((nodenum % numloads))
+               start_client_load ${clients[nodenum]} ${CLIENT_LOADS[load]}
+       done
+       # bug 22169: wait the background threads to start
+       sleep 2
 }
 
 # only for remote client
@@ -4433,6 +4435,48 @@ upper() {
        echo -n "$1" | tr '[:lower:]' '[:upper:]'
 }
 
+squash_opt() {
+       local var="$*"
+       local other=""
+       local opt_o=""
+       local opt_e=""
+       local first_e=0
+       local first_o=0
+       local take=""
+
+       var=$(echo "$var" | sed -e 's/,\( \)*/,/g')
+       for i in $(echo "$var"); do
+               if [ "$i" == "-O" ]; then
+                       take="o";
+                       first_o=$(($first_o + 1))
+                       continue;
+               fi
+               if [ "$i" == "-E" ]; then
+                       take="e";
+                       first_e=$(($first_e + 1 ))
+                       continue;
+               fi
+               case $take in
+                       "o")
+                               [ $first_o -gt 1 ] && opt_o+=",";
+                               opt_o+="$i";
+                               ;;
+                       "e")
+                               [ $first_e -gt 1 ] && opt_e+=",";
+                               opt_e+="$i";
+                               ;;
+                       *)
+                               other+=" $i";
+                               ;;
+               esac
+               take=""
+       done
+
+       echo -n "$other"
+       [ -n "$opt_o" ] && echo " -O $opt_o"
+       [ -n "$opt_e" ] && echo " -E $opt_e"
+}
+
 mkfs_opts() {
        local facet=$1
        local dev=$2
@@ -4473,7 +4517,7 @@ mkfs_opts() {
                opts+=${L_GETIDENTITY:+" --param=mdt.identity_upcall=$L_GETIDENTITY"}
 
                if [ $fstype == ldiskfs ]; then
-                       fs_mkfs_opts+="-O ea_inode"
+                       fs_mkfs_opts+="-O ea_inode,large_dir"
 
                        var=${facet}_JRN
                        if [ -n "${!var}" ]; then
@@ -4511,6 +4555,8 @@ mkfs_opts() {
        var=${type}_FS_MKFS_OPTS
        fs_mkfs_opts+=${!var:+" ${!var}"}
 
+       [ $fstype == ldiskfs ] && fs_mkfs_opts=$(squash_opt $fs_mkfs_opts)
+
        if [ -n "${fs_mkfs_opts## }" ]; then
                opts+=" --mkfsoptions=\\\"${fs_mkfs_opts## }\\\""
        fi
@@ -5109,7 +5155,7 @@ init_param_vars () {
        TIMEOUT=$(lctl get_param -n timeout)
        TIMEOUT=${TIMEOUT:-20}
 
-       if [ -n $arg1 ]; then
+       if [ -n "$arg1" ]; then
                [ "$arg1" = "server_only" ] && return
        fi
 
@@ -5226,11 +5272,11 @@ check_timeout () {
 }
 
 is_mounted () {
-    local mntpt=$1
-    [ -z $mntpt ] && return 1
-    local mounted=$(mounted_lustre_filesystems)
+       local mntpt=$1
+       [ -z $mntpt ] && return 1
+       local mounted=$(mounted_lustre_filesystems)
 
-    echo $mounted' ' | grep -w -q $mntpt' '
+       echo $mounted' ' | grep -w -q $mntpt' '
 }
 
 is_empty_dir() {
@@ -5912,8 +5958,10 @@ lru_resize_disable()
 
 flock_is_enabled()
 {
+       local mountpath=${1:-$MOUNT}
        local RC=0
-       [ -z "$(mount | grep "$MOUNT.*flock" | grep -v noflock)" ] && RC=1
+
+       [ -z "$(mount | grep "$mountpath .*flock" | grep -v noflock)" ] && RC=1
        return $RC
 }
 
@@ -5958,19 +6006,19 @@ debug_size_restore() {
 }
 
 start_full_debug_logging() {
-    debugsave
-    debug_size_save
+       debugsave
+       debug_size_save
 
-    local FULLDEBUG=-1
-    local DEBUG_SIZE=150
+       local fulldebug=-1
+       local debug_size=150
+       local nodes=$(comma_list $(nodes_list))
 
-    do_nodes $(comma_list $(nodes_list)) "$LCTL set_param debug_mb=$DEBUG_SIZE"
-    do_nodes $(comma_list $(nodes_list)) "$LCTL set_param debug=$FULLDEBUG;"
+       do_nodes $nodes "$LCTL set_param debug=$fulldebug debug_mb=$debug_size"
 }
 
 stop_full_debug_logging() {
-    debug_size_restore
-    debugrestore
+       debug_size_restore
+       debugrestore
 }
 
 # prints bash call stack
@@ -6027,7 +6075,7 @@ report_error() {
 # usage: stack_trap arg sigspec
 #
 # stack_trap() behaves like bash's built-in trap, except that it "stacks" the
-# command ``arg`` on top of previously defined commands for ``sigspec`` instead
+# command "arg" on top of previously defined commands for "sigspec" instead
 # of overwriting them.
 # stacked traps are executed in reverse order of their registration
 #
@@ -6035,7 +6083,7 @@ report_error() {
 stack_trap()
 {
        local arg="$1"
-       local sigspec="$2"
+       local sigspec="${2:-EXIT}"
 
        # Use "trap -p" to get the quoting right
        local old_trap="$(trap -p "$sigspec")"
@@ -6127,6 +6175,7 @@ skip_noexit() {
 
        [[ -n "$TESTSUITELOG" ]] &&
                echo "$TESTSUITE: SKIP: $TESTNAME $@" >> $TESTSUITELOG || true
+       unset TESTNAME
 }
 
 skip() {
@@ -6169,7 +6218,7 @@ basetest() {
     if [[ $1 = [a-z]* ]]; then
         echo $1
     else
-        echo ${1%%[a-z]*}
+       echo ${1%%[a-zA-Z]*}
     fi
 }
 
@@ -6177,59 +6226,63 @@ basetest() {
 export LAST_SKIPPED=
 export ALWAYS_SKIPPED=
 #
-# Main entry into test-framework. This is called with the name and
-# description of a test. The name is used to find the function to run
+# Main entry into test-framework. This is called with the number and
+# description of a test. The number is used to find the function to run
 # the test using "test_$name".
 #
 # This supports a variety of methods of specifying specific test to
-# run or not run.  These need to be documented...
+# run or not run:
+# - ONLY= env variable with space-separated list of test numbers to run
+# - EXCEPT= env variable with space-separated list of test numbers to exclude
 #
 run_test() {
        assert_DIR
-       export base=$(basetest $1)
-       TESTNAME=test_$1
+       local testnum=$1
+       local testmsg=$2
+       export base=$(basetest $testnum)
+       export TESTNAME=test_$testnum
        LAST_SKIPPED=
        ALWAYS_SKIPPED=
 
        # Check the EXCEPT, ALWAYS_EXCEPT and SLOW lists to see if we
        # need to skip the current test. If so, set the ALWAYS_SKIPPED flag.
-       local testname=EXCEPT_$1
-       local testname_base=EXCEPT_$base
-       if [ ${!testname}x != x ]; then
+       local isexcept=EXCEPT_$testnum
+       local isexcept_base=EXCEPT_$base
+       if [ ${!isexcept}x != x ]; then
                ALWAYS_SKIPPED="y"
-               skip_message="skipping excluded test $1"
-       elif [ ${!testname_base}x != x ]; then
+               skip_message="skipping excluded test $testnum"
+       elif [ ${!isexcept_base}x != x ]; then
                ALWAYS_SKIPPED="y"
-               skip_message="skipping excluded test $1 (base $base)"
+               skip_message="skipping excluded test $testnum (base $base)"
        fi
 
-       testname=EXCEPT_ALWAYS_$1
-       testname_base=EXCEPT_ALWAYS_$base
-       if [ ${!testname}x != x ]; then
+       isexcept=EXCEPT_ALWAYS_$testnum
+       isexcept_base=EXCEPT_ALWAYS_$base
+       if [ ${!isexcept}x != x ]; then
                ALWAYS_SKIPPED="y"
-               skip_message="skipping ALWAYS excluded test $1"
-       elif [ ${!testname_base}x != x ]; then
+               skip_message="skipping ALWAYS excluded test $testnum"
+       elif [ ${!isexcept_base}x != x ]; then
                ALWAYS_SKIPPED="y"
-               skip_message="skipping ALWAYS excluded test $1 (base $base)"
+               skip_message="skipping ALWAYS excluded test $testnum (base $base)"
        fi
 
-       testname=EXCEPT_SLOW_$1
-       testname_base=EXCEPT_SLOW_$base
-       if [ ${!testname}x != x ]; then
+       isexcept=EXCEPT_SLOW_$testnum
+       isexcept_base=EXCEPT_SLOW_$base
+       if [ ${!isexcept}x != x ]; then
                ALWAYS_SKIPPED="y"
-               skip_message="skipping SLOW test $1"
-       elif [ ${!testname_base}x != x ]; then
+               skip_message="skipping SLOW test $testnum"
+       elif [ ${!isexcept_base}x != x ]; then
                ALWAYS_SKIPPED="y"
-               skip_message="skipping SLOW test $1 (base $base)"
+               skip_message="skipping SLOW test $testnum (base $base)"
        fi
 
        # If there are tests on the ONLY list, check if the current test
        # is on that list and, if so, check if the test is to be skipped
        # and if we are supposed to honor the skip lists.
        if [ -n "$ONLY" ]; then
-               testname=ONLY_$1
-               testname_base=ONLY_$base
-               if [[ ${!testname}x != x || ${!testname_base}x != x ]]; then
+               local isonly=ONLY_$testnum
+               local isonly_base=ONLY_$base
+               if [[ ${!isonly}x != x || ${!isonly_base}x != x ]]; then
 
                        if [[ -n "$ALWAYS_SKIPPED" && -n "$HONOR_EXCEPT" ]]; then
                                LAST_SKIPPED="y"
@@ -6239,7 +6292,7 @@ run_test() {
                                [ -n "$LAST_SKIPPED" ] &&
                                        echo "" && LAST_SKIPPED=
                                ALWAYS_SKIPPED=
-                               run_one_logged $1 "$2"
+                               run_one_logged $testnum "$testmsg"
                                return $?
                        fi
 
@@ -6254,10 +6307,9 @@ run_test() {
                skip_noexit "$skip_message"
                return 0
        else
-               run_one_logged $1 "$2"
+               run_one_logged $testnum "$testmsg"
                return $?
        fi
-
 }
 
 log() {
@@ -6357,10 +6409,7 @@ group descriptors corrupted"
 #
 run_one() {
        local testnum=$1
-       local message=$2
-       export tfile=f${testnum}.${TESTSUITE}
-       export tdir=d${testnum}.${TESTSUITE}
-       export TESTNAME=test_$testnum
+       local testmsg="$2"
        local SAVE_UMASK=`umask`
        umask 0022
 
@@ -6368,7 +6417,7 @@ run_one() {
                $SETUP
        fi
 
-       banner "test $testnum: $message"
+       banner "test $testnum: $testmsg"
        test_${testnum} || error "test_$testnum failed with $?"
        cd $SAVE_PWD
        reset_fail_loc
@@ -6376,12 +6425,9 @@ run_one() {
        check_node_health
        check_dmesg_for_errors || error "Error in dmesg detected"
        if [ "$PARALLEL" != "yes" ]; then
-               ps auxww | grep -v grep | grep -q multiop &&
+               ps auxww | grep -v grep | grep -q "multiop " &&
                                        error "multiop still running"
        fi
-       unset TESTNAME
-       unset tdir
-       unset tfile
        umask $SAVE_UMASK
        $CLEANUP
        return 0
@@ -6394,49 +6440,74 @@ run_one() {
 #  - test result is saved to data file
 #
 run_one_logged() {
-       local BEFORE=$(date +%s)
-       local TEST_ERROR
-       local name=${TESTSUITE}.test_${1}.test_log.$(hostname -s).log
+       local before=$SECONDS
+       local testnum=$1
+       local testmsg=$2
+       export tfile=f${testnum}.${TESTSUITE}
+       export tdir=d${testnum}.${TESTSUITE}
+       local name=$TESTSUITE.$TESTNAME.test_log.$(hostname -s).log
        local test_log=$LOGDIR/$name
-       local zfs_log_name=${TESTSUITE}.test_${1}.zfs_log
+       local zfs_log_name=$TESTSUITE.$TESTNAME.zfs_log
        local zfs_debug_log=$LOGDIR/$zfs_log_name
-       rm -rf $LOGDIR/err
-       rm -rf $LOGDIR/ignore
-       rm -rf $LOGDIR/skip
        local SAVE_UMASK=$(umask)
+       local rc=0
        umask 0022
 
+       rm -f $LOGDIR/err $LOGDIR/ignore $LOGDIR/skip
        echo
-       log_sub_test_begin test_${1}
-       (run_one $1 "$2") 2>&1 | tee -i $test_log
-       local RC=${PIPESTATUS[0]}
-
-       [ $RC -ne 0 ] && [ ! -f $LOGDIR/err ] &&
-               echo "test_$1 returned $RC" | tee $LOGDIR/err
-
-       duration=$(($(date +%s) - $BEFORE))
-       pass "$1" "(${duration}s)"
+       # if ${ONLY_$testnum} set, repeat $ONLY_REPEAT times, otherwise once
+       local isonly=ONLY_$testnum
+       local repeat=${!isonly:+$ONLY_REPEAT}
+
+       for testiter in $(seq ${repeat:-1}); do
+               local before_sub=$SECONDS
+               log_sub_test_begin $TESTNAME
+
+               # remove temp files between repetitions to avoid test failures
+               [ -n "$append" -a -n "$DIR" -a -n "$tdir" -a -n "$tfile" ] &&
+                       rm -rf $DIR/$tdir* $DIR/$tfile*
+               # loop around subshell so stack_trap EXIT triggers each time
+               (run_one $testnum "$testmsg") 2>&1 | tee -i $append $test_log
+               rc=${PIPESTATUS[0]}
+               local append=-a
+               local duration_sub=$((SECONDS - before_sub))
+               local test_error
+
+               [[ $rc != 0 && ! -f $LOGDIR/err ]] &&
+                       echo "$TESTNAME returned $rc" | tee $LOGDIR/err
+
+               if [[ -f $LOGDIR/err ]]; then
+                       test_error=$(cat $LOGDIR/err)
+                       TEST_STATUS="FAIL"
+               elif [[ -f $LOGDIR/ignore ]]; then
+                       test_error=$(cat $LOGDIR/ignore)
+               elif [[ -f $LOGDIR/skip ]]; then
+                       test_error=$(cat $LOGDIR/skip)
+                       TEST_STATUS="SKIP"
+               else
+                       TEST_STATUS="PASS"
+               fi
 
-       if [[ -f $LOGDIR/err ]]; then
-               TEST_ERROR=$(cat $LOGDIR/err)
-       elif [[ -f $LOGDIR/ignore ]]; then
-               TEST_ERROR=$(cat $LOGDIR/ignore)
-       elif [[ -f $LOGDIR/skip ]]; then
-               TEST_ERROR=$(cat $LOGDIR/skip)
-       fi
-       log_sub_test_end $TEST_STATUS $duration "$RC" "$TEST_ERROR"
+               pass "$testnum" "($((SECONDS - before))s)"
+               log_sub_test_end $TEST_STATUS $duration_sub "$rc" "$test_error"
+               [[ $rc != 0 ]] && break
+       done
 
-       if [[ "$TEST_STATUS" != "SKIP" ]] && [[ -f $TF_SKIP ]]; then
+       if [[ "$TEST_STATUS" != "SKIP" && -f $TF_SKIP ]]; then
                rm -f $TF_SKIP
        fi
 
        if [ -f $LOGDIR/err ]; then
                log_zfs_info "$zfs_debug_log"
-               $FAIL_ON_ERROR && exit $RC
+               $FAIL_ON_ERROR && exit $rc
        fi
 
        umask $SAVE_UMASK
 
+       unset TESTNAME
+       unset tdir
+       unset tfile
+
        return 0
 }
 
@@ -6458,9 +6529,9 @@ check_grant() {
        export base=$(basetest $1)
        [ "$CHECK_GRANT" == "no" ] && return 0
 
-       testnamebase=GCHECK_ONLY_${base}
-       testname=GCHECK_ONLY_$1
-       [ ${!testnamebase}x == x -a ${!testname}x == x ] && return 0
+       local isonly_base=GCHECK_ONLY_${base}
+       local isonly=GCHECK_ONLY_$1
+       [ ${!isonly_base}x == x -a ${!isonly}x == x ] && return 0
 
        echo -n "checking grant......"
 
@@ -7113,19 +7184,21 @@ delayed_recovery_enabled () {
 ########################
 
 convert_facet2label() {
-    local facet=$1
+       local facet=$1
 
-    if [ x$facet = xost ]; then
-       facet=ost1
-    fi
+       if [ x$facet = xost ]; then
+               facet=ost1
+       elif [ x$facet = xmgs ] && combined_mgs_mds ; then
+               facet=mds1
+       fi
 
-    local varsvc=${facet}_svc
+       local varsvc=${facet}_svc
 
-    if [ -n ${!varsvc} ]; then
-        echo ${!varsvc}
-    else
-        error "No lablel for $facet!"
-    fi
+       if [ -n ${!varsvc} ]; then
+               echo ${!varsvc}
+       else
+               error "No label for $facet!"
+       fi
 }
 
 get_clientosc_proc_path() {
@@ -7585,6 +7658,8 @@ destroy_pool_int() {
        for ost in $OSTS; do
                do_facet mgs lctl pool_remove $1 $ost
        done
+       wait_update_facet $SINGLEMDS "lctl pool_list $1 | wc -l" "1" ||
+               error "MDS: pool_list $1 failed"
        do_facet mgs lctl pool_destroy $1
 }
 
@@ -8433,7 +8508,7 @@ get_block_size() {
 # ldiskfs xattrs over one block in size.  Allow both the historical
 # Lustre feature name (large_xattr) and the upstream name (ea_inode).
 large_xattr_enabled() {
-       [[ $(facet_fstype $SINGLEMDS) == zfs ]] && return 0
+       [[ $(facet_fstype $SINGLEMDS) == zfs ]] && return 1
 
        local mds_dev=$(mdsdevname ${SINGLEMDS//mds/})
 
@@ -8444,20 +8519,7 @@ large_xattr_enabled() {
 
 # Get the maximum xattr size supported by the filesystem.
 max_xattr_size() {
-    local size
-
-    if large_xattr_enabled; then
-       size=$($LCTL get_param -n llite.*.max_easize)
-    else
-        local mds_dev=$(mdsdevname ${SINGLEMDS//mds/})
-        local block_size=$(get_block_size $SINGLEMDS $mds_dev)
-
-        # maximum xattr size = size of block - size of header -
-        #                      size of 1 entry - 4 null bytes
-        size=$((block_size - 32 - 32 - 4))
-    fi
-
-    echo $size
+       $LCTL get_param -n llite.*.max_easize
 }
 
 # Dump the value of the named xattr from a file.
@@ -8698,7 +8760,7 @@ check_file_in_pool()
        local file=$1
        local pool=$2
        local tlist="$3"
-       local res=$($GETSTRIPE $file | grep 0x | cut -f2)
+       local res=$($LFS getstripe $file | grep 0x | cut -f2)
        for i in $res
        do
                for t in $tlist ; do
@@ -8770,7 +8832,7 @@ pool_set_dir() {
        local tdir=$2
        echo "Setting pool on directory $tdir"
 
-       $SETSTRIPE -c 2 -p $pool $tdir && return 0
+       $LFS setstripe -c 2 -p $pool $tdir && return 0
 
        error_noexit "Cannot set pool $pool to $tdir"
        return 1
@@ -8781,7 +8843,7 @@ pool_check_dir() {
        local tdir=$2
        echo "Checking pool on directory $tdir"
 
-       local res=$($GETSTRIPE --pool $tdir | sed "s/\s*$//")
+       local res=$($LFS getstripe --pool $tdir | sed "s/\s*$//")
        [ "$res" = "$pool" ] && return 0
 
        error_noexit "Pool on '$tdir' is '$res', not '$pool'"
@@ -8836,7 +8898,7 @@ pool_create_files() {
        for i in $(seq -w 1 $count)
        do
                local file=$tdir/spoo-$i
-               $SETSTRIPE -p $pool $file
+               $LFS setstripe -p $pool $file
                check_file_in_pool $file $pool "$tlist" || \
                        failed=$((failed + 1))
        done
@@ -8870,11 +8932,11 @@ pool_file_rel_path() {
        mkdir -p $tdir ||
                { error_noexit "unable to create $tdir"; return 1 ; }
        local file="/..$tdir/$tfile-1"
-       $SETSTRIPE -p $pool $file ||
+       $LFS setstripe -p $pool $file ||
                { error_noexit "unable to create $file" ; return 2 ; }
 
        cd $tdir
-       $SETSTRIPE -p $pool $tfile-2 || {
+       $LFS setstripe -p $pool $tfile-2 || {
                error_noexit "unable to create $tfile-2 in $tdir"
                return 3
        }
@@ -8933,7 +8995,7 @@ pool_remove_all_targets() {
                return 2
        }
        # setstripe on an empty pool should fail
-       $SETSTRIPE -p $pool $file 2>/dev/null && {
+       $LFS setstripe -p $pool $file 2>/dev/null && {
                error_noexit "expected failure when creating file" \
                                                        "with empty pool"
                return 3
@@ -8956,7 +9018,7 @@ pool_remove() {
                return 1
        }
        # setstripe on an empty pool should fail
-       $SETSTRIPE -p $pool $file 2>/dev/null && {
+       $LFS setstripe -p $pool $file 2>/dev/null && {
                error_noexit "expected failure when creating file" \
                                                        "with missing pool"
                return 2
@@ -9005,7 +9067,7 @@ check_obdidx() {
        [[ -z "$file" || -z "$expected" ]] &&
                error "check_obdidx: invalid argument!"
 
-       obdidx=$(comma_list $($GETSTRIPE $file | grep -A $OSTCOUNT obdidx |
+       obdidx=$(comma_list $($LFS getstripe $file | grep -A $OSTCOUNT obdidx |
                              grep -v obdidx | awk '{print $1}' | xargs))
 
        [[ $obdidx = $expected ]] ||
@@ -9023,8 +9085,8 @@ check_start_ost_idx() {
        [[ -z "$file" || -z "$expected" ]] &&
                error "check_start_ost_idx: invalid argument!"
 
-       start_ost_idx=$($GETSTRIPE $file | grep -A 1 obdidx | grep -v obdidx |
-                       awk '{print $1}')
+       start_ost_idx=$($LFS getstripe $file | grep -A 1 obdidx |
+                        grep -v obdidx | awk '{print $1}')
 
        [[ $start_ost_idx = $expected ]] ||
                error "OST index of the first stripe on $file is" \
@@ -9677,45 +9739,58 @@ verify_yaml_layout() {
 
 is_project_quota_supported() {
        $ENABLE_PROJECT_QUOTAS || return 1
-       [ "$(facet_fstype $SINGLEMDS)" == "ldiskfs" ] &&
-               [ $(lustre_version_code $SINGLEMDS) -gt \
-               $(version_code 2.9.55) ] &&
-               lfs --help | grep project >&/dev/null &&
-               egrep -q "7." /etc/redhat-release && return 0
 
-       if [ "$(facet_fstype $SINGLEMDS)" == "zfs" ]; then
-               [ $(lustre_version_code $SINGLEMDS) -le \
-                       $(version_code 2.10.53) ] && return 1
+       [[ "$(facet_fstype $SINGLEMDS)" == "ldiskfs" &&
+          $(lustre_version_code $SINGLEMDS) -gt $(version_code 2.9.55) ]] &&
+               do_facet mds1 lfs --help |& grep -q project && return 0
 
-               do_facet mds1 $ZPOOL upgrade -v |
-                       grep project_quota && return 0
-       fi
+       [[ "$(facet_fstype $SINGLEMDS)" == "zfs" &&
+          $(lustre_version_code $SINGLEMDS) -gt $(version_code 2.10.53) ]] &&
+               do_facet mds1 $ZPOOL get all | grep -q project_quota && return 0
 
        return 1
 }
 
+# ZFS project quota enable/disable:
+#   This  feature  will  become  active as soon as it is enabled and will never
+#   return to being disabled. Each filesystem will be upgraded automatically
+#   when remounted or when [a] new file is created under that filesystem. The
+#   upgrade can also be triggered on filesystems via `zfs set version=current
+#   <pool/fs>`. The upgrade process runs in the background and may take a
+#   while to complete for the filesystems containing a large number of files.
 enable_project_quota() {
        is_project_quota_supported || return 0
-       [ "$(facet_fstype $SINGLEMDS)" != "ldiskfs" ] && return 0
+       local zkeeper=${KEEP_ZPOOL}
+       stack_trap "KEEP_ZPOOL=$zkeeper" EXIT
+       KEEP_ZPOOL="true"
        stopall || error "failed to stopall (1)"
 
-       for num in $(seq $MDSCOUNT); do
-               do_facet mds$num $TUNE2FS -O project $(mdsdevname $num) ||
-                       error "tune2fs $(mdsdevname $num) failed"
-       done
+       local zfeat_en="feature@project_quota=enabled"
+       for facet in $(seq -f mds%g $MDSCOUNT) $(seq -f ost%g $OSTCOUNT); do
+               local facet_fstype=${facet:0:3}1_FSTYPE
+               local devname
 
-       for num in $(seq $OSTCOUNT); do
-               do_facet ost$num $TUNE2FS -O project $(ostdevname $num) ||
-                       error "tune2fs $(ostdevname $num) failed"
+               if [ "${!facet_fstype}" = "zfs" ]; then
+                       devname=$(zpool_name ${facet})
+                       do_facet ${facet} $ZPOOL set "$zfeat_en" $devname ||
+                               error "$ZPOOL set $zfeat_en $devname"
+               else
+                       [ ${facet:0:3} == "mds" ] &&
+                               devname=$(mdsdevname ${facet:3}) ||
+                               devname=$(ostdevname ${facet:3})
+                       do_facet ${facet} $TUNE2FS -O project $devname ||
+                               error "tune2fs $devname failed"
+               fi
        done
 
+       KEEP_ZPOOL="${zkeeper}"
        mount
        setupall
 }
 
 disable_project_quota() {
        is_project_quota_supported || return 0
-       [ "$(facet_fstype $SINGLEMDS)" != "ldiskfs" ] && return 0
+       [ "$mds1_FSTYPE" != "ldiskfs" ] && return 0
        stopall || error "failed to stopall (1)"
 
        for num in $(seq $MDSCOUNT); do
@@ -9935,6 +10010,11 @@ copytool()
        local action=$1
        shift
 
+       # Use default values
+       local facet=$SINGLEAGT
+       local mountpoint="${MOUNT2:-$MOUNT}"
+       local hsm_root="${hsm_root:-$(hsm_root "$facet")}"
+
        # Parse arguments
        local fail_on_error=true
        local -a misc_options
@@ -9942,11 +10022,11 @@ copytool()
                case "$1" in
                -f|--facet)
                        shift
-                       local facet="$1"
+                       facet="$1"
                        ;;
                -m|--mountpoint)
                        shift
-                       local mountpoint="$1"
+                       mountpoint="$1"
                        ;;
                -a|--archive-id)
                        shift
@@ -9954,7 +10034,7 @@ copytool()
                        ;;
                -h|--hsm-root)
                        shift
-                       local hsm_root="$1"
+                       hsm_root="$1"
                        ;;
                -b|--bwlimit)
                        shift
@@ -9971,11 +10051,6 @@ copytool()
                shift
        done
 
-       # Use default values if needed
-       local facet=${facet:-$SINGLEAGT}
-       local mountpoint="${mountpoint:-${MOUNT2:-$MOUNT}}"
-       local hsm_root="${hsm_root:-$(hsm_root "$facet")}"
-
        stack_trap "do_facet $facet rm -rf '$hsm_root'" EXIT
        do_facet $facet mkdir -p "$hsm_root" ||
                error "mkdir '$hsm_root' failed"
@@ -10220,8 +10295,8 @@ verify_comp_at_zero() {
                error "No component starting at zero(!)"
 }
 
-#TODO: This version is a placeholder, to be replaced before final commit
-SEL_VER="2.12.52"
+# version after which Self-Extending Layouts are available
+SEL_VER="2.12.55"
 
 sel_layout_sanity() {
        local file=$1