X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=blobdiff_plain;f=lustre%2Ftests%2Ftest-framework.sh;h=c507425a361f107fc5cf260e85cc798029a82ea3;hp=46ef4a6c3327fc421003bd3c6a204f58e9515849;hb=608cce73d5149cb4d317ee189db94f0b6ade1ff5;hpb=98566a61d0560aca1befbfaeb87470c34f25d2e9

diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh
index 46ef4a6..c507425 100755
--- a/lustre/tests/test-framework.sh
+++ b/lustre/tests/test-framework.sh
@@ -293,6 +293,8 @@ init_test_env() {
 	fi
 	export RSYNC_RSH=${RSYNC_RSH:-rsh}
 
+	export LNETCTL=${LNETCTL:-"$LUSTRE/../lnet/utils/lnetctl"}
+	[ ! -f "$LNETCTL" ] && export LNETCTL=$(which lnetctl 2> /dev/null)
 	export LCTL=${LCTL:-"$LUSTRE/utils/lctl"}
 	[ ! -f "$LCTL" ] && export LCTL=$(which lctl)
 	export LFS=${LFS:-"$LUSTRE/utils/lfs"}
@@ -426,14 +428,19 @@ init_test_env() {
 
 	# Constants used in more than one test script
 	export LOV_MAX_STRIPE_COUNT=2000
+	export DELETE_OLD_POOLS=${DELETE_OLD_POOLS:-false}
+	export KEEP_POOLS=${KEEP_POOLS:-false}
 
 	export MACHINEFILE=${MACHINEFILE:-$TMP/$(basename $0 .sh).machines}
 	. ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
 	get_lustre_env
 
 	# use localrecov to enable recovery for local clients, LU-12722
-	[[ $MDS1_VERSION -lt $(version_code 2.13.52) ]] ||
+	[[ $MDS1_VERSION -lt $(version_code 2.13.52) ]] || {
 		export MDS_MOUNT_OPTS=${MDS_MOUNT_OPTS:-"-o localrecov"}
+		export MGS_MOUNT_OPTS=${MGS_MOUNT_OPTS:-"-o localrecov"}
+	}
+
 	[[ $OST1_VERSION -lt $(version_code 2.13.52) ]] ||
 		export OST_MOUNT_OPTS=${OST_MOUNT_OPTS:-"-o localrecov"}
 }
@@ -531,6 +538,17 @@ module_loaded () {
 	/sbin/lsmod | grep -q "^\<$1\>"
 }
 
+check_lfs_df_ret_val() {
+	# Ignore only EOPNOTSUPP (which is 95; Operation not supported) error
+	# returned by 'lfs df' for valid dentry but not a lustrefs.
+	#
+	# 'lfs df' historically always returned success(0) instead of
+	# EOPNOTSUPP. This function for compatibility reason, ignores and
+	# masquerades EOPNOTSUPP as success.
+	[[ $1 -eq 95 ]] && return 0
+	return $1
+}
+
 PRLFS=false
 lustre_insmod() {
 	local module=$1
@@ -675,9 +693,9 @@ load_modules_local() {
 	# that obviously has nothing to do with this Lustre run
 	# Disable automatic memory scanning to avoid perf hit.
 	if [ -f /sys/kernel/debug/kmemleak ] ; then
-		echo scan=off > /sys/kernel/debug/kmemleak
-		echo scan > /sys/kernel/debug/kmemleak
-		echo clear > /sys/kernel/debug/kmemleak
+		echo scan=off > /sys/kernel/debug/kmemleak || true
+		echo scan > /sys/kernel/debug/kmemleak || true
+		echo clear > /sys/kernel/debug/kmemleak || true
 	fi
 
 	echo Loading modules from $LUSTRE
@@ -821,9 +839,7 @@ check_mem_leak () {
 	fi
 }
 
-unload_modules() {
-	wait_exit_ST client # bug 12845
-
+unload_modules_local() {
 	$LUSTRE_RMMOD ldiskfs || return 2
 
 	[ -f /etc/udev/rules.d/99-lustre-test.rules ] &&
@@ -831,15 +847,23 @@ unload_modules() {
 	udevadm control --reload-rules
 	udevadm trigger
 
+	check_mem_leak || return 254
+
+	return 0
+}
+
+unload_modules() {
+	local rc=0
+
+	wait_exit_ST client # bug 12845
+
+	unload_modules_local || rc=$?
+
 	if $LOAD_MODULES_REMOTE; then
 		local list=$(comma_list $(remote_nodes_list))
 		if [ -n "$list" ]; then
 			echo "unloading modules on: '$list'"
-			do_rpc_nodes "$list" $LUSTRE_RMMOD ldiskfs
-			do_rpc_nodes "$list" check_mem_leak
-			do_rpc_nodes "$list" "rm -f /etc/udev/rules.d/99-lustre-test.rules"
-			do_rpc_nodes "$list" "udevadm control --reload-rules"
-			do_rpc_nodes "$list" "udevadm trigger"
+			do_rpc_nodes "$list" unload_modules_local
 		fi
 	fi
 
@@ -850,10 +874,9 @@ unload_modules() {
 			rm -f $sbin_mount
 	fi
 
-	check_mem_leak || return 254
+	[[ $rc -eq 0 ]] && echo "modules unloaded."
 
-	echo "modules unloaded."
-	return 0
+	return $rc
 }
 
 fs_log_size() {
@@ -1311,20 +1334,6 @@ devicelabel() {
 	echo -n $label
 }
 
-mdsdevlabel() {
-	local num=$1
-	local device=$(mdsdevname $num)
-	local label=$(devicelabel mds$num ${device} | grep -v "CMD: ")
-	echo -n $label
-}
-
-ostdevlabel() {
-	local num=$1
-	local device=$(ostdevname $num)
-	local label=$(devicelabel ost$num ${device} | grep -v "CMD: ")
-	echo -n $label
-}
-
 #
 # Get the device of a facet.
 #
@@ -2196,7 +2205,7 @@ restore_quota() {
 	if [ "$old_MDT_QUOTA_TYPE" ]; then
 		if [[ $PERM_CMD == *"set_param -P"* ]]; then
 			do_facet mgs $PERM_CMD \
-				osd-*.$FSNAME-MDT*.quota_slave.enable = \
+				osd-*.$FSNAME-MDT*.quota_slave.enabled = \
 				$old_MDT_QUOTA_TYPE
 		else
 			do_facet mgs $PERM_CMD \
@@ -2206,7 +2215,7 @@ restore_quota() {
 	if [ "$old_OST_QUOTA_TYPE" ]; then
 		if [[ $PERM_CMD == *"set_param -P"* ]]; then
 			do_facet mgs $PERM_CMD \
-				osd-*.$FSNAME-OST*.quota_slave.enable = \
+				osd-*.$FSNAME-OST*.quota_slave.enabled = \
 				$old_OST_QUOTA_TYPE
 		else
 			do_facet mgs $LCTL conf_param \
@@ -2220,6 +2229,7 @@ restore_quota() {
 # This will allow fixing the "lfs df" summary line in the future.
 lfs_df() {
 	$LFS df $* | sed -e 's/filesystem /filesystem_/'
+	check_lfs_df_ret_val $?
 }
 
 # Get free inodes on the MDT specified by mdt index, free indoes on
@@ -2267,9 +2277,9 @@ setup_quota(){
 
 	if [[ $PERM_CMD == *"set_param -P"* ]]; then
 		do_facet mgs $PERM_CMD \
-			osd-*.$FSNAME-MDT*.quota_slave.enable=$QUOTA_TYPE
+			osd-*.$FSNAME-MDT*.quota_slave.enabled=$QUOTA_TYPE
 		do_facet mgs $PERM_CMD \
-			osd-*.$FSNAME-OST*.quota_slave.enable=$QUOTA_TYPE
+			osd-*.$FSNAME-OST*.quota_slave.enabled=$QUOTA_TYPE
 	else
 		do_facet mgs $PERM_CMD $FSNAME.quota.mdt=$QUOTA_TYPE ||
 			error "set mdt quota type failed"
@@ -2989,35 +2999,6 @@ stop_client_loads() {
 }
 # End recovery-scale functions
 
-# verify that lustre actually cleaned up properly
-cleanup_check() {
-	VAR=$(lctl get_param -n catastrophe 2>&1)
-	if [ $? = 0 ] ; then
-		if [ $VAR != 0 ]; then
-			error "LBUG/LASSERT detected"
-		fi
-	fi
-	BUSY=$(dmesg | grep -i destruct || true)
-	if [ -n "$BUSY" ]; then
-		echo "$BUSY" 1>&2
-		[ -e $TMP/debug ] && mv $TMP/debug $TMP/debug-busy.$(date +%s)
-		exit 205
-	fi
-
-	check_mem_leak || exit 204
-
-	[[ $($LCTL dl 2>/dev/null | wc -l) -gt 0 ]] && $LCTL dl &&
-		echo "$TESTSUITE: lustre didn't clean up..." 1>&2 &&
-		return 202 || true
-
-	if module_loaded lnet || module_loaded libcfs; then
-		echo "$TESTSUITE: modules still loaded..." 1>&2
-		/sbin/lsmod 1>&2
-		return 203
-	fi
-	return 0
-}
-
 ##
 # wait for a command to return the expected result
 #
@@ -3032,15 +3013,21 @@ cleanup_check() {
 # If --verbose is passed as the first argument, the result is printed on each
 # value change, otherwise it is only printed after every 10s interval.
 #
+# If --quiet is passed as the first/second argument, the do_node() command
+# will not print the remote command before executing it each time.
+#
 # Using wait_update_cond() or related helper function is preferable to adding
 # a "long enough" wait for some state to change in the background, since
 # "long enough" may be too short due to tunables, system config, or running in
 # a VM, and must by necessity wait too long for most cases or risk failure.
 #
-# usage: wait_update_cond [--verbose] node check cond expect [max_wait]
+# usage: wait_update_cond [--verbose] [--quiet] node check cond expect [max_wait]
 wait_update_cond() {
-	local verbose=false
-	[[ "$1" == "--verbose" ]] && verbose=true && shift
+	local verbose
+	local quiet
+
+	[[ "$1" == "--verbose" ]] && verbose="$1" && shift
+	[[ "$1" == "--quiet" || "$1" == "-q" ]] && quiet="$1" && shift
 
 	local node=$1
 	local check="$2"
@@ -3055,7 +3042,7 @@ wait_update_cond() {
 	local print=10
 
 	while (( $waited <= $max_wait )); do
-		result=$(do_node $node "$check")
+		result=$(do_node $quiet $node "$check")
 
 		eval [[ "'$result'" $cond "'$expect'" ]]
 		if [[ $? == 0 ]]; then
@@ -3063,7 +3050,7 @@ wait_update_cond() {
 				echo "Updated after ${waited}s: want '$expect' got '$result'"
 			return 0
 		fi
-		if $verbose && [[ "$result" != "$prev_result" ]]; then
+		if [[ -n "$verbose" && "$result" != "$prev_result" ]]; then
 			[[ -n "$prev_result" ]] &&
 				echo "Changed after ${waited}s: from '$prev_result' to '$result'"
 			prev_result="$result"
@@ -3077,23 +3064,29 @@ wait_update_cond() {
 	return 3
 }
 
-# usage: wait_update [--verbose] node check expect [max_wait]
+# usage: wait_update [--verbose] [--quiet] node check expect [max_wait]
 wait_update() {
-	local verbose=
-	[ "$1" = "--verbose" ] && verbose="$1" && shift
+	local verbose
+	local quiet
+
+	[[ "$1" == "--verbose" ]] && verbose="$1" && shift
+	[[ "$1" == "--quiet" || "$1" == "-q" ]] && quiet="$1" && shift
 
 	local node="$1"
 	local check="$2"
 	local expect="$3"
 	local max_wait=$4
 
-	wait_update_cond $verbose $node "$check" "==" "$expect" $max_wait
+	wait_update_cond $verbose $quiet $node "$check" "==" "$expect" $max_wait
 }
 
 # usage: wait_update_facet_cond [--verbose] facet check cond expect [max_wait]
 wait_update_facet_cond() {
-	local verbose=
-	[ "$1" = "--verbose" ] && verbose="$1" && shift
+	local verbose
+	local quiet
+
+	[[ "$1" == "--verbose" ]] && verbose="$1" && shift
+	[[ "$1" == "--quiet" || "$1" == "-q" ]] && quiet="$1" && shift
 
 	local node=$(facet_active_host $1)
 	local check="$2"
@@ -3101,20 +3094,23 @@ wait_update_facet_cond() {
 	local expect="$4"
 	local max_wait=$5
 
-	wait_update_cond $verbose $node "$check" "$cond" "$expect" $max_wait
+	wait_update_cond $verbose $quiet $node "$check" "$cond" "$expect" $max_wait
 }
 
 # usage: wait_update_facet [--verbose] facet check expect [max_wait]
 wait_update_facet() {
-	local verbose=
-	[ "$1" = "--verbose" ] && verbose="$1" && shift
+	local verbose
+	local quiet
+
+	[[ "$1" == "--verbose" ]] && verbose="$1" && shift
+	[[ "$1" == "--quiet" || "$1" == "-q" ]] && quiet="$1" && shift
 
 	local node=$(facet_active_host $1)
 	local check="$2"
 	local expect="$3"
 	local max_wait=$4
 
-	wait_update_cond $verbose $node "$check" "==" "$expect" $max_wait
+	wait_update_cond $verbose $quiet $node "$check" "==" "$expect" $max_wait
 }
 
 sync_all_data() {
@@ -3450,14 +3446,15 @@ wait_osts_up() {
 }
 
 wait_destroy_complete () {
-	echo "Waiting for local destroys to complete"
+	echo "Waiting for MDT destroys to complete"
 	# MAX value shouldn't be big as this mean server responsiveness
 	# never increase this just to make test pass but investigate
 	# why it takes so long time
-	local MAX=5
+	local MAX=${1:-5}
 	local WAIT=0
+	local list=$(comma_list $(mdts_nodes))
 	while [ $WAIT -lt $MAX ]; do
-		local -a RPCs=($($LCTL get_param -n osc.*.destroys_in_flight))
+		local -a RPCs=($(do_nodes $list $LCTL get_param -n osp.*.destroys_in_flight))
 		local con=1
 		local i
 
@@ -3472,7 +3469,7 @@ wait_destroy_complete () {
 		echo "Waiting ${WAIT}s for local destroys to complete"
 		WAIT=$((WAIT + 1))
 	done
-	echo "Local destroys weren't done in $MAX sec."
+	echo "MDT destroys weren't done in $MAX sec."
 	return 1
 }
 
@@ -3537,12 +3534,20 @@ wait_remote_prog () {
 
 lfs_df_check() {
 	local clients=${1:-$CLIENTS}
+	local rc
 
 	if [ -z "$clients" ]; then
-		$LFS df $MOUNT
+		$LFS df $MOUNT > /dev/null
+		rc=$?
 	else
 		$PDSH $clients "$LFS df $MOUNT" > /dev/null
+		rc=$?
 	fi
+
+	check_lfs_df_ret_val $rc
+	rc=$?
+
+	return $rc
 }
 
 clients_up() {
@@ -3670,10 +3675,6 @@ facet_failover() {
 	done
 }
 
-obd_name() {
-    local facet=$1
-}
-
 replay_barrier() {
 	local facet=$1
 	do_facet $facet "sync; sync; sync"
@@ -4044,26 +4045,27 @@ change_active() {
 }
 
 do_node() {
-    local verbose=false
-    # do not stripe off hostname if verbose, bug 19215
-    if [ x$1 = x--verbose ]; then
-        shift
-        verbose=true
-    fi
+	local verbose
+	local quiet
 
-    local HOST=$1
-    shift
-    local myPDSH=$PDSH
-    if [ "$HOST" = "$HOSTNAME" ]; then
-        myPDSH="no_dsh"
-    elif [ -z "$myPDSH" -o "$myPDSH" = "no_dsh" ]; then
-        echo "cannot run remote command on $HOST with $myPDSH"
-        return 128
-    fi
-    if $VERBOSE; then
-        echo "CMD: $HOST $@" >&2
-        $myPDSH $HOST "$LCTL mark \"$@\"" > /dev/null 2>&1 || :
-    fi
+	# do not strip off hostname if verbose, b=19215
+	[[ "$1" == "--verbose" ]] && verbose="$1" && shift
+	[[ "$1" == "--quiet" || "$1" == "-q" ]] && quiet="$1" && shift
+
+	local HOST=$1
+	shift
+	local myPDSH=$PDSH
+
+	if [ "$HOST" = "$HOSTNAME" ]; then
+		myPDSH="no_dsh"
+	elif [ -z "$myPDSH" -o "$myPDSH" = "no_dsh" ]; then
+		echo "cannot run remote command on $HOST with $myPDSH"
+		return 128
+	fi
+	if $VERBOSE && [[ -z "$quiet" ]]; then
+		echo "CMD: $HOST $@" >&2
+		$myPDSH $HOST "$LCTL mark \"$@\"" > /dev/null 2>&1 || :
+	fi
 
 	if [[ "$myPDSH" == "rsh" ]] ||
 	   [[ "$myPDSH" == *pdsh* && "$myPDSH" != *-S* ]]; then
@@ -4080,7 +4082,7 @@ do_node() {
 			return 0
 	fi
 
-    if $verbose ; then
+    if [[ -n "$verbose" ]]; then
         # print HOSTNAME for myPDSH="no_dsh"
         if [[ $myPDSH = no_dsh ]]; then
             $myPDSH $HOST "(PATH=\$PATH:$RLUSTRE/utils:$RLUSTRE/tests:/sbin:/usr/sbin; cd $RPWD; LUSTRE=\"$RLUSTRE\" sh -c \"$@\")" | sed -e "s/^/${HOSTNAME}: /"
@@ -4093,12 +4095,8 @@ do_node() {
     return ${PIPESTATUS[0]}
 }
 
-do_nodev() {
-    do_node --verbose "$@"
-}
-
 single_local_node () {
-   [ "$1" = "$HOSTNAME" ]
+	[ "$1" = "$HOSTNAME" ]
 }
 
 # Outputs environment variable assignments that should be passed to remote nodes
@@ -4144,45 +4142,42 @@ get_env_vars() {
 }
 
 do_nodes() {
-    local verbose=false
-    # do not stripe off hostname if verbose, bug 19215
-    if [ x$1 = x--verbose ]; then
-        shift
-        verbose=true
-    fi
+	local verbose
+	local quiet
 
-    local rnodes=$1
-    shift
+	# do not strip off hostname if verbose, b=19215
+	[[ "$1" == "--verbose" ]] && verbose="$1" && shift
+	[[ "$1" == "--quiet" || "$1" == "-q" ]] && quiet="$1" && shift
 
-    if single_local_node $rnodes; then
-        if $verbose; then
-           do_nodev $rnodes "$@"
-        else
-           do_node $rnodes "$@"
-        fi
-        return $?
-    fi
+	local rnodes=$1
+	shift
 
-    # This is part from do_node
-    local myPDSH=$PDSH
+	if single_local_node $rnodes; then
+		do_node $verbose $quiet $rnodes "$@"
+		return $?
+	fi
 
-    [ -z "$myPDSH" -o "$myPDSH" = "no_dsh" -o "$myPDSH" = "rsh" ] && \
-        echo "cannot run remote command on $rnodes with $myPDSH" && return 128
+	# This is part from do_node
+	local myPDSH=$PDSH
 
-    export FANOUT=$(get_node_count "${rnodes//,/ }")
-    if $VERBOSE; then
-        echo "CMD: $rnodes $@" >&2
-        $myPDSH $rnodes "$LCTL mark \"$@\"" > /dev/null 2>&1 || :
-    fi
+	[ -z "$myPDSH" -o "$myPDSH" = "no_dsh" -o "$myPDSH" = "rsh" ] &&
+		echo "cannot run remote command on $rnodes with $myPDSH" &&
+		return 128
 
-    # do not replace anything from pdsh output if -N is used
-    # -N     Disable hostname: prefix on lines of output.
-    if $verbose || [[ $myPDSH = *-N* ]]; then
-        $myPDSH $rnodes "(PATH=\$PATH:$RLUSTRE/utils:$RLUSTRE/tests:/sbin:/usr/sbin; cd $RPWD; LUSTRE=\"$RLUSTRE\" $(get_env_vars) sh -c \"$@\")"
-    else
-        $myPDSH $rnodes "(PATH=\$PATH:$RLUSTRE/utils:$RLUSTRE/tests:/sbin:/usr/sbin; cd $RPWD; LUSTRE=\"$RLUSTRE\" $(get_env_vars) sh -c \"$@\")" | sed -re "s/^[^:]*: //g"
-    fi
-    return ${PIPESTATUS[0]}
+	export FANOUT=$(get_node_count "${rnodes//,/ }")
+	if $VERBOSE && [[ -z "$quiet" ]]; then
+		echo "CMD: $rnodes $@" >&2
+		$myPDSH $rnodes "$LCTL mark \"$@\"" > /dev/null 2>&1 || :
+	fi
+
+	# do not replace anything from pdsh output if -N is used
+	# -N     Disable hostname: prefix on lines of output.
+	if [[ -n "$verbose" || $myPDSH = *-N* ]]; then
+		$myPDSH $rnodes "(PATH=\$PATH:$RLUSTRE/utils:$RLUSTRE/tests:/sbin:/usr/sbin; cd $RPWD; LUSTRE=\"$RLUSTRE\" $(get_env_vars) sh -c \"$@\")"
+	else
+		$myPDSH $rnodes "(PATH=\$PATH:$RLUSTRE/utils:$RLUSTRE/tests:/sbin:/usr/sbin; cd $RPWD; LUSTRE=\"$RLUSTRE\" $(get_env_vars) sh -c \"$@\")" | sed -re "s/^[^:]*: //g"
+	fi
+	return ${PIPESTATUS[0]}
 }
 
 ##
@@ -4193,11 +4188,18 @@ do_nodes() {
 #
 # usage: do_facet $facet command [arg ...]
 do_facet() {
+	local verbose
+	local quiet
+
+	[[ "$1" == "--verbose" ]] && verbose="$1" && shift
+	[[ "$1" == "--quiet" || "$1" == "-q" ]] && quiet="$1" && shift
+
 	local facet=$1
 	shift
-	local HOST=$(facet_active_host $facet)
-	[ -z $HOST ] && echo "No host defined for facet ${facet}" && exit 1
-	do_node $HOST "$@"
+	local host=$(facet_active_host $facet)
+
+	[ -z "$host" ] && echo "No host defined for facet ${facet}" && exit 1
+	do_node $verbose $quiet $host "$@"
 }
 
 # Function: do_facet_random_file $FACET $FILE $SIZE
@@ -4220,7 +4222,7 @@ do_facet_create_file() {
 }
 
 do_nodesv() {
-    do_nodes --verbose "$@"
+	do_nodes --verbose "$@"
 }
 
 add() {
@@ -5304,6 +5306,11 @@ init_param_vars () {
 	TIMEOUT=$(do_facet $SINGLEMDS "lctl get_param -n timeout")
 	log "Using TIMEOUT=$TIMEOUT"
 
+	# tune down to speed up testing on (usually) small setups
+	local mgc_timeout=/sys/module/mgc/parameters/mgc_requeue_timeout_min
+	do_nodes $(comma_list $(nodes_list)) \
+		"[ -f $mgc_timeout ] && echo 1 > $mgc_timeout; exit 0"
+
 	osc_ensure_active $SINGLEMDS $TIMEOUT
 	osc_ensure_active client $TIMEOUT
 	$LCTL set_param osc.*.idle_timeout=debug
@@ -5423,25 +5430,103 @@ is_mounted () {
 	echo $mounted' ' | grep -w -q $mntpt' '
 }
 
-is_empty_dir() {
-	[ $(find $1 -maxdepth 1 -print | wc -l) = 1 ] && return 0
-	return 1
+create_pools () {
+	local pool=$1
+	local ostsn=${2:-$OSTCOUNT}
+	local npools=${FS_NPOOLS:-$((OSTCOUNT / ostsn))}
+	local n
+
+	echo ostsn=$ostsn npools=$npools
+	if [[ $ostsn -gt $OSTCOUNT ]];  then
+		echo "request to use $ostsn OSTs in the pool, \
+			using max available OSTCOUNT=$OSTCOUNT"
+		ostsn=$OSTCOUNT
+	fi
+	for (( n=0; n < $npools; n++ )); do
+		p=${pool}$n
+		if ! $DELETE_OLD_POOLS; then
+			log "request to not delete old pools: $FSNAME.$p exist?"
+			if ! check_pool_not_exist $FSNAME.$p; then
+				echo "Using existing $FSNAME.$p"
+				$LCTL pool_list $FSNAME.$p
+				continue
+			fi
+		fi
+		create_pool $FSNAME.$p $KEEP_POOLS ||
+			error "create_pool $FSNAME.$p failed"
+
+		local first=$(( (n * ostsn) % OSTCOUNT ))
+		local last=$(( (first + ostsn - 1) % OSTCOUNT ))
+		if [[ $first -le $last ]]; then
+			pool_add_targets $p $first $last ||
+				error "pool_add_targets $p $first $last failed"
+		else
+			pool_add_targets $p $first $(( OSTCOUNT - 1 )) ||
+				error "pool_add_targets $p $first \
+					$(( OSTCOUNT - 1 )) failed"
+			pool_add_targets $p 0 $last ||
+				error "pool_add_targets $p 0 $last failed"
+		fi
+	done
 }
 
-# empty lustre filesystem may have empty directories lost+found and .lustre
-is_empty_fs() {
-	# exclude .lustre & lost+found
-	[ $(find $1 -maxdepth 1 -name lost+found -o -name .lustre -prune -o \
-		-print | wc -l) = 1 ] || return 1
-	[ ! -d $1/lost+found ] || is_empty_dir $1/lost+found || return 1
-	if [ $(lustre_version_code $SINGLEMDS) -gt $(version_code 2.4.0) ]; then
-		# exclude .lustre/fid (LU-2780)
-		[ $(find $1/.lustre -maxdepth 1 -name fid -prune -o \
-			-print | wc -l) = 1 ] || return 1
-	else
-		[ ! -d $1/.lustre ] || is_empty_dir $1/.lustre || return 1
-	fi
-	return 0
+set_pools_quota () {
+	local u
+	local o
+	local p
+	local i
+	local j
+
+	[[ $ENABLE_QUOTA ]] || error "Required Pool Quotas: \
+		$POOLS_QUOTA_USERS_SET, but ENABLE_QUOTA not set!"
+
+	# POOLS_QUOTA_USERS_SET=
+	#              "quota15_1:20M          -- for all of the found pools
+	#               quota15_2:1G:gpool0
+	#               quota15_3              -- for global limit only
+	#               quota15_4:200M:gpool0
+	#               quota15_4:200M:gpool1"
+
+	declare -a pq_userset=(${POOLS_QUOTA_USERS_SET="mpiuser"})
+	declare -a pq_users
+	declare -A pq_limits
+
+	for ((i=0; i<${#pq_userset[@]}; i++)); do
+		u=${pq_userset[i]%%:*}
+		o=""
+		# user gets no pool limits if
+		# POOLS_QUOTA_USERS_SET does not specify it
+		[[ ${pq_userset[i]} =~ : ]] && o=${pq_userset[i]##$u:}
+		pq_limits[$u]+=" $o"
+	done
+	pq_users=(${!pq_limits[@]})
+
+	declare -a opts
+	local pool
+
+	for ((i=0; i<${#pq_users[@]}; i++)); do
+		u=${pq_users[i]}
+		# set to max limit (_u64)
+		$LFS setquota -u $u -B $((2**24 - 1))T $DIR
+		opts=(${pq_limits[$u]})
+		for ((j=0; j<${#opts[@]}; j++)); do
+			p=${opts[j]##*:}
+			o=${opts[j]%%:*}
+			# Set limit for all existing pools if
+			# no pool specified
+			if [ $p == $o ];  then
+				p=$(list_pool $FSNAME | sed "s/$FSNAME.//")
+				echo "No pool specified for $u,
+					set limit $o for all existing pools"
+			fi
+			for pool in $p; do
+				$LFS setquota -u $u -B $o --pool $pool $DIR ||
+                                        error "setquota -u $u -B $o \
+						--pool $pool failed"
+			done
+		done
+		$LFS quota -uv $u --pool  $DIR
+	done
 }
 
 check_and_setup_lustre() {
@@ -5529,6 +5614,16 @@ check_and_setup_lustre() {
 		set_flavor_all $SEC
 	fi
 
+	if $DELETE_OLD_POOLS; then
+		destroy_all_pools
+	fi
+	if [[ -n "$FS_POOL" ]]; then
+		create_pools $FS_POOL $FS_POOL_NOSTS
+	fi
+
+	if [[ -n "$POOLS_QUOTA_USERS_SET" ]]; then
+		set_pools_quota
+	fi
 	if [ "$ONLY" == "setup" ]; then
 		exit 0
 	fi
@@ -5559,51 +5654,6 @@ cleanup_and_setup_lustre() {
     check_and_setup_lustre
 }
 
-# Get all of the server target devices from a given server node and type.
-get_mnt_devs() {
-	local node=$1
-	local type=$2
-	local devs
-	local dev
-
-	if [ "$type" == ost ]; then
-		devs=$(get_osd_param $node "" mntdev)
-	else
-		devs=$(do_node $node $LCTL get_param -n osd-*.$FSNAME-M*.mntdev)
-	fi
-	for dev in $devs; do
-		case $dev in
-		*loop*) do_node $node "losetup $dev" | \
-				sed -e "s/.*(//" -e "s/).*//" ;;
-		*) echo $dev ;;
-		esac
-	done
-}
-
-# Get all of the server target devices.
-get_svr_devs() {
-	local node
-	local i
-
-	# Master MDS parameters used by lfsck
-	MDTNODE=$(facet_active_host $SINGLEMDS)
-	MDTDEV=$(echo $(get_mnt_devs $MDTNODE mdt) | awk '{print $1}')
-
-	# MDT devices
-	i=0
-	for node in $(mdts_nodes); do
-		MDTDEVS[i]=$(get_mnt_devs $node mdt)
-		i=$((i + 1))
-	done
-
-	# OST devices
-	i=0
-	for node in $(osts_nodes); do
-		OSTDEVS[i]=$(get_mnt_devs $node ost)
-		i=$((i + 1))
-	done
-}
-
 # Run e2fsck on MDT or OST device.
 run_e2fsck() {
 	local node=$1
@@ -5929,10 +5979,6 @@ at_max_get() {
     at_get $1 at_max
 }
 
-at_min_get() {
-	at_get $1 at_min
-}
-
 at_max_set() {
     local at_max=$1
     shift
@@ -6522,10 +6568,10 @@ check_mds() {
 }
 
 reset_fail_loc () {
-	echo -n "Resetting fail_loc on all nodes..."
-	do_nodes $(comma_list $(nodes_list)) "lctl set_param -n fail_loc=0 \
-	    fail_val=0 2>/dev/null" || true
-	echo done.
+	#echo -n "Resetting fail_loc on all nodes..."
+	do_nodes --quiet $(comma_list $(nodes_list)) \
+		"lctl set_param -n fail_loc=0 fail_val=0 2>/dev/null" || true
+	#echo done.
 }
 
 
@@ -6534,7 +6580,8 @@ reset_fail_loc () {
 # Also appends a timestamp and prepends the testsuite name.
 #
 
-EQUALS="===================================================================================================="
+# ======================================================== 15:06:12 (1624050372)
+EQUALS="========================================================"
 banner() {
     msg="== ${TESTSUITE} $*"
     last=${msg: -1:1}
@@ -6550,7 +6597,7 @@ check_dmesg_for_errors() {
 ldiskfs_check_descriptors: Checksum for group 0 failed\|\
 group descriptors corrupted"
 
-	res=$(do_nodes $(comma_list $(nodes_list)) "dmesg" | grep "$errors")
+	res=$(do_nodes -q $(comma_list $(nodes_list)) "dmesg" | grep "$errors")
 	[ -z "$res" ] && return 0
 	echo "Kernel error detected: $res"
 	return 1
@@ -6673,10 +6720,6 @@ skip_logged(){
 	log_sub_test_end "SKIP" "0" "0" "$@"
 }
 
-canonical_path() {
-	(cd $(dirname $1); echo $PWD/$(basename $1))
-}
-
 grant_from_clients() {
 	local nodes="$1"
 
@@ -6796,7 +6839,7 @@ host_id() {
 # Description:
 #   Returns list of ip addresses for each interface
 local_addr_list() {
-	ip addr | awk '/inet\ / {print $2}' | awk -F\/ '{print $1}'
+	ip addr | awk '/inet / {print $2}' | awk -F/ '{print $1}'
 }
 
 is_local_addr() {
@@ -6920,11 +6963,6 @@ osts_nodes () {
 	echo -n $(facets_nodes $(get_facets OST))
 }
 
-# Get all of the active AGT (HSM agent) nodes.
-agts_nodes () {
-	echo -n $(facets_nodes $(get_facets AGT))
-}
-
 # Get all of the client nodes and active server nodes.
 nodes_list () {
 	local nodes=$HOSTNAME
@@ -7064,12 +7102,6 @@ get_node_count() {
     echo $nodes | wc -w || true
 }
 
-mixed_ost_devs () {
-    local nodes=$(osts_nodes)
-    local osscount=$(get_node_count "$nodes")
-    [ ! "$OSTCOUNT" = "$osscount" ]
-}
-
 mixed_mdt_devs () {
     local nodes=$(mdts_nodes)
     local mdtcount=$(get_node_count "$nodes")
@@ -7094,23 +7126,6 @@ get_stripe () {
 	rm -f $file
 }
 
-setstripe_nfsserver () {
-	local dir=$1
-	local nfsexportdir=$2
-	shift
-	shift
-
-	local -a nfsexport=($(awk '"'$dir'" ~ $2 && $3 ~ "nfs" && $2 != "/" \
-		{ print $1 }' /proc/mounts | cut -f 1 -d :))
-
-	# check that only one nfs mounted
-	[[ -z $nfsexport ]] && echo "$dir is not nfs mounted" && return 1
-	(( ${#nfsexport[@]} == 1 )) ||
-		error "several nfs mounts found for $dir: ${nfsexport[@]} !"
-
-	do_nodev ${nfsexport[0]} lfs setstripe $nfsexportdir "$@"
-}
-
 # Check and add a test group.
 add_group() {
 	local group_id=$1
@@ -7348,15 +7363,18 @@ restore_lustre_params() {
 
 check_node_health() {
 	local nodes=${1:-$(comma_list $(nodes_list))}
-
-	for node in ${nodes//,/ }; do
-		check_network "$node" 5
-		if [ $? -eq 0 ]; then
-			do_node $node "$LCTL get_param catastrophe 2>&1" |
-				grep -q "catastrophe=1" &&
-				error "$node:LBUG/LASSERT detected" || true
-		fi
-	done
+	local health=$TMP/node_health.$$
+
+	do_nodes -q $nodes "$LCTL get_param catastrophe 2>&1" | tee $health |
+		grep "catastrophe=1" && error "LBUG/LASSERT detected"
+	# Only check/report network health if get_param isn't reported, since
+	# *clearly* the network is working if get_param returned something.
+	if (( $(grep -c catastro $health) != $(wc -w <<< ${nodes//,/ }) )); then
+		for node in ${nodes//,/}; do
+			check_network $node 5
+		done
+	fi
+	rm -f $health
 }
 
 mdsrate_cleanup () {
@@ -7367,11 +7385,6 @@ mdsrate_cleanup () {
 	fi
 }
 
-delayed_recovery_enabled () {
-    local var=${SINGLEMDS}_svc
-    do_facet $SINGLEMDS lctl get_param -n mdd.${!var}.stale_export_age > /dev/null 2>&1
-}
-
 ########################
 
 convert_facet2label() {
@@ -7396,20 +7409,6 @@ get_clientosc_proc_path() {
 	echo "${1}-osc-[-0-9a-f]*"
 }
 
-# If the 2.0 MDS was mounted on 1.8 device, then the OSC and LOV names
-# used by MDT would not be changed.
-# mdt lov: fsname-mdtlov
-# mdt osc: fsname-OSTXXXX-osc
-mds_on_old_device() {
-    local mds=${1:-"$SINGLEMDS"}
-
-    if [ $(lustre_version_code $mds) -gt $(version_code 1.9.0) ]; then
-        do_facet $mds "lctl list_param osc.$FSNAME-OST*-osc \
-            > /dev/null 2>&1" && return 0
-    fi
-    return 1
-}
-
 get_mdtosc_proc_path() {
 	local mds_facet=$1
 	local ost_label=${2:-"*OST*"}
@@ -7569,8 +7568,9 @@ _wait_osc_import_state() {
 		params=$param
 	fi
 
+	local plist=$(comma_list $params)
 	if ! do_rpc_nodes "$(facet_active_host $facet)" \
-			wait_import_state $expected "$params" $maxtime; then
+			wait_import_state $expected $plist $maxtime; then
 		error "$facet: import is not in $expected state after $maxtime"
 		return 1
 	fi
@@ -7625,8 +7625,9 @@ _wait_mgc_import_state() {
 			params=$($LCTL list_param $param 2>/dev/null || true)
 		done
 	fi
+	local plist=$(comma_list $params)
 	if ! do_rpc_nodes "$(facet_active_host $facet)" \
-			wait_import_state $expected "$params" $maxtime \
+			wait_import_state $expected $plist $maxtime \
 					  $error_on_failure; then
 		if [ $error_on_failure -ne 0 ]; then
 		    error "import is not in ${expected} state"
@@ -7803,6 +7804,7 @@ check_pool_not_exist() {
 create_pool() {
 	local fsname=${1%%.*}
 	local poolname=${1##$fsname.}
+	local keep_pools=${2:-false}
 
 	stack_trap "destroy_test_pools $fsname" EXIT
 	do_facet mgs lctl pool_new $1
@@ -7821,7 +7823,7 @@ create_pool() {
 	wait_update $HOSTNAME "lctl get_param -n lov.$fsname-*.pools.$poolname \
 		2>/dev/null || echo foo" "" || error "pool_new failed $1"
 
-	add_pool_to_list $1
+	$keep_pools || add_pool_to_list $1
 	return $RC
 }
 
@@ -7839,10 +7841,18 @@ remove_pool_from_list () {
 	local poolname=${1##$fsname.}
 
 	local listvar=${fsname}_CREATED_POOLS
-	local temp=${listvar}=$(exclude_items_from_list ${!listvar} $poolname)
+	local temp=${listvar}=$(exclude_items_from_list "${!listvar}" $poolname)
 	eval export $temp
 }
 
+# cleanup all pools exist on $FSNAME
+destroy_all_pools () {
+	local i
+	for i in $(list_pool $FSNAME); do
+		destroy_pool $i
+	done
+}
+
 destroy_pool_int() {
 	local ost
 	local OSTS=$(list_pool $1)
@@ -7863,8 +7873,7 @@ destroy_pool() {
 
 	local RC
 
-	check_pool_not_exist $fsname.$poolname
-	[[ $? -eq 0 ]] && return 0
+	check_pool_not_exist $fsname.$poolname && return 0 || true
 
 	destroy_pool_int $fsname.$poolname
 	RC=$?
@@ -8532,94 +8541,6 @@ run_llverfs()
         llverfs $partial_arg $llverfs_opts $dir
 }
 
-#Remove objects from OST
-remove_ost_objects() {
-	local facet=$1
-	local ostdev=$2
-	local group=$3
-	shift 3
-	local objids="$@"
-	local mntpt=$(facet_mntpt $facet)
-	local opts=$OST_MOUNT_OPTS
-	local i
-	local rc
-
-	echo "removing objects from $ostdev on $facet: $objids"
-	if ! test -b $ostdev; then
-		opts=$(csa_add "$opts" -o loop)
-	fi
-	mount -t $(facet_fstype $facet) $opts $ostdev $mntpt ||
-		return $?
-	rc=0
-	for i in $objids; do
-		rm $mntpt/O/$group/d$((i % 32))/$i || { rc=$?; break; }
-	done
-	umount -f $mntpt || return $?
-	return $rc
-}
-
-#Remove files from MDT
-remove_mdt_files() {
-	local facet=$1
-	local mdtdev=$2
-	shift 2
-	local files="$@"
-	local mntpt=$(facet_mntpt $facet)
-	local opts=$MDS_MOUNT_OPTS
-
-	echo "removing files from $mdtdev on $facet: $files"
-	if [ $(facet_fstype $facet) == ldiskfs ] &&
-	   ! do_facet $facet test -b $mdtdev; then
-		opts=$(csa_add "$opts" -o loop)
-	fi
-	mount -t $(facet_fstype $facet) $opts $mdtdev $mntpt ||
-		return $?
-	rc=0
-	for f in $files; do
-		rm $mntpt/ROOT/$f || { rc=$?; break; }
-	done
-	umount -f $mntpt || return $?
-	return $rc
-}
-
-duplicate_mdt_files() {
-	local facet=$1
-	local mdtdev=$2
-	shift 2
-	local files="$@"
-	local mntpt=$(facet_mntpt $facet)
-	local opts=$MDS_MOUNT_OPTS
-
-	echo "duplicating files on $mdtdev on $facet: $files"
-	mkdir -p $mntpt || return $?
-	if [ $(facet_fstype $facet) == ldiskfs ] &&
-	   ! do_facet $facet test -b $mdtdev; then
-		opts=$(csa_add "$opts" -o loop)
-	fi
-	mount -t $(facet_fstype $facet) $opts $mdtdev $mntpt ||
-		return $?
-
-    do_umount() {
-        trap 0
-        popd > /dev/null
-        rm $tmp
-        umount -f $mntpt
-    }
-    trap do_umount EXIT
-
-    tmp=$(mktemp $TMP/setfattr.XXXXXXXXXX)
-    pushd $mntpt/ROOT > /dev/null || return $?
-    rc=0
-    for f in $files; do
-        touch $f.bad || return $?
-        getfattr -n trusted.lov $f | sed "s#$f#&.bad#" > $tmp
-        rc=${PIPESTATUS[0]}
-        [ $rc -eq 0 ] || return $rc
-        setfattr --restore $tmp || return $?
-    done
-    do_umount
-}
-
 run_sgpdd () {
     local devs=${1//,/ }
     shift
@@ -8707,17 +8628,6 @@ get_block_count() {
 	echo -n ${count:-0}
 }
 
-# Get the block size of the filesystem.
-get_block_size() {
-	local facet=$1
-	local device=$2
-	local size
-
-	[ -z "$CLIENTONLY" ] && size=$(do_facet $facet "$DUMPE2FS -h $device 2>&1" |
-		awk '/^Block size:/ {print $3}')
-	echo -n ${size:-0}
-}
-
 # Check whether the "ea_inode" feature is enabled or not, to allow
 # ldiskfs xattrs over one block in size.  Allow both the historical
 # Lustre feature name (large_xattr) and the upstream name (ea_inode).
@@ -8953,7 +8863,7 @@ check_mount_and_prep()
 	is_mounted $MOUNT || setupall
 
 	rm -rf $DIR/[df][0-9]* || error "Fail to cleanup the env!"
-	mkdir $DIR/$tdir || error "Fail to mkdir $DIR/$tdir."
+	mkdir_on_mdt0 $DIR/$tdir || error "Fail to mkdir $DIR/$tdir."
 	for idx in $(seq $MDSCOUNT); do
 		local name="MDT$(printf '%04x' $((idx - 1)))"
 		rm -rf $MOUNT/.lustre/lost+found/$name/*
@@ -9570,14 +9480,17 @@ check_clients_evicted() {
 	local rc=0
 
 	for osc in $oscs; do
-		((rc++))
 		echo "Check state for $osc"
 		local evicted=$(do_facet client $LCTL get_param osc.$osc.state |
-			tail -n 3 | awk -F"[ [,]" \
-			'/EVICTED ]$/ { if (mx<$5) {mx=$5;} } END { print mx }')
+			tail -n 5 | awk -F"[ ,]" \
+			'/EVICTED/ { if (mx<$4) { mx=$4; } } END { print mx }')
 		if (($? == 0)) && (($evicted > $before)); then
 			echo "$osc is evicted at $evicted"
-			((rc--))
+		else
+			((rc++))
+			echo "$osc was not evicted after $before:"
+			do_facet client $LCTL get_param osc.$osc.state |
+				tail -n 8
 		fi
 	done
 
@@ -9634,8 +9547,8 @@ changelog_register() {
 			error "$mdt: changelog_mask=+hsm failed: $?"
 
 		local cl_user
-		cl_user=$(do_facet $facet \
-				  $LCTL --device $mdt changelog_register -n) ||
+		cl_user=$(do_facet $facet $LCTL --device $mdt \
+			changelog_register -n $@) ||
 			error "$mdt: register changelog user failed: $?"
 		stack_trap "__changelog_deregister $facet $cl_user" EXIT
 
@@ -9722,6 +9635,7 @@ changelog_clear() {
 	# so reorder to get same order than in changelog_register()
 	local cl_facets=$(echo "${!CL_USERS[@]}" | tr " " "\n" | sort |
 			  tr "\n" " ")
+	local cl_user
 
 	for facet in $cl_facets; do
 		for cl_user in ${CL_USERS[$facet]}; do
@@ -10104,6 +10018,7 @@ init_agt_vars() {
 	export HSMTOOL_UPDATE_INTERVAL=${HSMTOOL_UPDATE_INTERVAL:=""}
 	export HSMTOOL_EVENT_FIFO=${HSMTOOL_EVENT_FIFO:=""}
 	export HSMTOOL_TESTDIR
+	export HSMTOOL_ARCHIVE_FORMAT=${HSMTOOL_ARCHIVE_FORMAT:-v2}
 
 	if ! [[ $HSMTOOL =~ hsmtool ]]; then
 		echo "HSMTOOL = '$HSMTOOL' does not contain 'hsmtool', GLWT" >&2
@@ -10195,27 +10110,27 @@ copytool_logfile()
 
 __lhsmtool_rebind()
 {
-	do_facet $facet $HSMTOOL -p "$hsm_root" --rebind "$@" "$mountpoint"
+	do_facet $facet $HSMTOOL "${hsmtool_options[@]}" --rebind "$@" "$mountpoint"
 }
 
 __lhsmtool_import()
 {
 	mkdir -p "$(dirname "$2")" ||
 		error "cannot create directory '$(dirname "$2")'"
-	do_facet $facet $HSMTOOL -p "$hsm_root" --import "$@" "$mountpoint"
+	do_facet $facet $HSMTOOL "${hsmtool_options[@]}" --import "$@" "$mountpoint"
 }
 
 __lhsmtool_setup()
 {
 	local host="$(facet_host "$facet")"
-	local cmd="$HSMTOOL $HSMTOOL_VERBOSE --daemon --pid-file=$HSMTOOL_PID_FILE --hsm-root \"$hsm_root\""
+	local cmd="$HSMTOOL ${hsmtool_options[@]} --daemon --pid-file=$HSMTOOL_PID_FILE"
 	[ -n "$bandwidth" ] && cmd+=" --bandwidth $bandwidth"
 	[ -n "$archive_id" ] && cmd+=" --archive $archive_id"
-	[ ${#misc_options[@]} -gt 0 ] &&
-		cmd+=" $(IFS=" " echo "$@")"
-	cmd+=" \"$mountpoint\""
+#	[ ${#misc_options[@]} -gt 0 ] &&
+#		cmd+=" $(IFS=" " echo "$@")"
+	cmd+=" $@ \"$mountpoint\""
 
-	echo "Starting copytool '$facet' on '$host'"
+	echo "Starting copytool '$facet' on '$host' with cmdline '$cmd'"
 	stack_trap "pkill_copytools $host TERM || true" EXIT
 	do_node "$host" "$cmd < /dev/null > \"$(copytool_logfile $facet)\" 2>&1"
 }
@@ -10250,7 +10165,17 @@ copytool()
 
 	# Parse arguments
 	local fail_on_error=true
-	local -a misc_options
+	local -a hsmtool_options=("--hsm-root=$hsm_root")
+	local -a action_options=()
+
+	if [[ -n "$HSMTOOL_ARCHIVE_FORMAT" ]]; then
+		hsmtool_options+=("--archive-format=$HSMTOOL_ARCHIVE_FORMAT")
+	fi
+
+	if [[ -n "$HSMTOOL_VERBOSE" ]]; then
+		hsmtool_options+=("$HSMTOOL_VERBOSE")
+	fi
+
 	while [ $# -gt 0 ]; do
 		case "$1" in
 		-f|--facet)
@@ -10278,7 +10203,7 @@ copytool()
 			;;
 		*)
 			# Uncommon(/copytool dependent) option
-			misc_options+=("$1")
+			action_options+=("$1")
 			;;
 		esac
 		shift
@@ -10294,7 +10219,7 @@ copytool()
 		;;
 	esac
 
-	__${copytool}_${action} "${misc_options[@]}"
+	__${copytool}_${action} "${action_options[@]}"
 	if [ $? -ne 0 ]; then
 		local error_msg
 
@@ -10304,8 +10229,8 @@ copytool()
 			error_msg="Failed to start copytool $facet on '$host'"
 			;;
 		import)
-			local src="${misc_options[0]}"
-			local dest="${misc_options[1]}"
+			local src="${action_options[0]}"
+			local dest="${action_options[1]}"
 			error_msg="Failed to import '$src' to '$dest'"
 			;;
 		rebind)
@@ -10552,12 +10477,12 @@ function createmany() {
 		local saved_debug=$($LCTL get_param -n debug)
 		local list=$(comma_list $(all_nodes))
 
-		do_nodes $list $LCTL set_param debug=0
+		do_nodes $list $LCTL set_param -n debug=0
 	}
 	$LUSTRE/tests/createmany $*
 	local rc=$?
 	(( count > 100 )) &&
-		do_nodes $list "$LCTL set_param debug=\\\"$saved_debug\\\""
+		do_nodes $list "$LCTL set_param -n debug=\\\"$saved_debug\\\""
 	return $rc
 }
 
@@ -10568,12 +10493,12 @@ function unlinkmany() {
 		local saved_debug=$($LCTL get_param -n debug)
 		local list=$(comma_list $(all_nodes))
 
-		do_nodes $list $LCTL set_param debug=0
+		do_nodes $list $LCTL set_param -n debug=0
 	}
 	$LUSTRE/tests/unlinkmany $*
 	local rc=$?
 	(( count > 100 )) &&
-		do_nodes $list "$LCTL set_param debug=\\\"$saved_debug\\\""
+		do_nodes $list "$LCTL set_param -n debug=\\\"$saved_debug\\\""
 	return $rc
 }
 
@@ -10605,3 +10530,134 @@ function check_set_fallocate_or_skip()
 	check_set_fallocate || skip "need at least 2.13.57 for fallocate"
 }
 
+function disable_opencache()
+{
+	local state=$($LCTL get_param -n "llite.*.opencache_threshold_count" | head -1)
+
+	test -z "${saved_OPENCACHE_value}" &&
+					export saved_OPENCACHE_value="$state"
+
+	[[ "$state" = "off" ]] && return
+
+	$LCTL set_param -n "llite.*.opencache_threshold_count"=off
+}
+
+function set_opencache()
+{
+	local newvalue="$1"
+	local state=$($LCTL get_param -n "llite.*.opencache_threshold_count")
+
+	[[ -n "$newvalue" ]] || return
+
+	[[ -n "${saved_OPENCACHE_value}" ]] ||
+					export saved_OPENCACHE_value="$state"
+
+	$LCTL set_param -n "llite.*.opencache_threshold_count"=$newvalue
+}
+
+
+
+function restore_opencache()
+{
+	[[ -z "${saved_OPENCACHE_value}" ]] ||
+		$LCTL set_param -n "llite.*.opencache_threshold_count"=${saved_OPENCACHE_value}
+}
+
+# LU-13417: XXX lots of tests assume the directory to be created under MDT0,
+# created on MDT0, use this function to create directory on specific MDT
+# explicitly, and set default LMV to create subdirs on the same MDT too.
+mkdir_on_mdt() {
+	local mdt
+	local OPTIND=1
+
+	while getopts "i:" opt $*; do
+		case $opt in
+			i) mdt=$OPTARG;;
+		esac
+	done
+
+	shift $((OPTIND - 1))
+
+	$LFS mkdir -i $mdt -c 1 $*
+	# setting default LMV in non-DNE system will cause sanity-quota 41 fail
+	((MDSCOUNT < 2)) || $LFS setdirstripe -D -i $mdt -c 1 $*
+}
+
+mkdir_on_mdt0() {
+	mkdir_on_mdt -i0 $*
+}
+
+# Wait for nodemap synchronization
+wait_nm_sync() {
+	local nodemap_name=$1
+	local key=$2
+	local value=$3
+	local opt=$4
+	local proc_param
+	local is_active=$(do_facet mgs $LCTL get_param -n nodemap.active)
+	local max_retries=20
+	local is_sync
+	local out1=""
+	local out2
+	local mgs_ip=$(host_nids_address $mgs_HOST $NETTYPE | cut -d' ' -f1)
+	local i
+
+	if [ "$nodemap_name" == "active" ]; then
+		proc_param="active"
+	elif [ -z "$key" ]; then
+		proc_param=${nodemap_name}
+	else
+		proc_param="${nodemap_name}.${key}"
+	fi
+	if [ "$opt" == "inactive" ]; then
+		# check nm sync even if nodemap is not activated
+		is_active=1
+		opt=""
+	fi
+	(( is_active == 0 )) && [ "$proc_param" != "active" ] && return
+
+	if [ -z "$value" ]; then
+		out1=$(do_facet mgs $LCTL get_param $opt \
+			nodemap.${proc_param} 2>/dev/null)
+		echo "On MGS ${mgs_ip}, ${proc_param} = $out1"
+	else
+		out1=$value;
+	fi
+
+	# if servers run on the same node, it is impossible to tell if they get
+	# synced with the mgs, so just wait an arbitrary 10 seconds
+	if [ $(facet_active_host mgs) == $(facet_active_host mds) ] &&
+	   [ $(facet_active_host mgs) == $(facet_active_host ost1) ]; then
+		echo "waiting 10 secs for sync"
+		sleep 10
+		return
+	fi
+
+	# wait up to 10 seconds for other servers to sync with mgs
+	for i in $(seq 1 10); do
+		for node in $(all_server_nodes); do
+			local node_ip=$(host_nids_address $node $NETTYPE |
+					cut -d' ' -f1)
+
+			is_sync=true
+			if [ -z "$value" ]; then
+				[ $node_ip == $mgs_ip ] && continue
+			fi
+
+			out2=$(do_node $node_ip $LCTL get_param $opt \
+			       nodemap.$proc_param 2>/dev/null)
+			echo "On $node ${node_ip}, ${proc_param} = $out2"
+			[ "$out1" != "$out2" ] && is_sync=false && break
+		done
+		$is_sync && break
+		sleep 1
+	done
+	if ! $is_sync; then
+		echo MGS
+		echo $out1
+		echo OTHER - IP: $node_ip
+		echo $out2
+		error "mgs and $nodemap_name ${key} mismatch, $i attempts"
+	fi
+	echo "waited $((i - 1)) seconds for sync"
+}