export GSS_KRB5=false
export GSS_PIPEFS=false
export IDENTITY_UPCALL=default
+export QUOTA_AUTO=1
#export PDSH="pdsh -S -Rssh -w"
LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)}
. $LUSTRE/tests/functions.sh
+LUSTRE_TESTS_CFG_DIR=${LUSTRE_TESTS_CFG_DIR:-${LUSTRE}/tests/cfg}
+
+EXCEPT_LIST_FILE=${EXCEPT_LIST_FILE:-${LUSTRE_TESTS_CFG_DIR}/tests-to-skip.sh}
+
+if [ -f "$EXCEPT_LIST_FILE" ]; then
+ echo "Reading test skip list from $EXCEPT_LIST_FILE"
+ cat $EXCEPT_LIST_FILE
+ . $EXCEPT_LIST_FILE
+fi
+
assert_DIR () {
local failed=""
[[ $DIR/ = $MOUNT/* ]] || \
IDENTITY_UPCALL=false
;;
esac
+ export LOAD_MODULES_REMOTE=${LOAD_MODULES_REMOTE:-false}
# Paths on remote nodes, if different
export RLUSTRE=${RLUSTRE:-$LUSTRE}
export RPWD=${RPWD:-$PWD}
export I_MOUNTED=${I_MOUNTED:-"no"}
+ if [ ! -f /lib/modules/$(uname -r)/kernel/fs/lustre/mds.ko -a \
+ ! -f `dirname $0`/../mds/mds.ko ]; then
+ export CLIENTMODSONLY=yes
+ fi
# command line
*) EXT=".ko"; USE_QUOTA=yes;;
esac
+
+module_loaded () {
+ /sbin/lsmod | grep -q $1
+}
+
load_module() {
EXT=".ko"
module=$1
shift
BASE=`basename $module $EXT`
- lsmod | grep -q ${BASE} || \
- if [ -f ${LUSTRE}/${module}${EXT} ]; then
+
+ module_loaded ${BASE} && return
+
+ if [ -f ${LUSTRE}/${module}${EXT} ]; then
insmod ${LUSTRE}/${module}${EXT} $@
else
# must be testing a "make install" or "rpm" installation
fi
}
-load_modules() {
+load_modules_local() {
if [ -n "$MODPROBE" ]; then
# use modprobe
return 0
load_module osc/osc
load_module lov/lov
load_module mgc/mgc
- if [ -z "$CLIENTONLY" ] && [ -z "$CLIENTMODSONLY" ]; then
+ if ! client_only; then
grep -q crc16 /proc/kallsyms || { modprobe crc16 2>/dev/null || true; }
grep -q jbd /proc/kallsyms || { modprobe jbd 2>/dev/null || true; }
[ "$FSTYPE" = "ldiskfs" ] && load_module ../ldiskfs/ldiskfs/ldiskfs
[ -f $LUSTRE/utils/mount.lustre ] && cp $LUSTRE/utils/mount.lustre /sbin/. || true
}
+load_modules () {
+ load_modules_local
+ # bug 19124
+ # load modules on remote nodes optionally
+ # lustre-tests have to be installed on these nodes
+ if $LOAD_MODULES_REMOTE ; then
+ local list=$(comma_list $(remote_nodes_list))
+ echo loading modules on $list
+ do_rpc_nodes $list load_modules
+ fi
+}
+
check_mem_leak () {
LEAK_LUSTRE=$(dmesg | tail -n 30 | grep "obd_memory.*leaked" || true)
LEAK_PORTALS=$(dmesg | tail -n 20 | grep "Portals memory leaked" || true)
$LUSTRE_RMMOD $FSTYPE || return 2
+ if $LOAD_MODULES_REMOTE ; then
+ local list=$(comma_list $(remote_nodes_list))
+ echo unloading modules on $list
+ do_rpc_nodes $list $LUSTRE_RMMOD $FSTYPE
+ do_rpc_nodes $list check_mem_leak
+ fi
+
HAVE_MODULES=false
check_mem_leak || return 254
setup_quota(){
local mntpt=$1
- # We need:
- # 1. run quotacheck only if quota is off
- # 2. save the original quota_type params, restore them after testing
+ # We need save the original quota_type params, and restore them after testing
# Suppose that quota type the same on mds and ost
local quota_type=$(quota_type | grep MDT | cut -d "=" -f2)
if [ "$quota_type" != "$QUOTA_TYPE" ]; then
export old_QUOTA_TYPE=$quota_type
quota_save_version $QUOTA_TYPE
+ else
+ qtype=$(tr -c -d "ug" <<< $QUOTA_TYPE)
+ $LFS quotacheck -$qtype $mntpt || error "quotacheck has failed for $type"
fi
local quota_usrs=$QUOTA_USERS
# mount clients if not mouted
zconf_mount_clients() {
- local OPTIONS
local clients=$1
local mnt=$2
-
+ local OPTIONS=${3:-$MOUNTOPT}
# Only supply -o to mount if we have options
- if [ -n "$MOUNTOPT" ]; then
- OPTIONS="-o $MOUNTOPT"
+ if [ "$OPTIONS" ]; then
+ OPTIONS="-o $OPTIONS"
fi
local device=$MGSNID:/$FSNAME
if [ -z "$mnt" -o -z "$FSNAME" ]; then
echo "Stopping clients: $clients $mnt (opts:$force)"
do_nodes $clients "running=\\\$(grep -c $mnt' ' /proc/mounts);
if [ \\\$running -ne 0 ] ; then
-echo Stopping client \\\$(hostname) client $mnt opts:$force;
+echo Stopping client \\\$(hostname) $mnt opts:$force;
lsof -t $mnt || need_kill=no;
if [ "x$force" != "x" -a "x\\\$need_kill" != "xno" ]; then
pids=\\\$(lsof -t $mnt | sort -u);
[ "`lctl dl 2> /dev/null | wc -l`" -gt 0 ] && lctl dl && \
echo "$0: lustre didn't clean up..." 1>&2 && return 202 || true
- if [ "`/sbin/lsmod 2>&1 | egrep 'lnet|libcfs'`" ]; then
+ if module_loaded lnet || module_loaded libcfs; then
echo "$0: modules still loaded..." 1>&2
/sbin/lsmod 1>&2
return 203
sleep 1
TOTAL=`lctl get_param -n osc.*.kbytesavail | \
awk 'BEGIN{total=0}; {total+=$1}; END{print total}'`
- [ "$TOTAL" -eq "$TOTALPREV" ] && break
+ [ "$TOTAL" -eq "$TOTALPREV" ] && return 0
echo "Waiting delete completed ... prev: $TOTALPREV current: $TOTAL "
TOTALPREV=$TOTAL
WAIT=$(( WAIT + 1))
done
- echo "Delete completed."
+ echo "Delete is not completed in $MAX_WAIT sec"
+ return 1
}
wait_for_host() {
wait_recovery_complete () {
local facet=$1
- # Use default policy if $2 is not passed by caller.
+ # Use default policy if $2 is not passed by caller.
#define OBD_RECOVERY_TIMEOUT (obd_timeout * 5 / 2)
# as we are in process of changing obd_timeout in different ways
# let's set MAX longer than that
local MAX=${2:-$(( TIMEOUT * 4 ))}
-
+
local var_svc=${facet}_svc
local procfile="*.${!var_svc}.recovery_status"
local WAIT=0
return 1
}
+wait_mds_ost_sync () {
+ # just because recovery is done doesn't mean we've finished
+ # orphan cleanup. Wait for llogs to get synchronized.
+ echo "Waiting for orphan cleanup..."
+ # MAX value includes time needed for MDS-OST reconnection
+ local MAX=$(( TIMEOUT * 2 ))
+ local WAIT=0
+ while [ $WAIT -lt $MAX ]; do
+ local -a sync=($(do_nodes $(comma_list $(osts_nodes)) \
+ "$LCTL get_param -n obdfilter.*.mds_sync"))
+ local con=1
+ for ((i=0; i<${#sync[@]}; i++)); do
+ [ ${sync[$i]} -eq 0 ] && continue
+ # there is a not finished MDS-OST synchronization
+ con=0
+ break;
+ done
+ sleep 2 # increase waiting time and cover statfs cache
+ [ ${con} -eq 1 ] && return 0
+ echo "Waiting $WAIT secs for $facet mds-ost sync done."
+ WAIT=$((WAIT + 2))
+ done
+ echo "$facet recovery not done in $MAX sec. $STATUS"
+ return 1
+}
+
+wait_destroy_complete () {
+ echo "Waiting for destroy to be done..."
+ # MAX value shouldn't be big as this mean server responsiveness
+ # never increase this just to make test pass but investigate
+ # why it takes so long time
+ local MAX=5
+ local WAIT=0
+ while [ $WAIT -lt $MAX ]; do
+ local -a RPCs=($($LCTL get_param -n osc.*.destroys_in_flight))
+ local con=1
+ for ((i=0; i<${#RPCs[@]}; i++)); do
+ [ ${RPCs[$i]} -eq 0 ] && continue
+ # there are still some destroy RPCs in flight
+ con=0
+ break;
+ done
+ sleep 1
+ [ ${con} -eq 1 ] && return 0 # done waiting
+ echo "Waiting $WAIT secs for destroys to be done."
+ WAIT=$((WAIT + 1))
+ done
+ echo "Destroys weren't done in $MAX sec."
+ return 1
+}
+
wait_exit_ST () {
local facet=$1
return $rc
}
-client_df() {
+clients_up() {
# not every config has many clients
+ sleep 1
if [ ! -z "$CLIENTS" ]; then
- $PDSH $CLIENTS "df $MOUNT" > /dev/null
+ $PDSH $CLIENTS "stat -f $MOUNT" > /dev/null
+ else
+ stat -f $MOUNT > /dev/null
+ fi
+}
+
+client_up() {
+ local client=$1
+ # usually checked on particular client or locally
+ sleep 1
+ if [ ! -z "$client" ]; then
+ $PDSH $client "stat -f $MOUNT" > /dev/null
else
- df $MOUNT > /dev/null
+ stat -f $MOUNT > /dev/null
fi
}
+client_evicted() {
+ ! client_up $1
+}
+
client_reconnect() {
uname -n >> $MOUNT/recon
if [ -z "$CLIENTS" ]; then
fail() {
facet_failover $* || error "failover: $?"
- client_df || error "post-failover df: $?"
+ clients_up || error "post-failover df: $?"
}
fail_nodf() {
stop $facet
change_active $facet
mount_facet $facet -o abort_recovery
- client_df || echo "first df failed: $?"
- sleep 1
- client_df || error "post-failover df: $?"
+ clients_up || echo "first df failed: $?"
+ clients_up || error "post-failover df: $?"
}
do_lmc() {
if [ x"$(som_check)" = x"enabled" ]; then
ENABLE_QUOTA=""
+ echo "disable quota temporary when SOM enabled"
fi
- if [ "$ENABLE_QUOTA" ]; then
- setup_quota $MOUNT || return 2
+ if [ $QUOTA_AUTO -ne 0 ]; then
+ if [ "$ENABLE_QUOTA" ]; then
+ echo "enable quota as required"
+ setup_quota $MOUNT || return 2
+ else
+ echo "disable quota as required"
+ $LFS quotaoff -ug $MOUNT > /dev/null 2>&1
+ fi
fi
+
+ return 0
}
nfs_client_mode () {
return 1
}
-check_config () {
- nfs_client_mode && return
-
+check_config_client () {
local mntpt=$1
local mounted=$(mount | grep " $mntpt ")
if [[ x$mgc != xMGC$MGSNID ]]; then
if [ "$mgs_HOST" ]; then
local mgc_ip=$(ping -q -c1 -w1 $mgs_HOST | grep PING | awk '{print $3}' | sed -e "s/(//g" -e "s/)//g")
- [[ x$mgc = xMGC$mgc_ip@$NETTYPE ]] ||
- error_exit "MGSNID=$MGSNID, mounted: $mounted, MGC : $mgc"
+# [[ x$mgc = xMGC$mgc_ip@$NETTYPE ]] ||
+# error_exit "MGSNID=$MGSNID, mounted: $mounted, MGC : $mgc"
fi
fi
return 0
local mgshost=$(mount | grep " $mntpt " | awk -F@ '{print $1}')
mgshost=$(echo $mgshost | awk -F: '{print $1}')
-# if [ "$mgshost" != "$myMGS_host" ]; then
-# error_exit "Bad config file: lustre is mounted with mgs $mgshost, but mgs_HOST=$mgs_HOST, NETTYPE=$NETTYPE
-# Please use correct config or set mds_HOST correctly!"
-# fi
+ if [ "$mgshost" != "$myMGS_host" ]; then
+ log "Bad config file: lustre is mounted with mgs $mgshost, but mgs_HOST=$mgs_HOST, NETTYPE=$NETTYPE
+ Please use correct config or set mds_HOST correctly!"
+ fi
+
+}
+
+check_config_clients () {
+ local clients=${CLIENTS:-$HOSTNAME}
+ local mntpt=$1
+
+ nfs_client_mode && return
+
+ do_rpc_nodes $clients check_config_client $mntpt
sanity_mount_check ||
error "environments are insane!"
fi
}
+is_mounted () {
+ local mntpt=$1
+ local mounted=$(mounted_lustre_filesystems)
+
+ echo $mounted' ' | grep -w -q $mntpt' '
+}
+
check_and_setup_lustre() {
nfs_client_mode && return
local MOUNTED=$(mounted_lustre_filesystems)
- if [ -z "$MOUNTED" ] || ! $(echo $MOUNTED | grep -w -q $MOUNT); then
+
+ local do_check=true
+ # 1.
+ # both MOUNT and MOUNT2 are not mounted
+ if ! is_mounted $MOUNT && ! is_mounted $MOUNT2; then
[ "$REFORMAT" ] && formatall
+ # setupall mounts both MOUNT and MOUNT2 (if MOUNT_2 is set)
setupall
- MOUNTED=$(mounted_lustre_filesystems | head -1)
- [ -z "$MOUNTED" ] && error "NAME=$NAME not mounted"
+ is_mounted $MOUNT || error "NAME=$NAME not mounted"
export I_MOUNTED=yes
- else
- check_config $MOUNT
+ do_check=false
+ # 2.
+ # MOUNT2 is mounted
+ elif is_mounted $MOUNT2; then
+ # 3.
+ # MOUNT2 is mounted, while MOUNT_2 is not set
+ if ! [ "$MOUNT_2" ]; then
+ cleanup_mount $MOUNT2
+ export I_UMOUNTED2=yes
+
+ # 4.
+ # MOUNT2 is mounted, MOUNT_2 is set
+ else
+ # FIXME: what to do if check_config failed?
+ # i.e. if:
+ # 1) remote client has mounted other Lustre fs ?
+ # 2) it has insane env ?
+ # let's try umount MOUNT2 on all clients and mount it again:
+ if ! check_config_clients $MOUNT2; then
+ cleanup_mount $MOUNT2
+ restore_mount $MOUNT2
+ export I_MOUNTED2=yes
+ fi
+ fi
+
+ # 5.
+ # MOUNT is mounted MOUNT2 is not mounted
+ elif [ "$MOUNT_2" ]; then
+ restore_mount $MOUNT2
+ export I_MOUNTED2=yes
+ fi
+
+ if $do_check; then
+ # FIXME: what to do if check_config failed?
+ # i.e. if:
+ # 1) remote client has mounted other Lustre fs?
+ # 2) lustre is mounted on remote_clients atall ?
+ check_config_clients $MOUNT
init_facets_vars
init_param_vars
fi
}
+restore_mount () {
+ local clients=${CLIENTS:-$HOSTNAME}
+ local mntpt=$1
+
+ zconf_mount_clients $clients $mntpt
+}
+
+cleanup_mount () {
+ local clients=${CLIENTS:-$HOSTNAME}
+ local mntpt=$1
+
+ zconf_umount_clients $clients $mntpt
+}
+
cleanup_and_setup_lustre() {
if [ "$ONLY" == "cleanup" -o "`mount | grep $MOUNT`" ]; then
lctl set_param debug=0 || true
}
check_and_cleanup_lustre() {
- if [ "`mount | grep $MOUNT`" ]; then
+ if is_mounted $MOUNT; then
[ -n "$DIR" ] && rm -rf $DIR/[Rdfs][0-9]*
[ "$ENABLE_QUOTA" ] && restore_quota_type || true
fi
+
+ if [ "$I_UMOUNTED2" = "yes" ]; then
+ restore_mount $MOUNT2 || error "restore $MOUNT2 failed"
+ fi
+
+ if [ "$I_MOUNTED2" = "yes" ]; then
+ cleanup_mount $MOUNT2
+ fi
+
if [ "$I_MOUNTED" = "yes" ]; then
cleanupall -f || error "cleanup failed"
+ unset I_MOUNTED
fi
- unset I_MOUNTED
}
#######
echo $(comma_list $expanded)
}
+testslist_filter () {
+ local script=$LUSTRE/tests/${TESTSUITE}.sh
+
+ [ -f $script ] || return 0
+
+ local start_at=$START_AT
+ local stop_at=$STOP_AT
+
+ local var=${TESTSUITE//-/_}_START_AT
+ [ x"${!var}" != x ] && start_at=${!var}
+ var=${TESTSUITE//-/_}_STOP_AT
+ [ x"${!var}" != x ] && stop_at=${!var}
+
+ sed -n 's/^test_\([^ (]*\).*/\1/p' $script | \
+ awk ' BEGIN { if ("'${start_at:-0}'" != 0) flag = 1 }
+ /^'${start_at}'$/ {flag = 0}
+ {if (flag == 1) print $0}
+ /^'${stop_at}'$/ { flag = 1 }'
+}
+
absolute_path() {
(cd `dirname $1`; echo $PWD/`basename $1`)
}
error_noexit() {
local TYPE=${TYPE:-"FAIL"}
local ERRLOG
- lctl set_param fail_loc=0 2>/dev/null || true
local dump=true
# do not dump logs if $1=false
error() {
error_noexit "$@"
- $FAIL_ON_ERROR && exit 1 || true
+ if $FAIL_ON_ERROR; then
+ reset_fail_loc
+ exit 1
+ fi
}
error_exit() {
}
build_test_filter() {
+ EXCEPT="$EXCEPT $(testslist_filter)"
+
[ "$ONLY" ] && log "only running test `echo $ONLY`"
for O in $ONLY; do
eval ONLY_${O}=true
}
basetest() {
- echo ${1%%[a-z]*}
+ if [[ $1 = [a-z]* ]]; then
+ echo $1
+ else
+ echo ${1%%[a-z]*}
+ fi
}
# print a newline if the last test was skipped
log() {
echo "$*"
- lsmod | grep lnet > /dev/null || load_modules
+ module_loaded lnet || load_modules
local MSG="$*"
# Get rid of '
echo ${nodes[i]}
}
+client_only () {
+ [ "$CLIENTONLY" ] || [ "$CLIENTMODSONLY" = yes ]
+}
+
is_patchless ()
{
lctl get_param version | grep -q patchless
local dir=$1
local file=$dir/f0.get_mds_dir_tmpfile
+ mkdir -p $dir
rm -f $file
sleep 1
local iused=$(lfs df -i $dir | grep MDT | awk '{print $3}')
local -a oldused=($iused)
- touch $file
+ openfile -f O_CREAT:O_LOV_DELAY_CREATE -m 0644 $file > /dev/null
sleep 1
iused=$(lfs df -i $dir | grep MDT | awk '{print $3}')
local -a newused=($iused)
}
mdsrate_cleanup () {
- mpi_run -np $1 -machinefile $2 ${MDSRATE} --unlink --nfiles $3 --dir $4 --filefmt $5 $6
+ if [ -d $4 ]; then
+ mpi_run -np $1 -machinefile $2 ${MDSRATE} --unlink --nfiles $3 --dir $4 --filefmt $5 $6
+ rmdir $4
+ fi
}
delayed_recovery_enabled () {
}
########################
-convert_facet2name() {
- case "$1" in
- "ost" ) echo "OST0000" ;;
- "ost1") echo "OST0000" ;;
- "ost2") echo "OST0001" ;;
- "ost3") echo "OST0002" ;;
- "ost4") echo "OST0003" ;;
- "ost5") echo "OST0004" ;;
- *) error "unknown facet!" ;;
- esac
+
+convert_facet2label() {
+ local facet=$1
+
+ if [ x$facet = xost ]; then
+ facet=ost1
+ fi
+
+ local varsvc=${facet}_svc
+
+ if [ -n ${!varsvc} ]; then
+ echo ${!varsvc}
+ else
+ error "No lablel for $facet!"
+ fi
}
get_clientosc_proc_path() {
}
get_osc_import_name() {
- local node=$1
+ local facet=$1
local ost=$2
- local name=$(convert_facet2name $ost)
+ local label=$(convert_facet2label $ost)
- if [ "$node" == "mds" ]; then
- get_mdtosc_proc_path $name
+ if [ "$facet" == "mds" ]; then
+ get_mdtosc_proc_path $label
return 0
fi
- get_clientosc_proc_path $name
+ get_clientosc_proc_path $label
+ return 0
+}
+
+wait_import_state () {
+ local expected=$1
+ local CONN_PROC=$2
+ local CONN_STATE
+ local i=0
+
+ CONN_STATE=$($LCTL get_param -n $CONN_PROC 2>/dev/null | cut -f2)
+ while [ "${CONN_STATE}" != "${expected}" ]; do
+ if [ "${expected}" == "DISCONN" ]; then
+ # for disconn we can check after proc entry is removed
+ [ "x${CONN_STATE}" == "x" ] && return 0
+ # with AT we can have connect request timeout ~ reconnect timeout
+ # and test can't see real disconnect
+ [ "${CONN_STATE}" == "CONNECTING" ] && return 0
+ fi
+ # disconnect rpc should be wait not more obd_timeout
+ [ $i -ge $(($TIMEOUT * 3 / 2)) ] && \
+ error "can't put import for $CONN_PROC into ${expected} state" && return 1
+ sleep 1
+ CONN_STATE=$($LCTL get_param -n $CONN_PROC 2>/dev/null | cut -f2)
+ i=$(($i + 1))
+ done
+
+ log "$CONN_PROC now in ${CONN_STATE} state"
return 0
}
wait_osc_import_state() {
- local node=$1
+ local facet=$1
local ost_facet=$2
local expected=$3
- local ost=$(get_osc_import_name $node $ost_facet)
+ local ost=$(get_osc_import_name $facet $ost_facet)
local CONN_PROC
local CONN_STATE
local i=0
- CONN_PROC="osc.${FSNAME}-${ost}.ost_server_uuid"
- CONN_STATE=$(do_facet $node lctl get_param -n $CONN_PROC 2>/dev/null | cut -f2)
+ CONN_PROC="osc.${ost}.ost_server_uuid"
+ CONN_STATE=$(do_facet $facet lctl get_param -n $CONN_PROC 2>/dev/null | cut -f2)
while [ "${CONN_STATE}" != "${expected}" ]; do
if [ "${expected}" == "DISCONN" ]; then
# for disconn we can check after proc entry is removed
[ $i -ge $(($TIMEOUT * 3 / 2)) ] && \
error "can't put import for ${ost}(${ost_facet}) into ${expected} state" && return 1
sleep 1
- CONN_STATE=$(do_facet $node lctl get_param -n $CONN_PROC 2>/dev/null | cut -f2)
+ CONN_STATE=$(do_facet $facet lctl get_param -n $CONN_PROC 2>/dev/null | cut -f2)
i=$(($i + 1))
done
log "${ost_facet} now in ${CONN_STATE} state"
return 0
}
+
+get_clientmdc_proc_path() {
+ echo "${1}-mdc-*"
+}
+
+do_rpc_nodes () {
+ local list=$1
+ shift
+
+ do_nodes --verbose $list "PATH=$LUSTRE/tests/:$PATH sh rpc.sh $@ "
+}
+
+wait_clients_import_state () {
+ local list=$1
+ local facet=$2
+ local expected=$3
+ shift
+
+ local label=$(convert_facet2label $facet)
+ local proc_path
+ case $facet in
+ ost* ) proc_path="osc.$(get_clientosc_proc_path $label).ost_server_uuid" ;;
+ mds* ) proc_path="mdc.$(get_clientmdc_proc_path $label).mds_server_uuid" ;;
+ *) error "unknown facet!" ;;
+ esac
+
+ if ! do_rpc_nodes $list wait_import_state $expected $proc_path; then
+ error "import is not in ${expected} state"
+ return 1
+ fi
+}
+
+oos_full() {
+ local -a AVAILA
+ local -a GRANTA
+ local OSCFULL=1
+ AVAILA=($(do_nodes $(comma_list $(osts_nodes)) \
+ $LCTL get_param obdfilter.*.kbytesavail))
+ GRANTA=($(do_nodes $(comma_list $(osts_nodes)) \
+ $LCTL get_param -n obdfilter.*.tot_granted))
+ for ((i=0; i<${#AVAILA[@]}; i++)); do
+ local -a AVAIL1=(${AVAILA[$i]//=/ })
+ GRANT=$((${GRANTA[$i]}/1024))
+ echo -n $(echo ${AVAIL1[0]} | cut -d"." -f2) avl=${AVAIL1[1]} grnt=$GRANT diff=$((AVAIL1[1] - GRANT))
+ [ $((AVAIL1[1] - GRANT)) -lt 400 ] && OSCFULL=0 && echo " FULL" || echo
+ done
+ return $OSCFULL
+}
+
+pool_list () {
+ do_facet mgs lctl pool_list $1
+}
+
+create_pool() {
+ local fsname=${1%%.*}
+ local poolname=${1##$fsname.}
+
+ do_facet mgs lctl pool_new $1
+ local RC=$?
+ # get param should return err unless pool is created
+ [[ $RC -ne 0 ]] && return $RC
+
+ wait_update $HOSTNAME "lctl get_param -n lov.$fsname-*.pools.$poolname \
+ 2>/dev/null || echo foo" "" || RC=1
+ if [[ $RC -eq 0 ]]; then
+ add_pool_to_list $1
+ else
+ error "pool_new failed $1"
+ fi
+ return $RC
+}
+
+add_pool_to_list () {
+ local fsname=${1%%.*}
+ local poolname=${1##$fsname.}
+
+ local listvar=${fsname}_CREATED_POOLS
+ eval export ${listvar}=$(expand_list ${!listvar} $poolname)
+}
+
+remove_pool_from_list () {
+ local fsname=${1%%.*}
+ local poolname=${1##$fsname.}
+
+ local listvar=${fsname}_CREATED_POOLS
+ eval export ${listvar}=$(exclude_items_from_list ${!listvar} $poolname)
+}
+
+destroy_pool_int() {
+ local ost
+ local OSTS=$(do_facet $SINGLEMDS lctl pool_list $1 | \
+ awk '$1 !~ /^Pool:/ {print $1}')
+ for ost in $OSTS; do
+ do_facet mgs lctl pool_remove $1 $ost
+ done
+ do_facet mgs lctl pool_destroy $1
+}
+
+# <fsname>.<poolname> or <poolname>
+destroy_pool() {
+ local fsname=${1%%.*}
+ local poolname=${1##$fsname.}
+
+ [[ x$fsname = x$poolname ]] && fsname=$FSNAME
+
+ local RC
+
+ pool_list $fsname.$poolname || return $?
+
+ destroy_pool_int $fsname.$poolname
+ RC=$?
+ [[ $RC -ne 0 ]] && return $RC
+
+ wait_update $HOSTNAME "lctl get_param -n lov.$fsname-*.pools.$poolname \
+ 2>/dev/null || echo foo" "foo" || RC=1
+
+ if [[ $RC -eq 0 ]]; then
+ remove_pool_from_list $fsname.$poolname
+ else
+ error "destroy pool failed $1"
+ fi
+ return $RC
+}
+
+destroy_pools () {
+ local fsname=${1:-$FSNAME}
+ local poolname
+ local listvar=${fsname}_CREATED_POOLS
+
+ pool_list $fsname
+
+ [ x${!listvar} = x ] && return 0
+
+ echo destroy the created pools: ${!listvar}
+ for poolname in ${!listvar//,/ }; do
+ destroy_pool $fsname.$poolname
+ done
+}
+
+cleanup_pools () {
+ local fsname=${1:-$FSNAME}
+ trap 0
+ destroy_pools $fsname
+}
+
+gather_logs () {
+ local list=$1
+
+ local ts=$(date +%s)
+
+ # bug 20237, comment 11
+ # It would also be useful to provide the option
+ # of writing the file to an NFS directory so it doesn't need to be copied.
+ local tmp=$TMP
+ local docp=true
+ [ -d "$SHARED_DIR_LOGS" ] && tmp=$SHARED_DIR_LOGS && docp=false
+
+ # dump lustre logs, dmesg
+ do_nodes $list "log=$tmp/\\\$(hostname)-debug-$ts.log ;
+lctl dk \\\$log >/dev/null;
+log=$tmp/\\\$(hostname)-dmesg-$ts.log;
+dmesg > \\\$log; "
+
+ # FIXME: does it make sense to collect the logs for $ts only, but all
+ # TESTSUITE logs?
+ # rsync $TMP/*${TESTSUITE}* to gather the logs dumped by error fn
+ local logs=$TMP/'*'${TESTSUITE}'*'
+ if $docp; then
+ logs=$logs' '$tmp/'*'$ts'*'
+ fi
+ for node in ${list//,/ }; do
+ rsync -az $node:"$logs" $TMP
+ done
+
+ local archive=$TMP/${TESTSUITE}-$ts.tar.bz2
+ tar -jcf $archive $tmp/*$ts* $TMP/*${TESTSUITE}*
+
+ echo $archive
+}
+
+cleanup_logs () {
+ local list=${1:-$(comma_list $(nodes_list))}
+
+ [ -n ${TESTSUITE} ] && do_nodes $list "rm -f $TMP/*${TESTSUITE}*" || true
+}
+
+do_ls () {
+ local mntpt_root=$1
+ local num_mntpts=$2
+ local dir=$3
+ local i
+ local cmd
+ local pids
+ local rc=0
+
+ for i in $(seq 0 $num_mntpts); do
+ cmd="ls -laf ${mntpt_root}$i/$dir"
+ echo + $cmd;
+ $cmd > /dev/null &
+ pids="$pids $!"
+ done
+ echo pids=$pids
+ for pid in $pids; do
+ wait $pid || rc=$?
+ done
+
+ return $rc
+}
+