+
+get_clientmdc_proc_path() {
+ echo "${1}-mdc-*"
+}
+
+do_rpc_nodes () {
+ local list=$1
+ shift
+
+ do_nodes --verbose $list "PATH=$LUSTRE/tests/:$PATH sh rpc.sh $@ "
+}
+
+wait_clients_import_state () {
+ local list=$1
+ local facet=$2
+ local expected=$3
+ shift
+
+ local label=$(convert_facet2label $facet)
+ local proc_path
+ case $facet in
+ ost* ) proc_path="osc.$(get_clientosc_proc_path $label).ost_server_uuid" ;;
+ mds* ) proc_path="mdc.$(get_clientmdc_proc_path $label).mds_server_uuid" ;;
+ *) error "unknown facet!" ;;
+ esac
+
+ if ! do_rpc_nodes $list wait_import_state $expected $proc_path; then
+ error "import is not in ${expected} state"
+ return 1
+ fi
+}
+
+oos_full() {
+ local -a AVAILA
+ local -a GRANTA
+ local OSCFULL=1
+ AVAILA=($(do_nodes $(comma_list $(osts_nodes)) \
+ $LCTL get_param obdfilter.*.kbytesavail))
+ GRANTA=($(do_nodes $(comma_list $(osts_nodes)) \
+ $LCTL get_param -n obdfilter.*.tot_granted))
+ for ((i=0; i<${#AVAILA[@]}; i++)); do
+ local -a AVAIL1=(${AVAILA[$i]//=/ })
+ GRANT=$((${GRANTA[$i]}/1024))
+ echo -n $(echo ${AVAIL1[0]} | cut -d"." -f2) avl=${AVAIL1[1]} grnt=$GRANT diff=$((AVAIL1[1] - GRANT))
+ [ $((AVAIL1[1] - GRANT)) -lt 400 ] && OSCFULL=0 && echo " FULL" || echo
+ done
+ return $OSCFULL
+}
+
+pool_list () {
+ do_facet mgs lctl pool_list $1
+}
+
+create_pool() {
+ local fsname=${1%%.*}
+ local poolname=${1##$fsname.}
+
+ do_facet mgs lctl pool_new $1
+ local RC=$?
+ # get param should return err unless pool is created
+ [[ $RC -ne 0 ]] && return $RC
+
+ wait_update $HOSTNAME "lctl get_param -n lov.$fsname-*.pools.$poolname \
+ 2>/dev/null || echo foo" "" || RC=1
+ if [[ $RC -eq 0 ]]; then
+ add_pool_to_list $1
+ else
+ error "pool_new failed $1"
+ fi
+ return $RC
+}
+
+add_pool_to_list () {
+ local fsname=${1%%.*}
+ local poolname=${1##$fsname.}
+
+ local listvar=${fsname}_CREATED_POOLS
+ eval export ${listvar}=$(expand_list ${!listvar} $poolname)
+}
+
+remove_pool_from_list () {
+ local fsname=${1%%.*}
+ local poolname=${1##$fsname.}
+
+ local listvar=${fsname}_CREATED_POOLS
+ eval export ${listvar}=$(exclude_items_from_list ${!listvar} $poolname)
+}
+
+destroy_pool_int() {
+ local ost
+ local OSTS=$(do_facet $SINGLEMDS lctl pool_list $1 | \
+ awk '$1 !~ /^Pool:/ {print $1}')
+ for ost in $OSTS; do
+ do_facet mgs lctl pool_remove $1 $ost
+ done
+ do_facet mgs lctl pool_destroy $1
+}
+
+# <fsname>.<poolname> or <poolname>
+destroy_pool() {
+ local fsname=${1%%.*}
+ local poolname=${1##$fsname.}
+
+ [[ x$fsname = x$poolname ]] && fsname=$FSNAME
+
+ local RC
+
+ pool_list $fsname.$poolname || return $?
+
+ destroy_pool_int $fsname.$poolname
+ RC=$?
+ [[ $RC -ne 0 ]] && return $RC
+
+ wait_update $HOSTNAME "lctl get_param -n lov.$fsname-*.pools.$poolname \
+ 2>/dev/null || echo foo" "foo" || RC=1
+
+ if [[ $RC -eq 0 ]]; then
+ remove_pool_from_list $fsname.$poolname
+ else
+ error "destroy pool failed $1"
+ fi
+ return $RC
+}
+
+destroy_pools () {
+ local fsname=${1:-$FSNAME}
+ local poolname
+ local listvar=${fsname}_CREATED_POOLS
+
+ pool_list $fsname
+
+ [ x${!listvar} = x ] && return 0
+
+ echo destroy the created pools: ${!listvar}
+ for poolname in ${!listvar//,/ }; do
+ destroy_pool $fsname.$poolname
+ done
+}
+
+cleanup_pools () {
+ local fsname=${1:-$FSNAME}
+ trap 0
+ destroy_pools $fsname
+}
+
+gather_logs () {
+ local list=$1
+
+ local ts=$(date +%s)
+
+ # bug 20237, comment 11
+ # It would also be useful to provide the option
+ # of writing the file to an NFS directory so it doesn't need to be copied.
+ local tmp=$TMP
+ local docp=true
+ [ -d "$SHARED_DIR_LOGS" ] && tmp=$SHARED_DIR_LOGS && docp=false
+
+ # dump lustre logs, dmesg
+ do_nodes $list "log=$tmp/\\\$(hostname)-debug-$ts.log ;
+lctl dk \\\$log >/dev/null;
+log=$tmp/\\\$(hostname)-dmesg-$ts.log;
+dmesg > \\\$log; "
+
+ # FIXME: does it make sense to collect the logs for $ts only, but all
+ # TESTSUITE logs?
+ # rsync $TMP/*${TESTSUITE}* to gather the logs dumped by error fn
+ local logs=$TMP/'*'${TESTSUITE}'*'
+ if $docp; then
+ logs=$logs' '$tmp/'*'$ts'*'
+ fi
+ for node in ${list//,/ }; do
+ rsync -az $node:"$logs" $TMP
+ done
+
+ local archive=$TMP/${TESTSUITE}-$ts.tar.bz2
+ tar -jcf $archive $tmp/*$ts* $TMP/*${TESTSUITE}*
+
+ echo $archive
+}
+
+cleanup_logs () {
+ local list=${1:-$(comma_list $(nodes_list))}
+
+ [ -n ${TESTSUITE} ] && do_nodes $list "rm -f $TMP/*${TESTSUITE}*" || true
+}
+
+do_ls () {
+ local mntpt_root=$1
+ local num_mntpts=$2
+ local dir=$3
+ local i
+ local cmd
+ local pids
+ local rc=0
+
+ for i in $(seq 0 $num_mntpts); do
+ cmd="ls -laf ${mntpt_root}$i/$dir"
+ echo + $cmd;
+ $cmd > /dev/null &
+ pids="$pids $!"
+ done
+ echo pids=$pids
+ for pid in $pids; do
+ wait $pid || rc=$?
+ done
+
+ return $rc
+}
+