+# $1 node
+# $2 file
+# $3 $RUNAS
+get_stripe_info() {
+ local tmp_file
+
+ stripe_size=0
+ stripe_count=0
+ stripe_index=0
+ tmp_file=$(mktemp)
+
+ do_facet $1 $3 lfs getstripe -v $2 > $tmp_file
+
+ stripe_size=`awk '$1 ~ /size/ {print $2}' $tmp_file`
+ stripe_count=`awk '$1 ~ /count/ {print $2}' $tmp_file`
+ stripe_index=`awk '$1 ~ /stripe_offset/ {print $2}' $tmp_file`
+ rm -f $tmp_file
+}
+
+# CMD: determine mds index where directory inode presents
+get_mds_dir () {
+ local dir=$1
+ local file=$dir/f0.get_mds_dir_tmpfile
+
+ rm -f $file
+ sleep 1
+ local iused=$(lfs df -i $dir | grep MDT | awk '{print $3}')
+ local -a oldused=($iused)
+
+ touch $file
+ sleep 1
+ iused=$(lfs df -i $dir | grep MDT | awk '{print $3}')
+ local -a newused=($iused)
+
+ local num=0
+ for ((i=0; i<${#newused[@]}; i++)); do
+ if [ ${oldused[$i]} -lt ${newused[$i]} ]; then
+ echo $(( i + 1 ))
+ rm -f $file
+ return 0
+ fi
+ done
+ error "mdt-s : inodes count OLD ${oldused[@]} NEW ${newused[@]}"
+}
+
+mdsrate_cleanup () {
+ mpi_run -np $1 -machinefile $2 ${MDSRATE} --unlink --nfiles $3 --dir $4 --filefmt $5 $6
+}
+
+delayed_recovery_enabled () {
+ local var=${SINGLEMDS}_svc
+ do_facet $SINGLEMDS lctl get_param -n mdd.${!var}.stale_export_age > /dev/null 2>&1
+}
+
+########################
+
+convert_facet2label() {
+ local facet=$1
+
+ if [ x$facet = xost ]; then
+ facet=ost1
+ fi
+
+ local varsvc=${facet}_svc
+
+ if [ -n ${!varsvc} ]; then
+ echo ${!varsvc}
+ else
+ error "No lablel for $facet!"
+ fi
+}
+
+get_clientosc_proc_path() {
+ local ost=$1
+
+ echo "{$1}-osc-*"
+}
+
+get_lustre_version () {
+ local node=${1:-"mds"}
+ do_facet $node $LCTL get_param -n version | awk '/^lustre:/ {print $2}'
+}
+
+get_mds_version_major () {
+ local version=$(get_lustre_version mds)
+ echo $version | awk -F. '{print $1}'
+}
+
+get_mds_version_minor () {
+ local version=$(get_lustre_version mds)
+ echo $version | awk -F. '{print $2}'
+}
+
+get_mdtosc_proc_path() {
+ local ost=$1
+ local major=$(get_mds_version_major)
+ local minor=$(get_mds_version_minor)
+ if [ $major -le 1 -a $minor -le 8 ] ; then
+ echo "${ost}-osc"
+ else
+ echo "${ost}-osc-MDT0000"
+ fi
+}
+
+get_osc_import_name() {
+ local facet=$1
+ local ost=$2
+ local label=$(convert_facet2label $ost)
+
+ if [ "$facet" == "mds" ]; then
+ get_mdtosc_proc_path $label
+ return 0
+ fi
+
+ get_clientosc_proc_path $label
+ return 0
+}
+
+wait_import_state () {
+ local expected=$1
+ local CONN_PROC=$2
+ local CONN_STATE
+ local i=0
+
+ CONN_STATE=$($LCTL get_param -n $CONN_PROC 2>/dev/null | cut -f2)
+ while [ "${CONN_STATE}" != "${expected}" ]; do
+ if [ "${expected}" == "DISCONN" ]; then
+ # for disconn we can check after proc entry is removed
+ [ "x${CONN_STATE}" == "x" ] && return 0
+ # with AT we can have connect request timeout ~ reconnect timeout
+ # and test can't see real disconnect
+ [ "${CONN_STATE}" == "CONNECTING" ] && return 0
+ fi
+ # disconnect rpc should be wait not more obd_timeout
+ [ $i -ge $(($TIMEOUT * 3 / 2)) ] && \
+ error "can't put import for $CONN_PROC into ${expected} state" && return 1
+ sleep 1
+ CONN_STATE=$($LCTL get_param -n $CONN_PROC 2>/dev/null | cut -f2)
+ i=$(($i + 1))
+ done
+
+ log "$CONN_PROC now in ${CONN_STATE} state"
+ return 0
+}
+
+wait_osc_import_state() {
+ local facet=$1
+ local ost_facet=$2
+ local expected=$3
+ local ost=$(get_osc_import_name $facet $ost_facet)
+ local CONN_PROC
+ local CONN_STATE
+ local i=0
+
+ CONN_PROC="osc.${ost}.ost_server_uuid"
+ CONN_STATE=$(do_facet $facet lctl get_param -n $CONN_PROC 2>/dev/null | cut -f2)
+ while [ "${CONN_STATE}" != "${expected}" ]; do
+ if [ "${expected}" == "DISCONN" ]; then
+ # for disconn we can check after proc entry is removed
+ [ "x${CONN_STATE}" == "x" ] && return 0
+ # with AT we can have connect request timeout ~ reconnect timeout
+ # and test can't see real disconnect
+ [ "${CONN_STATE}" == "CONNECTING" ] && return 0
+ fi
+ # disconnect rpc should be wait not more obd_timeout
+ [ $i -ge $(($TIMEOUT * 3 / 2)) ] && \
+ error "can't put import for ${ost}(${ost_facet}) into ${expected} state" && return 1
+ sleep 1
+ CONN_STATE=$(do_facet $facet lctl get_param -n $CONN_PROC 2>/dev/null | cut -f2)
+ i=$(($i + 1))
+ done
+
+ log "${ost_facet} now in ${CONN_STATE} state"
+ return 0
+}
+
+get_clientmdc_proc_path() {
+ echo "${1}-mdc-*"
+}
+
+do_rpc_nodes () {
+ local list=$1
+ shift
+
+ do_nodes --verbose $list "PATH=$LUSTRE/tests/:$PATH sh rpc.sh $@ "
+}
+
+wait_clients_import_state () {
+ local list=$1
+ local facet=$2
+ local expected=$3
+ shift
+
+ local label=$(convert_facet2label $facet)
+ local proc_path
+ case $facet in
+ ost* ) proc_path="osc.$(get_clientosc_proc_path $label).ost_server_uuid" ;;
+ mds* ) proc_path="mdc.$(get_clientmdc_proc_path $label).mds_server_uuid" ;;
+ *) error "unknown facet!" ;;
+ esac
+
+
+ if ! do_rpc_nodes $list wait_import_state $expected $proc_path; then
+ error "import is not in ${expected} state"
+ return 1
+ fi
+}
+
+oos_full() {
+ local -a AVAILA
+ local -a GRANTA
+ local OSCFULL=1
+ AVAILA=($(do_nodes $(comma_list $(osts_nodes)) \
+ $LCTL get_param obdfilter.*.kbytesavail))
+ GRANTA=($(do_nodes $(comma_list $(osts_nodes)) \
+ $LCTL get_param -n obdfilter.*.tot_granted))
+ for ((i=0; i<${#AVAILA[@]}; i++)); do
+ local -a AVAIL1=(${AVAILA[$i]//=/ })
+ GRANT=$((${GRANTA[$i]}/1024))
+ echo -n $(echo ${AVAIL1[0]} | cut -d"." -f2) avl=${AVAIL1[1]} grnt=$GRANT diff=$((AVAIL1[1] - GRANT))
+ [ $((AVAIL1[1] - GRANT)) -lt 400 ] && OSCFULL=0 && echo " FULL" || echo
+ done
+ return $OSCFULL
+}
+
+gather_logs () {
+ local list=$1
+
+ local ts=$(date +%s)
+
+ # bug 20237, comment 11
+ # It would also be useful to provide the option
+ # of writing the file to an NFS directory so it doesn't need to be copied.
+ local tmp=$TMP
+ local docp=true
+ [ -d "$SHARED_DIR_LOGS" ] && tmp=$SHARED_DIR_LOGS && docp=false
+
+ # dump lustre logs, dmesg
+ do_nodes $list "log=$tmp/\\\$(hostname)-debug-$ts.log ;
+lctl dk \\\$log >/dev/null;
+log=$tmp/\\\$(hostname)-dmesg-$ts.log;
+dmesg > \\\$log; "
+
+ # FIXME: does it make sense to collect the logs for $ts only, but all
+ # TESTSUITE logs?
+ # rsync $TMP/*${TESTSUITE}* to gather the logs dumped by error fn
+ local logs=$TMP/'*'${TESTSUITE}'*'
+ if $docp; then
+ logs=$logs' '$tmp/'*'$ts'*'
+ fi
+ for node in ${list//,/ }; do
+ rsync -az $node:"$logs" $TMP
+ done
+
+ local archive=$TMP/${TESTSUITE}-$ts.tar.bz2
+ tar -jcf $archive $tmp/*$ts* $TMP/*${TESTSUITE}*
+
+ echo $archive
+}
+
+cleanup_logs () {
+ local list=${1:-$(comma_list $(nodes_list))}
+
+ [ -n ${TESTSUITE} ] && do_nodes $list "rm -f $TMP/*${TESTSUITE}*" || true
+}
+