+sync_clients() {
+ [ -d $DIR1 ] && cd $DIR1 && sync; sleep 1; sync
+ [ -d $DIR2 ] && cd $DIR2 && sync; sleep 1; sync
+ cd $SAVE_PWD
+}
+
+check_grant() {
+ export base=`basetest $1`
+ [ "$CHECK_GRANT" == "no" ] && return 0
+
+ testname=GCHECK_ONLY_${base}
+ [ ${!testname}x == x ] && return 0
+
+ echo -n "checking grant......"
+ cd $SAVE_PWD
+ # write some data to sync client lost_grant
+ rm -f $DIR1/${tfile}_check_grant_* 2>&1
+ for i in `seq $OSTCOUNT`; do
+ $LFS setstripe $DIR1/${tfile}_check_grant_$i -i $(($i -1)) -c 1
+ dd if=/dev/zero of=$DIR1/${tfile}_check_grant_$i bs=4k \
+ count=1 > /dev/null 2>&1
+ done
+ # sync all the data and make sure no pending data on server
+ sync_clients
+
+ #get client grant and server grant
+ client_grant=0
+ for d in `lctl get_param -n osc.*.cur_grant_bytes`; do
+ client_grant=$((client_grant + $d))
+ done
+ server_grant=0
+ for d in `lctl get_param -n obdfilter.*.tot_granted`; do
+ server_grant=$((server_grant + $d))
+ done
+
+ # cleanup the check_grant file
+ for i in `seq $OSTCOUNT`; do
+ rm $DIR1/${tfile}_check_grant_$i
+ done
+
+ #check whether client grant == server grant
+ if [ $client_grant != $server_grant ]; then
+ echo "failed: client:${client_grant} server: ${server_grant}"
+ return 1
+ else
+ echo "pass"
+ fi
+}
+
+########################
+# helper functions
+
+osc_to_ost()
+{
+ osc=$1
+ ost=`echo $1 | awk -F_ '{print $3}'`
+ if [ -z $ost ]; then
+ ost=`echo $1 | sed 's/-osc.*//'`
+ fi
+ echo $ost
+}
+
+remote_node () {
+ local node=$1
+ [ "$node" != "$(hostname)" ]
+}
+
+remote_mds ()
+{
+ local node
+ for node in $(mdts_nodes); do
+ remote_node $node && return 0
+ done
+ return 1
+}
+
+remote_mds_nodsh()
+{
+ remote_mds && [ "$PDSH" = "no_dsh" -o -z "$PDSH" -o -z "$mds_HOST" ]
+}
+
+remote_ost ()
+{
+ local node
+ for node in $(osts_nodes) ; do
+ remote_node $node && return 0
+ done
+ return 1
+}
+
+remote_ost_nodsh()
+{
+ remote_ost && [ "$PDSH" = "no_dsh" -o -z "$PDSH" -o -z "$ost_HOST" ]
+}
+
+mdts_nodes () {
+ local MDSNODES
+ local NODES_sort
+ for num in `seq $MDSCOUNT`; do
+ MDSNODES="$MDSNODES $(facet_host mds$num)"
+ done
+ NODES_sort=$(for i in $MDSNODES; do echo $i; done | sort -u)
+
+ echo $NODES_sort
+}
+
+remote_servers () {
+ remote_ost && remote_mds
+}
+
+osts_nodes () {
+ local OSTNODES=$(facet_host ost1)
+ local NODES_sort
+
+ for num in `seq $OSTCOUNT`; do
+ local myOST=$(facet_host ost$num)
+ OSTNODES="$OSTNODES $myOST"
+ done
+ NODES_sort=$(for i in $OSTNODES; do echo $i; done | sort -u)
+
+ echo $NODES_sort
+}
+
+nodes_list () {
+ # FIXME. We need a list of clients
+ local myNODES=$HOSTNAME
+ local myNODES_sort
+
+ # CLIENTS (if specified) contains the local client
+ [ -n "$CLIENTS" ] && myNODES=${CLIENTS//,/ }
+
+ if [ "$PDSH" -a "$PDSH" != "no_dsh" ]; then
+ myNODES="$myNODES $(osts_nodes) $(mdts_nodes)"
+ fi
+
+ myNODES_sort=$(for i in $myNODES; do echo $i; done | sort -u)
+
+ echo $myNODES_sort
+}
+
+remote_nodes_list () {
+ local rnodes=$(nodes_list)
+ rnodes=$(echo " $rnodes " | sed -re "s/\s+$HOSTNAME\s+/ /g")
+ echo $rnodes
+}
+
+init_clients_lists () {
+ # Sanity check: exclude the local client from RCLIENTS
+ local rclients=$(echo " $RCLIENTS " | sed -re "s/\s+$HOSTNAME\s+/ /g")
+
+ # Sanity check: exclude the dup entries
+ rclients=$(for i in $rclients; do echo $i; done | sort -u)
+
+ local clients="$SINGLECLIENT $HOSTNAME $rclients"
+
+ # Sanity check: exclude the dup entries from CLIENTS
+ # for those configs which has SINGLCLIENT set to local client
+ clients=$(for i in $clients; do echo $i; done | sort -u)
+
+ CLIENTS=`comma_list $clients`
+ local -a remoteclients=($rclients)
+ for ((i=0; $i<${#remoteclients[@]}; i++)); do
+ varname=CLIENT$((i + 2))
+ eval $varname=${remoteclients[i]}
+ done
+
+ CLIENTCOUNT=$((${#remoteclients[@]} + 1))
+}
+
+is_patchless ()
+{
+ lctl get_param version | grep -q patchless
+}
+
+get_node_count() {
+ local nodes="$@"
+ echo $nodes | wc -w || true
+}
+
+mixed_ost_devs () {
+ local nodes=$(osts_nodes)
+ local osscount=$(get_node_count "$nodes")
+ [ ! "$OSTCOUNT" = "$osscount" ]
+}
+
+mixed_mdt_devs () {
+ local nodes=$(mdts_nodes)
+ local mdtcount=$(get_node_count "$nodes")
+ [ ! "$MDSCOUNT" = "$mdtcount" ]
+}
+
+generate_machine_file() {
+ local nodes=${1//,/ }
+ local machinefile=$2
+ rm -f $machinefile || error "can't rm $machinefile"
+ for node in $nodes; do
+ echo $node >>$machinefile
+ done
+}
+
+get_stripe () {
+ local file=$1/stripe
+ touch $file
+ $LFS getstripe -v $file || error
+ rm -f $file
+}
+
+check_runas_id_ret() {
+ local myRC=0
+ local myRUNAS_ID=$1
+ shift
+ local myRUNAS=$@
+ if [ -z "$myRUNAS" ]; then
+ error_exit "myRUNAS command must be specified for check_runas_id"
+ fi
+ if $GSS_KRB5; then
+ $myRUNAS krb5_login.sh || \
+ error "Failed to refresh Kerberos V5 TGT for UID $myRUNAS_ID."
+ fi
+ mkdir $DIR/d0_runas_test
+ chmod 0755 $DIR
+ chown $myRUNAS_ID:$myRUNAS_ID $DIR/d0_runas_test
+ $myRUNAS touch $DIR/d0_runas_test/f$$ || myRC=$?
+ rm -rf $DIR/d0_runas_test
+ return $myRC
+}
+
+check_runas_id() {
+ local myRUNAS_ID=$1
+ shift
+ local myRUNAS=$@
+ check_runas_id_ret $myRUNAS_ID $myRUNAS || \
+ error "unable to write to $DIR/d0_runas_test as UID $myRUNAS_ID.
+ Please set RUNAS_ID to some UID which exists on MDS and client or
+ add user $myRUNAS_ID:$myRUNAS_ID on these nodes."
+}
+
+# Run multiop in the background, but wait for it to print
+# "PAUSING" to its stdout before returning from this function.
+multiop_bg_pause() {
+ MULTIOP_PROG=${MULTIOP_PROG:-multiop}
+ FILE=$1
+ ARGS=$2
+
+ TMPPIPE=/tmp/multiop_open_wait_pipe.$$
+ mkfifo $TMPPIPE
+
+ echo "$MULTIOP_PROG $FILE v$ARGS"
+ $MULTIOP_PROG $FILE v$ARGS > $TMPPIPE &
+
+ echo "TMPPIPE=${TMPPIPE}"
+ read -t 60 multiop_output < $TMPPIPE
+ if [ $? -ne 0 ]; then
+ rm -f $TMPPIPE
+ return 1
+ fi
+ rm -f $TMPPIPE
+ if [ "$multiop_output" != "PAUSING" ]; then
+ echo "Incorrect multiop output: $multiop_output"
+ kill -9 $PID
+ return 1
+ fi
+
+ return 0
+}
+
+check_rate() {
+ local OP=$1
+ local TARGET_RATE=$2
+ local NUM_CLIENTS=$3
+ local LOG=$4
+
+ local RATE=$(awk '/^Rate: [0-9\.]+ '"${OP}"'s\/sec/ { print $2}' ${LOG})
+
+ # We need to use bc since the rate is a floating point number
+ local RES=$(echo "${RATE} < ${TARGET_RATE}" | bc -l )
+ if [ ${RES} -eq 0 ]; then
+ echo "Success: ${RATE} ${OP}s/sec met target rate" \
+ "${TARGET_RATE} ${OP}s/sec for ${NUM_CLIENTS} client(s)."
+ return 0
+ else
+ echo "Failure: ${RATE} ${OP}s/sec did not meet target rate" \
+ "${TARGET_RATE} ${OP}s/sec for ${NUM_CLIENTS} client(s)."
+ return 1
+ fi
+}
+
+# reset llite stat counters
+clear_llite_stats(){
+ lctl set_param -n llite.*.stats 0
+}
+
+# sum llite stat items
+calc_llite_stats() {
+ local res=$(lctl get_param -n llite.*.stats |
+ awk 'BEGIN {s = 0} END {print s} /^'"$1"'/ {s += $2}')
+ echo $res
+}
+
+# reset osc stat counters
+clear_osc_stats(){
+ lctl set_param -n osc.*.osc_stats 0
+}
+
+# sum osc stat items
+calc_osc_stats() {
+ local res=$(lctl get_param -n osc.*.osc_stats |
+ awk 'BEGIN {s = 0} END {print s} /^'"$1"'/ {s += $2}')
+ echo $res
+}
+
+calc_sum () {
+ awk 'BEGIN {s = 0}; {s += $1}; END {print s}'
+}
+
+calc_osc_kbytes () {
+ $LCTL get_param -n osc.*[oO][sS][cC][-_][0-9a-f]*.$1 | calc_sum
+}
+
+# save_lustre_params(node, parameter_mask)
+# generate a stream of formatted strings (<node> <param name>=<param value>)
+save_lustre_params() {
+ local s
+ do_node $1 "lctl get_param $2" | while read s; do echo "$1 $s"; done
+}
+
+# restore lustre parameters from input stream, produces by save_lustre_params
+restore_lustre_params() {
+ local node
+ local name
+ local val
+ while IFS=" =" read node name val; do
+ do_node $node "lctl set_param -n $name $val"
+ done
+}
+
+check_catastrophe () {
+ local rnodes=$(comma_list $(remote_nodes_list))
+
+ [ -f $CATASTROPHE ] && [ `cat $CATASTROPHE` -ne 0 ] && return 1
+ if [ $rnodes ]; then
+ do_nodes $rnodes "[ -f $CATASTROPHE ] && { [ \`cat $CATASTROPHE\` -eq 0 ] || false; } || true"
+ fi
+}
+
+# $1 node
+# $2 file
+get_stripe_info() {
+ local tmp_file
+
+ stripe_size=0
+ stripe_count=0
+ stripe_index=0
+ tmp_file=$(mktemp)
+
+ do_facet $1 lfs getstripe -v $2 > $tmp_file
+
+ stripe_size=`awk '$1 ~ /size/ {print $2}' $tmp_file`
+ stripe_count=`awk '$1 ~ /count/ {print $2}' $tmp_file`
+ stripe_index=`awk '/obdidx/ {start = 1; getline; print $1; exit}' $tmp_file`
+ rm -f $tmp_file