b=19451

i=Brian
IOR load for recovery scale tests

diff --git a/lustre/tests/Makefile.am b/lustre/tests/Makefile.am
index b0815c3..4529a8c 100644 (file)
--- a/lustre/tests/Makefile.am
+++ b/lustre/tests/Makefile.am
@@ -19,7 +19,7 @@ noinst_SCRIPTS += mdsrate-lookup-1dir.sh mdsrate-lookup-10dirs.sh
 noinst_SCRIPTS += mdsrate-stat-small.sh mdsrate-stat-large.sh replay-vbr.sh
 noinst_SCRIPTS += lockorder.sh socketclient socketserver runmultiop_bg_pause
 noinst_SCRIPTS += recovery-mds-scale.sh run_dd.sh run_tar.sh run_iozone.sh
-noinst_SCRIPTS += run_dbench.sh recovery-double-scale.sh
+noinst_SCRIPTS += run_dbench.sh run_IOR.sh recovery-double-scale.sh
 noinst_SCRIPTS += recovery-random-scale.sh parallel-scale.sh
 nobase_noinst_SCRIPTS = cfg/local.sh
 nobase_noinst_SCRIPTS += acl/make-tree acl/run cfg/ncli.sh

diff --git a/lustre/tests/functions.sh b/lustre/tests/functions.sh
index c5b806e..83f109d 100644 (file)
--- a/lustre/tests/functions.sh
+++ b/lustre/tests/functions.sh
@@ -24,3 +24,27 @@ signaled() {
     kill -KILL -$PGID
 }
 
+mpi_run () {
+    local mpirun="$MPIRUN $MPIRUN_OPTIONS"
+    local command="$mpirun $@"
+    local mpilog=$TMP/mpi.log
+    local rc
+
+    if [ "$MPI_USER" != root -a $mpirun ]; then
+        echo "+ chmod 0777 $MOUNT"
+        chmod 0777 $MOUNT
+        command="su $MPI_USER sh -c \"$command \""
+    fi
+
+    ls -ald $MOUNT
+    echo "+ $command"
+    eval $command 2>&1 > $mpilog || true
+
+    rc=${PIPESTATUS[0]}
+    if [ $rc -eq 0 ] && grep -q "p4_error: : [^0]" $mpilog ; then
+       rc=1
+    fi
+    cat $mpilog
+    return $rc
+}
+

diff --git a/lustre/tests/run_IOR.sh b/lustre/tests/run_IOR.sh
new file mode 100755 (executable)
index 0000000..6da7f54
--- /dev/null
+++ b/lustre/tests/run_IOR.sh
@@ -0,0 +1,65 @@
+#!/bin/bash
+set -x
+
+TMP=${TMP:-/tmp}
+
+TESTSUITELOG=${TESTSUITELOG:-$TMP/recovery-mds-scale}
+LOG=${TESTSUITELOG}_$(basename $0)-$(hostname)
+DEBUGLOG=${LOG}.debug
+
+mkdir -p ${LOG%/*}
+
+rm -f $LOG $DEBUGLOG
+exec 2>$DEBUGLOG
+
+. $(dirname $0)/functions.sh
+
+IOR=${IOR:-"$(which IOR)"}
+
+assert_env MOUNT END_RUN_FILE LOAD_PID_FILE IOR
+
+trap signaled TERM
+
+# if MACHINEFILE set and exists -- use it
+if [ -z $MACHINEFILE ] || [ ! -e $MACHINEFILE ]; then
+    MACHINEFILE=$TMP/$(basename $0)-$(hostname).machines
+    echo $(hostname) >$MACHINEFILE
+fi
+
+THREADS_PER_CLIENT=${THREADS_PER_CLIENT:-3}
+NUM_CLIENTS=$(cat $MACHINEFILE | wc -l)
+
+# recovery-*-scale scripts use this to signal the client loads to die
+echo $$ >$LOAD_PID_FILE
+
+TESTDIR=${TESTDIR:-$MOUNT/d0.ior-$(hostname)}
+
+CONTINUE=true
+while [ ! -e "$END_RUN_FILE" ] && $CONTINUE; do
+    echoerr "$(date +'%F %H:%M:%S'): IOR run starting"
+    mkdir -p $TESTDIR
+    # need this only if TESTDIR is not default
+    chmod -R 777 $TESTDIR
+
+    mpi_run -np $((NUM_CLIENTS * THREADS_PER_CLIENT)) -machinefile ${MACHINEFILE} \
+        $IOR -a POSIX -b 1g -o $TESTDIR/IOR-file -s 1 -t 1m -v -w -r 1>$LOG &
+    load_pid=$!
+    wait $load_pid
+    if [ ${PIPESTATUS[0]} -eq 0 ]; then
+       echoerr "$(date +'%F %H:%M:%S'): IOR succeeded"
+       cd $TMP
+       rm -rf $TESTDIR
+       echoerr "$(date +'%F %H:%M:%S'): IOR run finished"
+    else
+       echoerr "$(date +'%F %H:%M:%S'): IOR failed"
+       if [ -z "$ERRORS_OK" ]; then
+           echo $(hostname) >> $END_RUN_FILE
+       fi
+       if [ $BREAK_ON_ERROR ]; then
+           # break
+            CONTINUE=false
+       fi
+    fi
+done
+
+echoerr "$(date +'%F %H:%M:%S'): IOR run exiting"

diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh
index 0047ab2..5605ba2 100644 (file)
--- a/lustre/tests/test-framework.sh
+++ b/lustre/tests/test-framework.sh
@@ -679,10 +679,14 @@ check_progs_installed () {
     shift
     local progs=$@
 
-    do_nodes $clients "set -x ; PATH=:$PATH status=true; for prog in $progs; do
-        which \\\$prog || { echo \\\$prog missing on \\\$(hostname) && status=false; }
-        done;
-        eval \\\$status"
+    do_nodes $clients "set -x ; PATH=:$PATH; status=true;
+for prog in $progs; do
+    if ! [ \\\"\\\$(which \\\$prog)\\\"  -o  \\\"\\\${!prog}\\\" ]; then
+       echo \\\$prog missing on \\\$(hostname);
+       status=false;
+    fi
+done;
+eval \\\$status"
 }
 
 client_var_name() {
@@ -1806,7 +1810,7 @@ error_noexit() {
     ERRLOG=$TMP/lustre_${TESTSUITE}_${TESTNAME}.$(date +%s)
     echo "Dumping lctl log to $ERRLOG"
     # We need to dump the logs on all nodes
-    local NODES=${NODES:-$(nodes_list)}
+    local NODES=$(nodes_list)
     for NODE in $NODES; do
         do_node $NODE $LCTL dk $ERRLOG
     done
@@ -2358,30 +2362,6 @@ get_stripe_info() {
        rm -f $tmp_file
 }
 
-mpi_run () {
-    local mpirun="$MPIRUN $MPIRUN_OPTIONS"
-    local command="$mpirun $@"
-    local mpilog=$TMP/mpi.log
-    local rc
-
-    if [ "$MPI_USER" != root -a $mpirun ]; then
-        echo "+ chmod 0777 $MOUNT"
-        chmod 0777 $MOUNT
-        command="su $MPI_USER sh -c \"$command \""
-    fi
-
-    ls -ald $MOUNT
-    echo "+ $command"
-    eval $command 2>&1 > $mpilog || true
-
-    rc=${PIPESTATUS[0]}
-    if [ $rc -eq 0 ] && grep -q "p4_error: : [^0]" $mpilog ; then
-       rc=1
-    fi
-    cat $mpilog
-    return $rc
-}
-
 mdsrate_cleanup () {
     mpi_run -np $1 -machinefile $2 ${MDSRATE} --unlink --nfiles $3 --dir $4 --filefmt $5 $6
 }