From e2cd3061d9419393bc55b19df2d9d51d0cbe8932 Mon Sep 17 00:00:00 2001
From: grev <grev>
Date: Fri, 26 Jun 2009 11:04:59 +0000
Subject: [PATCH] b=19451 i=Brian IOR client load for recovery scale tests

---
 lustre/tests/Makefile.am       |  2 +-
 lustre/tests/functions.sh      | 24 ++++++++++++++++
 lustre/tests/run_IOR.sh        | 65 ++++++++++++++++++++++++++++++++++++++++++
 lustre/tests/test-framework.sh | 36 ++++++-----------------
 4 files changed, 98 insertions(+), 29 deletions(-)
 create mode 100755 lustre/tests/run_IOR.sh

diff --git a/lustre/tests/Makefile.am b/lustre/tests/Makefile.am
index cb76ebc..b39cddb 100644
--- a/lustre/tests/Makefile.am
+++ b/lustre/tests/Makefile.am
@@ -21,7 +21,7 @@ noinst_SCRIPTS += mdsrate-stat-small.sh mdsrate-stat-large.sh
 noinst_SCRIPTS += lockorder.sh socketclient socketserver runmultiop_bg_pause
 noinst_SCRIPTS += sanity-sec.sh sanity-gss.sh krb5_login.sh setup_kerberos.sh
 noinst_SCRIPTS += recovery-mds-scale.sh run_dd.sh run_tar.sh run_iozone.sh
-noinst_SCRIPTS += run_dbench.sh recovery-double-scale.sh
+noinst_SCRIPTS += run_dbench.sh run_IOR.sh recovery-double-scale.sh
 noinst_SCRIPTS += recovery-random-scale.sh parallel-scale.sh
 noinst_SCRIPTS += lreplicate-test.sh
 nobase_noinst_SCRIPTS = cfg/local.sh
diff --git a/lustre/tests/functions.sh b/lustre/tests/functions.sh
index c5b806e..83f109d 100644
--- a/lustre/tests/functions.sh
+++ b/lustre/tests/functions.sh
@@ -24,3 +24,27 @@ signaled() {
     kill -KILL -$PGID
 }
 
+mpi_run () {
+    local mpirun="$MPIRUN $MPIRUN_OPTIONS"
+    local command="$mpirun $@"
+    local mpilog=$TMP/mpi.log
+    local rc
+
+    if [ "$MPI_USER" != root -a $mpirun ]; then
+        echo "+ chmod 0777 $MOUNT"
+        chmod 0777 $MOUNT
+        command="su $MPI_USER sh -c \"$command \""
+    fi
+
+    ls -ald $MOUNT
+    echo "+ $command"
+    eval $command 2>&1 > $mpilog || true
+
+    rc=${PIPESTATUS[0]}
+    if [ $rc -eq 0 ] && grep -q "p4_error: : [^0]" $mpilog ; then
+       rc=1
+    fi
+    cat $mpilog
+    return $rc
+}
+
diff --git a/lustre/tests/run_IOR.sh b/lustre/tests/run_IOR.sh
new file mode 100755
index 0000000..6da7f54
--- /dev/null
+++ b/lustre/tests/run_IOR.sh
@@ -0,0 +1,65 @@
+#!/bin/bash
+set -x
+
+TMP=${TMP:-/tmp}
+
+TESTSUITELOG=${TESTSUITELOG:-$TMP/recovery-mds-scale}
+LOG=${TESTSUITELOG}_$(basename $0)-$(hostname)
+DEBUGLOG=${LOG}.debug
+
+mkdir -p ${LOG%/*}
+
+rm -f $LOG $DEBUGLOG
+exec 2>$DEBUGLOG
+
+. $(dirname $0)/functions.sh
+
+IOR=${IOR:-"$(which IOR)"}
+
+assert_env MOUNT END_RUN_FILE LOAD_PID_FILE IOR
+
+trap signaled TERM
+
+# if MACHINEFILE set and exists -- use it
+if [ -z $MACHINEFILE ] || [ ! -e $MACHINEFILE ]; then
+    MACHINEFILE=$TMP/$(basename $0)-$(hostname).machines
+    echo $(hostname) >$MACHINEFILE
+fi
+
+THREADS_PER_CLIENT=${THREADS_PER_CLIENT:-3}
+NUM_CLIENTS=$(cat $MACHINEFILE | wc -l)
+
+# recovery-*-scale scripts use this to signal the client loads to die
+echo $$ >$LOAD_PID_FILE
+
+TESTDIR=${TESTDIR:-$MOUNT/d0.ior-$(hostname)}
+
+CONTINUE=true
+while [ ! -e "$END_RUN_FILE" ] && $CONTINUE; do
+    echoerr "$(date +'%F %H:%M:%S'): IOR run starting"
+    mkdir -p $TESTDIR
+    # need this only if TESTDIR is not default
+    chmod -R 777 $TESTDIR
+
+    mpi_run -np $((NUM_CLIENTS * THREADS_PER_CLIENT)) -machinefile ${MACHINEFILE} \
+        $IOR -a POSIX -b 1g -o $TESTDIR/IOR-file -s 1 -t 1m -v -w -r 1>$LOG &
+    load_pid=$!
+    wait $load_pid
+    if [ ${PIPESTATUS[0]} -eq 0 ]; then
+	echoerr "$(date +'%F %H:%M:%S'): IOR succeeded"
+	cd $TMP
+	rm -rf $TESTDIR
+	echoerr "$(date +'%F %H:%M:%S'): IOR run finished"
+    else
+	echoerr "$(date +'%F %H:%M:%S'): IOR failed"
+	if [ -z "$ERRORS_OK" ]; then
+	    echo $(hostname) >> $END_RUN_FILE
+	fi
+	if [ $BREAK_ON_ERROR ]; then
+	    # break
+            CONTINUE=false
+	fi
+    fi
+done
+
+echoerr "$(date +'%F %H:%M:%S'): IOR run exiting"
diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh
index 11fae0e..a1b9f03 100644
--- a/lustre/tests/test-framework.sh
+++ b/lustre/tests/test-framework.sh
@@ -834,10 +834,14 @@ check_progs_installed () {
     shift
     local progs=$@
 
-    do_nodes $clients "set -x ; PATH=:$PATH status=true; for prog in $progs; do
-        which \\\$prog || { echo \\\$prog missing on \\\$(hostname) && status=false; }
-        done;
-        eval \\\$status"
+    do_nodes $clients "set -x ; PATH=:$PATH; status=true;
+for prog in $progs; do
+    if ! [ \\\"\\\$(which \\\$prog)\\\"  -o  \\\"\\\${!prog}\\\" ]; then
+       echo \\\$prog missing on \\\$(hostname);
+       status=false;
+    fi
+done;
+eval \\\$status"
 }
 
 client_var_name() {
@@ -2716,30 +2720,6 @@ get_mds_dir () {
     error "mdt-s : inodes count OLD ${oldused[@]} NEW ${newused[@]}"
 }
 
-mpi_run () {
-    local mpirun="$MPIRUN $MPIRUN_OPTIONS"
-    local command="$mpirun $@"
-    local mpilog=$TMP/mpi.log
-    local rc
-
-    if [ "$MPI_USER" != root -a $mpirun ]; then
-        echo "+ chmod 0777 $MOUNT"
-        chmod 0777 $MOUNT
-        command="su $MPI_USER sh -c \"$command \""
-    fi
-
-    ls -ald $MOUNT
-    echo "+ $command"
-    eval $command 2>&1 > $mpilog || true
-
-    rc=${PIPESTATUS[0]}
-    if [ $rc -eq 0 ] && grep -q "p4_error: : [^0]" $mpilog ; then
-       rc=1
-    fi
-    cat $mpilog
-    return $rc
-}
-
 mdsrate_cleanup () {
     mpi_run -np $1 -machinefile $2 ${MDSRATE} --unlink --nfiles $3 --dir $4 --filefmt $5 $6
 }
-- 
1.8.3.1