r=2152

replay-single.sh now supports multiple nodes. The script must be run
on the client, but the mds, mdsfailover, and ost can all be different
nodes. If mdsfailover_HOST is set, then the MDS service  will be
failed between the two mds nodes.

uml1# PDSH="pdsh -w" mds_HOST=uml2 ost_HOST=uml3 mdsfailover_HOST=uml4
  ./replay-single.sh

It still runs on a single node as before.

diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh
index 8a11555..6a1c7ed 100644 (file)
--- a/lustre/tests/test-framework.sh
+++ b/lustre/tests/test-framework.sh
@@ -2,46 +2,65 @@
 
 set -e
 
+# eg, assert_env LUSTRE MDSNODES OSTNODES CLIENTS
+assert_env() {
+    local failed=""
+    for name in $@; do
+      if [ -z "${!name}" ]; then
+         echo "$0: $name must be set"
+          failed=1
+      fi
+    done
+    [ $failed ] && exit 1 || true
+}
+
 init_test_env() {
     export TESTSUITE=`basename $0 .sh`
     export XMLCONFIG="${TESTSUITE}.xml"
     export LTESTDIR=${LTESTDIR:-$LUSTRE/../ltest}
-    export PATH=$LUSTRE/utils:$LUSTRE/tests:$PATH
 
+    [ -d /r ] && export ROOT=/r
     export RLUSTRE=${RLUSTRE:-$LUSTRE}
     export RPWD=${RPWD:-$PWD}
+    export PATH=$RLUSTRE/utils:$RLUSTRE/tests:$PATH
+    
     export CHECKSTAT="${CHECKSTAT:-checkstat} -v"
 }
 
 start() {
     facet=$1
     shift
-    lconf --node ${facet}_facet $@ $XMLCONFIG
+    active=`facet_active $facet`
+    do_facet $facet lconf --select ${facet}1=${active}_facet --node ${active}_facet $@ $XMLCONFIG
 }
 
 stop() {
     facet=$1
+    active=`facet_active $facet`
     shift
-    lconf --node ${facet}_facet $@ --cleanup $XMLCONFIG
+    do_facet $facet lconf --select ${facet}1=${active}_facet --node ${active}_facet $@ --cleanup $XMLCONFIG
 }
 
 replay_barrier() {
-    local dev=$1
-    sync
+    local facet=$1
+    do_facet $facet sync
     df $MOUNT
-    lctl --device %${dev}1 readonly
-    lctl --device %${dev}1 notransno
+    do_facet $facet lctl --device %${facet}1 readonly
+    do_facet $facet lctl --device %${facet}1 notransno
+    do_facet $facet lctl mark "REPLAY BARRIER"
     lctl mark "REPLAY BARRIER"
 }
 
 mds_evict_client() {
-    cat /proc/fs/lustre/mdc/*_MNT_*/uuid > /proc/fs/lustre/mds/mds1/evict_client
+    UUID= `cat /proc/fs/lustre/mdc/*_MNT_*/uuid`
+    do_facet mds "echo $UUID > /proc/fs/lustre/mds/mds1/evict_client"
 }
 
 fail() {
     local facet=$1
     stop $facet --force --failover --nomod
-    start $facet --nomod
+    change_active $facet
+    start $facet
     df $MOUNT || error "post-failover df: $?"
 }
 
@@ -49,11 +68,79 @@ do_lmc() {
     lmc -m ${XMLCONFIG} $@
 }
 
+h2tcp() {
+    echo $1
+}
+declare -fx h2tcp
+
+h2elan() {
+    echo $1 | sed "s/[^0-9]*//"
+}
+declare -fx h2elan
+
+facet_host() {
+   local facet=$1
+   varname=${facet}_HOST
+   echo -n ${!varname}
+}
+
+facet_nid() {
+   facet=$1
+   HOST=`facet_host $facet`
+   if [ -z "$HOST" ]; then
+       echo "The env variable ${facet}_HOST must be set."
+       exit 1
+   fi
+   echo `h2$NETWORKTYPE $HOST`
+}
+
+no_dsh() {
+   shift
+   eval $@
+}
+
+facet_active() {
+    local facet=$1
+    local activevar=${facet}active
+    active=${!activevar}
+    if [ -z "$active" ] ; then 
+       echo -n ${facet}
+    else
+       echo -n ${active}
+    fi
+}
+
+change_active() {
+    local facet=$1
+    failover=${facet}failover 
+    host=`facet_host $failover`
+    [ -z "$host" ] && return
+    curactive=`facet_active $facet`
+    if [ -z "${curactive}" -o "$curactive" == "$failover" ] ; then
+        eval export ${facet}active=$facet
+    else
+        eval export ${facet}active=$failover
+    fi
+    # save the active host for this facet
+    activevar=${facet}active
+    echo "$activevar=${!activevar}" > ./$activevar
+}
+
+do_facet() {
+    facet=$1
+    shift
+    active=`facet_active $facet`
+    HOST=`facet_host $active`
+    $PDSH $HOST "(PATH=\$PATH:$LUSTRE/utils:$LUSTRE/tests; cd $PWD; sh -c \"$@\")"
+}
+
 add_facet() {
     local facet=$1
     shift
+    echo "add facet $facet: `facet_host $facet`"
     do_lmc --add node --node ${facet}_facet $@ --timeout $TIMEOUT
-    do_lmc --add net --node ${facet}_facet --nid localhost --nettype tcp
+    do_lmc --add net --node ${facet}_facet --nid `facet_nid $facet` \
+       --nettype $NETWORKTYPE
 }
 
 error() {
@@ -113,7 +200,8 @@ EQUALS="======================================================================"
 equals_msg() {
    msg="$@"
 
-   local suffixlen=$((65 - ${#msg}))
+   local suffixlen=$((${#EQUALS} - ${#msg}))
+   [ $suffixlen -lt 5 ] && suffixlen=5
    printf '===== %s %.*s\n' "$msg" $suffixlen $EQUALS
 }