b=15255

i=Adilger

config file for several remote clients; replay-single test_70a, test_70b

diff --git a/lustre/tests/Makefile.am b/lustre/tests/Makefile.am
index 9485b64..2e8d5ab 100644 (file)
--- a/lustre/tests/Makefile.am
+++ b/lustre/tests/Makefile.am
@@ -14,7 +14,7 @@ noinst_SCRIPTS += replay-ost-single.sh replay-single.sh run-llog.sh sanityN.sh
 noinst_SCRIPTS += lockorder.sh socketclient socketserver runmultiop_bg_pause
 noinst_SCRIPTS += sanity-sec.sh sanity-gss.sh krb5_login.sh
 nobase_noinst_SCRIPTS = cfg/insanity-local.sh
 noinst_SCRIPTS += lockorder.sh socketclient socketserver runmultiop_bg_pause
 noinst_SCRIPTS += sanity-sec.sh sanity-gss.sh krb5_login.sh
 nobase_noinst_SCRIPTS = cfg/insanity-local.sh

-nobase_noinst_SCRIPTS += cfg/local.sh acl/make-tree acl/run
+nobase_noinst_SCRIPTS += cfg/local.sh acl/make-tree acl/run cfg/ncli.sh

 nobase_noinst_SCRIPTS += rmtacl/make-tree rmtacl/run
 nobase_noinst_DATA = acl/cp.test acl/getfacl-noacl.test acl/inheritance.test
 nobase_noinst_DATA += acl/misc.test acl/permissions.test acl/setfacl.test
 nobase_noinst_SCRIPTS += rmtacl/make-tree rmtacl/run
 nobase_noinst_DATA = acl/cp.test acl/getfacl-noacl.test acl/inheritance.test
 nobase_noinst_DATA += acl/misc.test acl/permissions.test acl/setfacl.test

diff --git a/lustre/tests/cfg/ncli.sh b/lustre/tests/cfg/ncli.sh
new file mode 100644 (file)
index 0000000..2735b9a
--- /dev/null
+++ b/lustre/tests/cfg/ncli.sh
@@ -0,0 +1,17 @@
+. $LUSTRE/tests/cfg/local.sh
+
+CLIENT1=${CLIENT1:-`hostname`}
+SINGLECLIENT=$CLIENT1
+RCLIENTS=${RCLIENTS:-""}
+CLIENTS=`comma_list $SINGLECLIENT $RCLIENTS`
+REMOTECLIENTS=($RCLIENTS)
+for ((i=0; $i<${#REMOTECLIENTS[@]}; i++)); do
+       varname=CLIENT$((i + 2))
+       eval $varname=${REMOTECLIENTS[i]}
+done
+
+CLIENTCOUNT=$((${#REMOTECLIENTS[@]} + 1))
+
+[ -n "$RCLIENTS" -a "$PDSH" = "no_dsh" ] && \
+                error "tests for remote clients $RCLIENTS needs pdsh != do_dsh " || true
+

diff --git a/lustre/tests/replay-single.sh b/lustre/tests/replay-single.sh
index 13ee927..e7e716a 100755 (executable)
--- a/lustre/tests/replay-single.sh
+++ b/lustre/tests/replay-single.sh
@@ -1408,6 +1408,76 @@ test_61c() {
 }
 run_test 61c "test race mds llog sync vs llog cleanup"
 
 }
 run_test 61c "test race mds llog sync vs llog cleanup"
 

+# start multi-client tests
+test_70a () {
+       [ -z "$CLIENTS" ] && \
+               { skip "Need two or more clients." && return; }
+       [ $CLIENTCOUNT -lt 2 ] && \
+               { skip "Need two or more clients, have $CLIENTCOUNT" && return; }
+
+       echo "mount clients $CLIENTS ..."
+       zconf_mount_clients $CLIENTS $DIR
+
+       local clients=${CLIENTS//,/ }
+       echo "Write/read files on $DIR ; clients $CLIENTS ... "
+       for CLIENT in $clients; do
+               do_node $CLIENT dd bs=1M count=10 if=/dev/zero \
+                       of=$DIR/${tfile}_${CLIENT} 2>/dev/null || \
+                               error "dd failed on $CLIENT"
+       done
+
+       local prev_client=$(echo $clients | sed 's/^.* \(\w\+\)$/\1/') 
+       for C in ${CLIENTS//,/ }; do
+               do_node $prev_client dd if=$DIR/${tfile}_${C} of=/dev/null 2>/dev/null || \
+                       error "dd if=$DIR/${tfile}_${C} failed on $prev_client"
+               prev_client=$C
+       done
+       
+       ls $DIR
+
+       zconf_umount_clients $CLIENTS $DIR
+}
+run_test 70a "check multi client t-f"
+
+test_70b () {
+       [ -z "$CLIENTS" ] && \
+               { skip "Need two or more clients." && return; }
+       [ $CLIENTCOUNT -lt 2 ] && \
+               { skip "Need two or more clients, have $CLIENTCOUNT" && return; }
+
+       zconf_mount_clients $CLIENTS $DIR
+       
+       local duration="-t 60"
+       local cmd="rundbench 1 $duration "
+       local PID=""
+       for CLIENT in ${CLIENTS//,/ }; do
+               $PDSH $CLIENT "set -x; PATH=:$PATH:$LUSTRE/utils:$LUSTRE/tests/:${DBENCH_LIB} DBENCH_LIB=${DBENCH_LIB} $cmd" &
+               PID=$!
+               echo $PID >pid.$CLIENT
+               echo "Started load PID=`cat pid.$CLIENT`"
+       done
+
+       replay_barrier $SINGLEMDS 
+       sleep 3 # give clients a time to do operations
+
+       log "$TESTNAME fail mds 1"
+       fail $SINGLEMDS
+
+# wait for client to reconnect to MDS
+       sleep $TIMEOUT
+
+       for CLIENT in ${CLIENTS//,/ }; do
+               PID=`cat pid.$CLIENT`
+               wait $PID
+               rc=$?
+               echo "load on ${CLIENT} returned $rc"
+       done
+
+       zconf_umount_clients $CLIENTS $DIR 
+}
+run_test 70b "mds recovery; $CLIENTCOUNT clients"
+# end multi-client tests
+

 equals_msg `basename $0`: test complete, cleaning up
 check_and_cleanup_lustre
 [ -f "$TESTSUITELOG" ] && cat $TESTSUITELOG || true
 equals_msg `basename $0`: test complete, cleaning up
 check_and_cleanup_lustre
 [ -f "$TESTSUITELOG" ] && cat $TESTSUITELOG || true

diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh
index 8a4f0f3..8ffc340 100644 (file)
--- a/lustre/tests/test-framework.sh
+++ b/lustre/tests/test-framework.sh
@@ -469,7 +469,7 @@ zconf_mount() {
         exit 1
     fi
 
         exit 1
     fi
 

-    echo "Starting client: $OPTIONS $device $mnt"
+    echo "Starting client: $client: $OPTIONS $device $mnt"

     do_node $client mkdir -p $mnt
     do_node $client mount -t lustre $OPTIONS $device $mnt || return 1
 
     do_node $client mkdir -p $mnt
     do_node $client mount -t lustre $OPTIONS $device $mnt || return 1
 


@@ -481,17 +481,40 @@ zconf_mount() {
 }
 
 zconf_umount() {
 }
 
 zconf_umount() {

-    client=$1
-    mnt=$2
+    local client=$1
+    local mnt=$2

     [ "$3" ] && force=-f
     local running=$(do_node $client "grep -c $mnt' ' /proc/mounts") || true
     if [ $running -ne 0 ]; then
     [ "$3" ] && force=-f
     local running=$(do_node $client "grep -c $mnt' ' /proc/mounts") || true
     if [ $running -ne 0 ]; then

-        echo "Stopping client $mnt (opts:$force)"
+        echo "Stopping client $client $mnt (opts:$force)"

         lsof | grep "$mnt" || true
         do_node $client umount $force $mnt
     fi
 }
 
         lsof | grep "$mnt" || true
         do_node $client umount $force $mnt
     fi
 }
 

+zconf_mount_clients() {
+    local clients=$1
+    local mnt=$2
+
+    echo "Mounting clients: $clients"
+    local client
+    for client in ${clients//,/ }; do
+        zconf_mount $client $mnt  || true
+    done
+}
+
+zconf_umount_clients() {
+    local clients=$1
+    local mnt=$2
+    [ "$3" ] && force=-f
+
+    echo "Umounting clients: $clients"
+    local client
+    for client in ${clients//,/ }; do
+        zconf_umount $client $mnt $force || true
+    done
+}
+

 shutdown_facet() {
     facet=$1
     if [ "$FAILURE_MODE" = HARD ]; then
 shutdown_facet() {
     facet=$1
     if [ "$FAILURE_MODE" = HARD ]; then


@@ -613,6 +636,8 @@ client_df() {
     # not every config has many clients
     if [ ! -z "$CLIENTS" ]; then
         $PDSH $CLIENTS "df $MOUNT" > /dev/null
     # not every config has many clients
     if [ ! -z "$CLIENTS" ]; then
         $PDSH $CLIENTS "df $MOUNT" > /dev/null

+    else
+       df $MOUNT > /dev/null

     fi
 }
 
     fi
 }
 


@@ -884,6 +909,12 @@ stopall() {
     # assume client mount is local
     grep " $MOUNT " /proc/mounts && zconf_umount $HOSTNAME $MOUNT $*
     grep " $MOUNT2 " /proc/mounts && zconf_umount $HOSTNAME $MOUNT2 $*
     # assume client mount is local
     grep " $MOUNT " /proc/mounts && zconf_umount $HOSTNAME $MOUNT $*
     grep " $MOUNT2 " /proc/mounts && zconf_umount $HOSTNAME $MOUNT2 $*

+
+    if [ -n "$CLIENTS" ]; then
+            zconf_umount_clients $CLIENTS $MOUNT "$*" || true
+            zconf_umount_clients $CLIENTS $MOUNT2 "$*" || true
+    fi
+

     [ "$CLIENTONLY" ] && return
     for num in `seq $MDSCOUNT`; do
         stop mds$num -f
     [ "$CLIENTONLY" ] && return
     for num in `seq $MDSCOUNT`; do
         stop mds$num -f


@@ -1016,8 +1047,11 @@ setupall() {
     fi
     [ "$DAEMONFILE" ] && $LCTL debug_daemon start $DAEMONFILE $DAEMONSIZE
     mount_client $MOUNT
     fi
     [ "$DAEMONFILE" ] && $LCTL debug_daemon start $DAEMONFILE $DAEMONSIZE
     mount_client $MOUNT

+    [ -n "$CLIENTS" ] && zconf_mount_clients $CLIENTS $MOUNT
+

     if [ "$MOUNT_2" ]; then
        mount_client $MOUNT2
     if [ "$MOUNT_2" ]; then
        mount_client $MOUNT2

+        [ -n "$CLIENTS" ] && zconf_mount_clients $CLIENTS $MOUNT2

     fi
 
     # by remounting mdt before ost, initial connect from mdt to ost might
     fi
 
     # by remounting mdt before ost, initial connect from mdt to ost might


@@ -1577,6 +1611,9 @@ nodes_list () {
     local myNODES=$HOSTNAME
     local myNODES_sort
 
     local myNODES=$HOSTNAME
     local myNODES_sort
 

+    # CLIENTS (if specified) contains the local client
+    [ -n "$CLIENTS" ] && myNODES=${CLIENTS//,/ }
+

     if [ "$PDSH" -a "$PDSH" != "no_dsh" ]; then
         myNODES="$myNODES $(osts_nodes) $(mdts_nodes)"
     fi
     if [ "$PDSH" -a "$PDSH" != "no_dsh" ]; then
         myNODES="$myNODES $(osts_nodes) $(mdts_nodes)"
     fi