From d2d1432f140be4766a9f522147d722d2b885850b Mon Sep 17 00:00:00 2001 From: rread Date: Thu, 20 Nov 2003 07:43:00 +0000 Subject: [PATCH] - Integrate insanity.sh with test-framework - Centralize the config data to files in tests/cfg/ - Different config file can be used by setting CONFIG=file or using command line option: --config - FAILURE_MODE option for insanity.sh: FAILURE_MODE=SOFT - shutdown services with lconf --force FAILURE_MODE=HARD - power down the service nodes using $POWER_DOWN and $POWER_UP to reboot. This feature should be integrated with the other tests. - replay-single.sh, replay-dual.sh, and conf-sanity.sh all pass on hardware - replay-ost-dual.sh and insanity.sh don't pass yet --- lustre/tests/cfg/insanity-local.sh | 30 +++ lustre/tests/cfg/local.sh | 33 +++ lustre/tests/conf-sanity.sh | 147 +++++------ lustre/tests/insanity.sh | 494 ++++++++++++++++------------------- lustre/tests/replay-dual.sh | 76 ++---- lustre/tests/replay-single-upcall.sh | 27 +- lustre/tests/replay-single.sh | 52 +--- 7 files changed, 414 insertions(+), 445 deletions(-) create mode 100644 lustre/tests/cfg/insanity-local.sh create mode 100644 lustre/tests/cfg/local.sh diff --git a/lustre/tests/cfg/insanity-local.sh b/lustre/tests/cfg/insanity-local.sh new file mode 100644 index 0000000..4ce742f --- /dev/null +++ b/lustre/tests/cfg/insanity-local.sh @@ -0,0 +1,30 @@ +mds_HOST=${mds_HOST:-`hostname`} +mdsfailover_HOST=${mdsfailover_HOST:-""} +ost1_HOST=${ost1_HOST:-"`hostname`"} +ost2_HOST=${ost2_HOST:-"`hostname`"} +client_HOST="'*'" +LIVE_CLIENT=${LIVE_CLIENT:-"`hostname`"} +# This should always be a list, not a regexp +FAIL_CLIENTS=${FAIL_CLIENTS:-""} + +NETTYPE=${NETTYPE:-tcp} +TIMEOUT=${TIMEOUT:-30} +PTLDEBUG=${PTLDEBUG:-'"ha|info|ioctl|malloc"'} +MOUNT=${MOUNT:-"/mnt/lustre"} +#CLIENT_UPCALL=${CLIENT_UPCALL:-`pwd`/client-upcall-mdev.sh} +UPCALL=${CLIENT_UPCALL:-`pwd`/replay-single-upcall.sh} + +MDSDEV=${MDSDEV:-$ROOT/tmp/mds-`hostname`} +MDSSIZE=${MDSSIZE:-10000} #50000000 + +OSTDEV=${OSTDEV:-$ROOT/tmp/ost-`hostname`} +OSTSIZE=${OSTSIZE:=10000} #50000000 +FSTYPE=${FSTYPE:-ext3} +STRIPE_BYTES=${STRIPE_BYTES:-65536} #1048576 +STRIPES_PER_OBJ=${STRIPES_PER_OBJ:-0} + +FAILURE_MODE=${FAILURE_MODE:-SOFT} # or HARD +POWER_DOWN=${POWER_DOWN:-"powerman --off"} +POWER_UP=${POWER_UP:-"powerman --on"} + +PDSH=no_dsh diff --git a/lustre/tests/cfg/local.sh b/lustre/tests/cfg/local.sh new file mode 100644 index 0000000..5dad6f5 --- /dev/null +++ b/lustre/tests/cfg/local.sh @@ -0,0 +1,33 @@ +# oldstyle +MDSNODE=${MDSNODE:-`hostname`} +OSTNODE=${OSTNODE:-`hostname`} + +mds_HOST=${mds_HOST:-$MDSNODE} +mdsfailover_HOST=${mdsfailover_HOST} +ost_HOST=${ost_HOST:-$OSTNODE} +ost2_HOST=${ost2_HOST:-$ost_HOST} +client_HOST=${client_HOST:-"'*'"} +NETTYPE=${NETTYPE:-tcp} + +MOUNT=${MOUNT:-"/mnt/lustre"} +MOUNT1=${MOUNT1:-$MOUNT} +MOUNT2=${MOUNT2:-"/mnt/lustre2"} +DIR=${DIR:-$MOUNT} +DIR2=${DIR2:-$MOUNT1} + +PDSH=${PDSH:-no_dsh} + +MDSDEV=${MDSDEV:-$ROOT/tmp/mds-`hostname`} +MDSSIZE=${MDSSIZE:-10000} +OSTDEV=${OSTDEV:-$ROOT/tmp/ost-`hostname`} +OSTSIZE=${OSTSIZE:-10000} +FSTYPE=${FSTYPE:-ext3} +TIMEOUT=${TIMEOUT:-10} +UPCALL=${UPCALL:-$PWD/replay-single-upcall.sh} + +STRIPE_BYTES=${STRIPE_BYTES:-65536} +STRIPES_PER_OBJ=${STRIPES_PER_OBJ:-0} + +FAILURE_MODE=${FAILURE_MODE:-SOFT} # or HARD +POWER_DOWN=${POWER_DOWN:-"powerman --off"} +POWER_UP=${POWER_UP:-"powerman --on"} diff --git a/lustre/tests/conf-sanity.sh b/lustre/tests/conf-sanity.sh index 79abdd2..f71e39e 100644 --- a/lustre/tests/conf-sanity.sh +++ b/lustre/tests/conf-sanity.sh @@ -9,102 +9,69 @@ PATH=$PWD/$SRCDIR:$SRCDIR:$SRCDIR/../utils:$PATH LUSTRE=${LUSTRE:-`dirname $0`/..} RLUSTRE=${RLUSTRE:-$LUSTRE} -ONLY=${ONLY:-"$*"} . $LUSTRE/tests/test-framework.sh -init_test_env - -mds_HOST=${mds_HOST:-`hostname`} -mdsfailover_HOST=${mdsfailover_HOST} -ost_HOST=${ost_HOST:-`hostname`} -client_HOST=${client_HOST:-`hostname`} -NETTYPE=${NETTYPE:-tcp} - -MOUNT=${MOUNT:-"/mnt/lustre"} -MOUNT2=${MOUNT2:-"/mnt/lustre2"} -DIR=${DIR:-$MOUNT} -DIR2=${DIR2:-$MOUNT2} -PDSH=${PDSH:-'pdsh -S -w'} -MDSDEV=${MDSDEV:-/tmp/mds-`hostname`} -MDSSIZE=${MDSSIZE:-10000} -OSTDEV=${OSTDEV:-/tmp/ost-`hostname`} -OSTSIZE=${OSTSIZE:-10000} -FSTYPE=${FSTYPE:-ext3} -TIMEOUT=${TIMEOUT:-5} -CONFIG=${CONFIG:-"$XMLCONFIG"} -LCONF=${LCONF:-"lconf"} -LCTL=${LCTL:-"lctl"} -FORCE=${FORCE:-" --force"} -DAEMONFILE=${DAEMONFILE:-"/r/tmp/debug-daemon"} -DAEMONSIZE=${DAEMONSIZE:-"40"} +init_test_env $@ -STRIPE_BYTES=65536 -STRIPES_PER_OBJ=0 +. ${CONFIG:=$LUSTRE/tests/cfg/local.sh} -do_command() { - local node=$1 - shift - $PDSH $node "PATH=\$PATH:$RLUSTRE/utils:$RLUSTRE/tests; cd $RPWD; $@" -} +FORCE=${FORCE:-" --force"} gen_config() { rm -f $XMLCONFIG - add_facet mds - add_facet ost - add_facet client - do_lmc --add mds --node mds_facet --mds mds1 --dev $MDSDEV --size $MDSSIZE - do_lmc --add lov --mds mds1 --lov lov1 --stripe_sz $STRIPE_BYTES --stripe_cnt $STRIPES_PER_OBJ --stripe_pattern 0 - do_lmc --add ost --lov lov1 --node ost_facet --ost ost1 --dev $OSTDEV --size $OSTSIZE - do_lmc --add mtpt --node client_facet --path $MOUNT --mds mds1 --lov lov1 + add_mds mds --dev $MDSDEV --size $MDSSIZE + add_lov lov1 mds --stripe_sz $STRIPE_BYTES\ + --stripe_cnt $STRIPES_PER_OBJ --stripe_pattern 0 + add_ost ost --lov lov1 --dev $OSTDEV --size $OSTSIZE + add_client client mds --lov lov1 --path $MOUNT } gen_second_config() { rm -f $XMLCONFIG - add_facet mds - add_facet ost - add_facet client - do_lmc --add mds --node mds_facet --mds mds1 --dev $MDSDEV --size $MDSSIZE - do_lmc --add lov --mds mds1 --lov lov2 --stripe_sz $STRIPE_BYTES --stripe_cnt $STRIPES_PER_OBJ --stripe_pattern 0 - do_lmc --add ost --lov lov2 --node ost_facet --ost ost1 --dev $OSTDEV --size $OSTSIZE - do_lmc --add mtpt --node client_facet --path $MOUNT2 --mds mds1 --lov lov2 + add_mds mds2 --dev $MDSDEV --size $MDSSIZE + add_lov lov2 mds2 --stripe_sz $STRIPE_BYTES\ + --stripe_cnt $STRIPES_PER_OBJ --stripe_pattern 0 + add_ost ost2 --lov lov2 --dev $OSTDEV --size $OSTSIZE + add_client client mds2 --lov lov2 --path $MOUNT2 } + start_mds() { - echo "start mds service on ${mds_HOST}...." - start mds --reformat $MDSLCONFARGS > /dev/null || exit 94 + echo "start mds service on `facet_active_host mds`" + start mds --reformat $MDSLCONFARGS > /dev/null || return 94 } stop_mds() { - echo "stop mds service on ${mds_HOST}...." - stop mds $@ > /dev/null || exit 97 + echo "stop mds service on `facet_active_host mds`" + stop mds $@ > /dev/null || return 97 } start_ost() { - echo "start ost service on ${mds_HOST}...." - start ost --reformat $OSTLCONFARGS > /dev/null || exit 95 + echo "start ost service on `facet_active_host ost`" + start ost --reformat $OSTLCONFARGS > /dev/null || return 95 } stop_ost() { - echo "stop ost service on ${mds_HOST}...." - stop ost $@ > /dev/null || exit 98 + echo "stop ost service on `facet_active_host ost`" + stop ost $@ > /dev/null || return 98 } mount_client() { local MOUNTPATH=$1 echo "mount lustre on ${MOUNTPATH}....." - zconf_mount $MOUNTPATH > /dev/null || exit 96 + zconf_mount $MOUNTPATH > /dev/null || return 96 } umount_client() { local MOUNTPATH=$1 echo "umount lustre on ${MOUNTPATH}....." - zconf_umount $MOUNTPATH > /dev/null || exit 97 + zconf_umount $MOUNTPATH > /dev/null || return 97 } manual_umount_client(){ echo "manual umount lustre on ${MOUNTPATH}...." - do_command $client_HOST "umount $MOUNT" + do_facet client "umount $MOUNT" } setup() { @@ -114,22 +81,22 @@ setup() { } cleanup() { - umount_client $MOUNT - stop_mds - stop_ost + umount_client $MOUNT || return -200 + stop_mds || return -201 + stop_ost || return -202 } check_mount() { - do_command $client_HOST "touch $DIR/a" || exit 71 - do_command $client_HOST "rm $DIR/a" || exit 72 + do_facet client "touch $DIR/a" || return 71 + do_facet client "rm $DIR/a" || return 72 echo "setup single mount lustre success" } check_mount2() { - do_command $client_HOST "touch $DIR/a" || exit 71 - do_command $client_HOST "rm $DIR/a" || exit 72 - do_command $client_HOST "touch $DIR2/a" || exit 73 - do_command $client_HOST "rm $DIR2/a" || exit 74 + do_facet client "touch $DIR/a" || return 71 + do_facet client "rm $DIR/a" || return 72 + do_facet client "touch $DIR2/a" || return 73 + do_facet client "rm $DIR2/a" || return 74 echo "setup double mount lustre success" } @@ -144,7 +111,7 @@ test_0() { start_ost start_mds mount_client $MOUNT - check_mount || exit 41 + check_mount || return 41 cleanup } run_test 0 "single mount setup" @@ -155,7 +122,7 @@ test_1() { start ost --reformat $OSTLCONFARGS > /dev/null start_mds mount_client $MOUNT - check_mount || exit 42 + check_mount || return 42 cleanup } run_test 1 "start up ost twice" @@ -167,46 +134,57 @@ test_2() { start mds --reformat $MDSLCONFARGS > /dev/null mount_client $MOUNT - check_mount || exit 43 + check_mount || return 43 cleanup } run_test 2 "start up mds twice" test_3() { - start_ost - start_mds - - mount_client $MOUNT + setup mount_client $MOUNT - check_mount || exit 44 + check_mount || return 44 + umount_client $MOUNT cleanup } run_test 3 "mount client twice" test_4() { setup - touch $DIR/a || exit 85 + touch $DIR/$tfile || return 85 stop_ost ${FORCE} - cleanup + # cleanup may return an error from the failed + # disconnects; for now I'll consider this successful + # if all the modules have unloaded. + if ! cleanup ; then + lsmod | grep -q portals && return 1 + fi + return 0 } run_test 4 "force cleanup ost, then cleanup" test_5() { setup - touch $DIR/a || exit 86 - stop_mds ${FORCE} || exit 98 - cleanup + touch $DIR/$tfile || return 86 + stop_mds ${FORCE} || return 98 + + # cleanup may return an error from the failed + # disconnects; for now I'll consider this successful + # if all the modules have unloaded. + if ! cleanup ; then + lsmod | grep -q portals && return 1 + fi + return 0 } run_test 5 "force cleanup mds, then cleanup" test_6() { setup manual_umount_client - mount_client ${MOUNT} || exit 87 - touch $DIR/a || exit 86 + mount_client ${MOUNT} || return 87 + touch $DIR/a || return 86 cleanup } run_test 6 "manual umount, then mount again" @@ -225,15 +203,14 @@ test_8() { mount_client $MOUNT mount_client $MOUNT2 - check_mount2 || exit 45 + check_mount2 || return 45 umount $MOUNT umount_client $MOUNT2 stop_mds stop_ost - } run_test 8 "double mount setup" -gen_config +equals_msg "Done" diff --git a/lustre/tests/insanity.sh b/lustre/tests/insanity.sh index 55fbece..c2c030f 100755 --- a/lustre/tests/insanity.sh +++ b/lustre/tests/insanity.sh @@ -3,160 +3,151 @@ set -e -# command line -set - - `getopt -o r -l ostdev:,mdsdev:,stripecnt:,reformat -- $*` - -for i in $*; do - case $i in - --ostdev) OSTDEV=$2; shift;; - --mdsdev) MDSDEV=$2; shift;; - --stripecnt) STRIPECNT=$2; shift;; - -r|--reformat) REFORMAT=--reformat;; - esac - shift -done - LUSTRE=${LUSTRE:-`dirname $0`/..} . $LUSTRE/tests/test-framework.sh -init_test_env +init_test_env $@ -# . ./config-uml.sh +. ${CONFIG:=$LUSTRE/tests/cfg/insanity-local.sh} -ALWAYS_EXCEPT="3 4 5 6 7 8 9 10" +ALWAYS_EXCEPT="7 8 9 10" build_test_filter -assert_env OSTDEV MDSDEV STRIPECNT +assert_env mds_HOST ost1_HOST ost2_HOST client_HOST LIVE_CLIENT -MDS1=${MDS1:-"mdev4"} -MDS2=${MDS2:-"mdev5"} -OST1=${OST1:-"mdev2"} -OST2=${OST2:-"mdev3"} -LIVE_CLIENT=${LIVE_CLIENT:-"mdev6"} -CLIENT1=${CLIENT1:-"mdev7"} -CLIENT2=${CLIENT2:-"mdev8"} -CLIENT3=${CLIENT3:-"mdev9"} -CLIENT4=${CLIENT4:-"mdev11"} -CLIENTS=${CLIENTS:-"mdev[6-9,11]"} +# This can be a regexp, to allow more clients +CLIENTS=${CLIENTS:-"`comma_list $LIVE_CLIENT $FAIL_CLIENTS`"} -SERVERS="$MDS1 $MDS2 $OST1 $OST2" -CLIENTLIST="$LIVE_CLIENT $CLIENT1 $CLIENT2 $CLIENT3 $CLIENT4" +CLIENTLIST="$LIVE_CLIENT $FAIL_CLIENTS" -TIMEOUT=${TIMEOUT:-30} -PTLDEBUG=${PTLDEBUG:-0} -MOUNTPT=${MOUNTPT:-"/mnt/lustre"} -CLIENT_UPCALL=${CLIENT_UPCALL:-`pwd`/client-upcall-mdev.sh} +DIR=${DIR:-$MOUNT} +##### +# fail clients round robin -gen_config() { - rm -f $XMLCONFIG - # Add all nodes - for node in $SERVERS; do - ${LMC} -m $XMLCONFIG --add net --node $node --timeout $TIMEOUT\ - --nid `h2$NETTYPE $node` --nettype $NETTYPE - done +# list of failable clients +FAIL_LIST=($FAIL_CLIENTS) +FAIL_NUM=${#FAIL_LIST[*]} +FAIL_NEXT=0 +DOWN_NUM=0 # number of nodes currently down - ${LMC} -m $XMLCONFIG --add net --node client \ - --timeout $TIMEOUT --upcall $CLIENT_UPCALL \ - --nid '*' --nettype $NETTYPE +# return next client to fail +fail_client() { + ret=${FAIL_LIST[$FAIL_NEXT]} + FAIL_NEXT=$(( (FAIL_NEXT+1) % FAIL_NUM )) + echo $ret +} - # Configure MDS nodes - for node in $MDS1 $MDS2; do - $LMC -m $XMLCONFIG --node $node --add mds --mds mds_service \ - --fstype ext3 --dev $MDSDEV --size ${MDSSIZE:=50000000} +fail_clients() { + num=$1 + if [ -z "$num" -o $num -gt $((FAIL_NUM - DOWN_NUM)) ]; then + num=$((FAIL_NUM - DOWN_NUM)) + fi + + if [ -z "$num" -o $num -le 0 ]; then + return + fi + + for i in `seq $num`; do + client=`fail_client` + DOWN_CLIENTS="$DOWN_CLIENTS $client" + client_mkdirs + shutdown_client $client done - $LMC -m $XMLCONFIG --add lov --lov lov1 --mds mds_service \ - --stripe_sz 1048576 --stripe_cnt $STRIPECNT --stripe_pattern 0 + for client in $DOWN_CLIENTS; do + restart_node $client + done + DOWN_NUM=`echo $DOWN_CLIENTS | wc -w` + $PDSH $LIVE_CLIENT "cd $MOUNT && rmdir $CLIENTLIST" +} - # Configure ost - for node in $OST1 $OST2; do - $LMC -m $XMLCONFIG --add ost --node $node --lov lov1 \ - --fstype ext3 --dev $OSTDEV --size ${OSTSIZE:=50000000} +reintegrate_clients() { + for client in $DOWN_CLIENTS; do + wait_for $client + $PDSH $client "$LCONF --node client --select mds_svc=`facet_active mds` $CLIENTOPTS $XMLCONFIG" done + DOWN_CLIENTS="" + DOWN_NUM=0 +} + +gen_config() { + rm -f $XMLCONFIG + add_mds mds --dev $MDSDEV --size $MDSSIZE + + if [ ! -z "$mdsfailover_HOST" ]; then + add_mdsfailover mds --dev $MDSDEV --size $MDSSIZE + fi - $LMC -m $XMLCONFIG --node client --add mtpt --path $MOUNTPT \ - --mds mds_service --lov lov1 + add_lov lov1 mds --stripe_sz $STRIPE_BYTES\ + --stripe_cnt $STRIPES_PER_OBJ --stripe_pattern 0 + add_ost ost1 --lov lov1 --dev $OSTDEV --size $OSTSIZE + add_ost ost2 --lov lov1 --dev ${OSTDEV}-2 --size $OSTSIZE + add_client client mds --lov lov1 --path $MOUNT } setup() { - echo "$MDS1" > CURRENT_MDS - ACTIVEMDS=$MDS1 - set -xv - $DSH $OST1,$OST2 "$LCONF --ptldebug $PTLDEBUG $REFORMAT $XMLCONFIG" - $DSH $MDS1 "$LCONF --select mds_service=$ACTIVEMDS --ptldebug $PTLDEBUG $REFORMAT $XMLCONFIG" - $DSH $CLIENTS "$LCONF --node client --ptldebug $PTLDEBUG --select mds_service=$ACTIVEMDS $CLIENTOPTS $XMLCONFIG" - set -+v + start ost1 ${REFORMAT} $OSTLCONFARGS + start ost2 ${REFORMAT} $OSTLCONFARGS + [ "$DAEMONFILE" ] && $LCTL debug_daemon start $DAEMONFILE $DAEMONSIZE + start mds $MDSLCONFARGS ${REFORMAT} + for node in $CLIENTS; do + $PDSH $node lconf --node client_facet --ptldebug $PTLDEBUG --select mds_service=$ACTIVEMDS $XMLCONFIG + done } cleanup() { - ACTIVEMDS=`cat CURRENT_MDS` - $DSH $CLIENTS "$LCONF --cleanup --node client --select mds_service=$ACTIVEMDS $XMLCONFIG" - $DSH $ACTIVEMDS "$LCONF --cleanup --select mds_service=$ACTIVE_MDS $XMLCONFIG" - $DSH $OST1,$OST2 "$LCONF --cleanup $XMLCONFIG" + # make sure we are using the primary MDS, so the config log will + # be able to clean up properly. + activemds=`facet_active mds` + if [ $activemds != "mds" ]; then + fail mds + fi + for node in $CLIENTS; do + $PDSH $node lconf ${FORCE} --cleanup --node client_facet $XMLCONFIG + done + + stop mds ${FORCE} $MDSLCONFARGS + stop ost1 ${FORCE} + stop ost2 ${FORCE} --dump cleanup.log } wait_for() { - NODE=$1 - set +x - check_network $NODE 900 - while ! $DSH2 $NODE "ls -ld $LUSTRE"; do sleep 5; done - set -x + facet=$1 + check_network `facet_active_host $facet` 900 + while ! $PDSH $NODE "ls -ld $LUSTRE"; do sleep 5; done } client_df() { - $DSH2 $CLIENTS "df $MOUNTPT" | dshbak -c + $PDSH $CLIENTS "df $MOUNT" | dshbak -c } trap exit INT client_mkdirs() { - $DSH2 $CLIENTS "mkdir $MOUNTPT/\`hostname\`; ls $MOUNTPT/\`hostname\` > /dev/null" + $PDSH $CLIENTLIST "mkdir $MOUNT/\`hostname\`; ls $MOUNT/\`hostname\` > /dev/null" } -fail_mds() { - FROM=$1 - TO=$2 - echo "Failing MDS $FROM" - rm CURRENT_MDS - shutdown_node $FROM +facet_failover() { + facet=$1 + echo "Failing $facet node `facet_active_host $facet`" + shutdown_facet $facet sleep 2 + reboot_facet $facet client_df & DFPID=$! - echo "Failover MDS is $TO" - wait_for $TO - $DSH2 $TO "$LCONF --select mds_service=$TO $XMLCONFIG" - - #pdsh -w $MDS1 nodeup - #cd /usr/local/admin/lustre-alc/cfg - #../bin/mount-lustre-shaver.sh -f alc-ga1-ddn-llp-2mds.cfg -s mds -m alc2 -c alc36 - #cd /home/jodorizz - - restart_node $FROM - echo "$TO" > CURRENT_MDS + change_active $facet + TO=`facet_active_host $facet` + echo "Failover MDS to $TO" + wait_for $facet + start $facet } -fail_clients() { - client_mkdirs - shutdown_node $CLIENT1 - shutdown_node $CLIENT2 - restart_node $CLIENT1 - restart_node $CLIENT2 - $DSH2 $LIVE_CLIENT "cd $MOUNTPT && rmdir $CLIENTLIST" -} - -reintegrate_clients() { - ACTIVEMDS=`cat CURRENT_MDS` - wait_for $CLIENT1 - wait_for $CLIENT2 - $DSH2 $CLIENT1 "$LCONF --node client --select mds_service=$ACTIVEMDS $CLIENTOPTS $XMLCONFIG" - $DSH2 $CLIENT2 "$LCONF --node client --select mds_service=$ACTIVEMDS $CLIENTOPTS $XMLCONFIG" -} clients_recover_osts() { - OST=$1 - $DSH2 $CLIENTS "$LCTL "'--device %OSC_`hostname`_OST_'"${OST}_MNT_client recover" + facet=$1 + $PDSH $CLIENTS "$LCTL "'--device %OSC_`hostname`_OST_'"${facet}_svc_MNT_client recover" } if [ "$ONLY" == "cleanup" ]; then @@ -174,6 +165,37 @@ fi # 9 Different Failure Modes Combinations echo "Starting Test 17 at `date`" +test_0() { + echo "Failover MDS" + shutdown_facet mds + reboot_facet mds + # prepare for MDS failover + change_active mds + reboot_facet mds + start mds + client_df + + echo "Failing OST1" + shutdown_facet ost1 + reboot_facet ost1 + wait_for ost1 + start ost1 + client_df + # should force the MDS to reconnect to the ost + echo "ost1 recover" >> $DIR/$tfile + + echo "Failing OST2" + shutdown_facet ost2 + reboot_facet ost2 + wait_for ost2 + start ost2 + client_df + # should force the MDS to reconnect to the ost + echo "ost2 recover" >> $DIR/${tfile}2 + +} +run_test 0 "Fail all nodes, independently" + ############### First Failure Mode ############### test_1() { echo "Don't do a MDS - MDS Failure Case" @@ -185,36 +207,36 @@ run_test 1 "MDS/MDS failure" ############### Second Failure Mode ############### test_2() { - echo "Verify Lustre filesystem is up and running" client_df echo "Failing MDS" - rm CURRENT_MDS - shutdown_node $MDS1 - restart_node $MDS2 + shutdown_facet mds + reboot_facet mds + + # prepare for MDS failover + change_active mds + reboot_facet mds client_df & DFPID=$! sleep 5 echo "Failing OST" - shutdown_node $OST1 + shutdown_facet ost1 echo "Reintegrating OST" - restart_node $OST1 - wait_for $OST1 - $DSH2 $OST1 "$LCONF $XMLCONFIG" + reboot_facet ost1 + wait_for ost1 + start ost1 echo "Failover MDS" - wait_for $MDS2 - $DSH2 $MDS2 "$LCONF --select mds_service=$MDS2 $XMLCONFIG" - echo "$MDS2" > CURRENT_MDS - restart_node $MDS1 - #Check FS + wait_for mds + start mds + #Check FS wait $DFPID - clients_recover_osts $OST1 + clients_recover_osts ost1 echo "Verify reintegration" client_df @@ -225,16 +247,12 @@ run_test 2 "Second Failure Mode: MDS/OST `date`" ############### Third Failure Mode ############### test_3() { - echo "Third Failure Mode: MDS/CLIENT `date`" - #Create files echo "Verify Lustre filesystem is up and running" - $DSH2 $CLIENTS "df -h $MOUNTPT" | dshbak -c #MDS Portion - fail_mds $MDS2 $MDS1 + facet_failover mds wait $DFPID || echo df failed: $? - restart_node $MDS2 #Check FS echo "Test Lustre stability after MDS failover" @@ -242,7 +260,7 @@ test_3() { #CLIENT Portion echo "Failing 2 CLIENTS" - fail_clients + fail_clients 2 #Check FS echo "Test Lustre stability after CLIENT failure" @@ -250,11 +268,9 @@ test_3() { #Reintegration echo "Reintegrating CLIENTS" - reintegrate_clients $MDS1 + reintegrate_clients client_df - - #Sleep } run_test 3 "Thirdb Failure Mode: MDS/CLIENT `date`" ################################################### @@ -264,8 +280,8 @@ test_4() { echo "Fourth Failure Mode: OST/MDS `date`" #OST Portion - echo "Failing OST $OST1" - shutdown_node $OST1 + echo "Failing OST ost1" + shutdown_facet ost1 #Check FS echo "Test Lustre stability after OST failure" @@ -273,9 +289,12 @@ test_4() { #MDS Portion echo "Failing MDS" - rm CURRENT_MDS - shutdown_node $MDS1 - restart_node $MDS2 + shutdown_facet mds + reboot_facet mds + + # prepare for MDS failover + change_active mds + reboot_facet mds client_df & DFPID=$! @@ -283,18 +302,17 @@ test_4() { #Reintegration echo "Reintegrating OST" - restart_node $OST1 - wait_for $OST1 - $DSH2 $OST1 "$LCONF $XMLCONFIG" + reboot_node ost1 + wait_for ost1 + start ost1 echo "Failover MDS" - wait_for $MDS2 - $DSH2 $MDS2 "$LCONF--select mds_service=$MDS2 $XMLCONFIG" - echo "$MDS2" > CURRENT_MDS + wait_for mds + start mds #Check FS wait $DFPID - clients_recover_oscs + clients_recover_osts ost1 echo "Test Lustre stability after MDS failover" client_df } @@ -311,8 +329,8 @@ test_5() { #OST Portion echo "Failing OST" - shutdown_node $OST1 - restart_node $OST1 + shutdown_facet ost1 + reboot_node ost1 #Check FS echo "Test Lustre stability after OST failure" @@ -320,8 +338,8 @@ test_5() { #OST Portion echo "Failing OST" - shutdown_node $OST2 - restart_node $OST1 + shutdown_node ost2 + reboot_node ost2 #Check FS echo "Test Lustre stability after OST failure" @@ -329,13 +347,13 @@ test_5() { #Reintegration echo "Reintegrating OSTs" - wait_for $OST1 - wait_for $OST2 - $DSH2 $OST1 "$LCONF $XMLCONFIG" - $DSH2 $OST2 "$LCONF $XMLCONFIG" + wait_for ost1 + wait_for ost1 + start ost1 + start ost2 - clients_recover_osts $OST1 - clients_recover_osts $OST2 + clients_recover_osts ost1 + clients_recover_osts ost2 client_df } run_test 5 "Fifth Failure Mode: OST/OST `date`" @@ -347,13 +365,13 @@ test_6() { #Create files echo "Verify Lustre filesystem is up and running" - $DSH2 $CLIENTS "df $MOUNTPT" - $DSH2 $CLIENTS "/bin/touch $MOUNTPT/\`hostname\`_testfile" + client_df + $PDSH $CLIENTS "/bin/touch $MOUNT/\`hostname\`_testfile" #OST Portion echo "Failing OST" - shutdown_node $OST1 - restart_node $OST1 + shutdown_node ost1 + reboot_node ost1 #Check FS echo "Test Lustre stability after OST failure" @@ -369,8 +387,8 @@ test_6() { #Reintegration echo "Reintegrating OST/CLIENTs" - wait_for $OST1 - $DSH2 $OST1 "$LCONF $XMLCONFIG" + wait_for ost1 + start ost1 reintegrate_clients echo "Verifying mount" @@ -386,23 +404,18 @@ test_7() { #Create files echo "Verify Lustre filesystem is up and running" - $DSH2 $CLIENTS "df $MOUNTPT" - $DSH2 $CLIENTS "/bin/touch $MOUNTPT/\`hostname\`_testfile" + client_df + $PDSH $CLIENTS "/bin/touch $MOUNT/\`hostname\`_testfile" #CLIENT Portion echo "Part 1: Failing CLIENT" - shutdown_node $CLIENT1 - shutdown_node $CLIENT2 - restart_node $CLIENT1 - restart_node $CLIENT2 - check_network $CLIENT1 900 - check_network $CLIENT2 900 + fail_clients 2 #Check FS echo "Test Lustre stability after CLIENTs failure" - $DSH2 $CLIENTS "df $MOUNTPT" - $DSH2 $LIVE_CLIENT "ls -l $MOUNTPT" - $DSH2 $LIVE_CLIENT "rm -f $MOUNTPT/*_testfile" + client_df + $PDSH $LIVE_CLIENT "ls -l $MOUNT" + $PDSH $LIVE_CLIENT "rm -f $MOUNT/*_testfile" #Sleep echo "Wait 1 minutes" @@ -410,42 +423,23 @@ test_7() { #Create files echo "Verify Lustre filesystem is up and running" - $DSH2 $CLIENTS "df $MOUNTPT" - $DSH2 $CLIENTS "/bin/touch $MOUNTPT/\`hostname\`_testfile" + client_df + $PDSH $CLIENTS "/bin/touch $MOUNT/\`hostname\`_testfile" #MDS Portion echo "Failing MDS" - shutdown_node $MDS2 - sleep 5 - restart_node $MDS2 - echo "Failover MDS" - check_port $MDS1 988 - $DSH2 $MDS1 "hostname" - sleep 2 - $DSH2 $MDS1 "$LCONF --select mds_service=$MDS1 $XMLCONFIG" - #pdsh -w $MDS1 nodeup - #cd /usr/local/admin/lustre-alc/cfg - #../bin/mount-lustre-shaver.sh -f alc-ga1-ddn-llp-2mds.cfg -s mds -m alc2 -c alc36 - #cd /home/jodorizz - echo "$MDS1" > CURRENT_MDS + facet_failover mds #Check FS echo "Test Lustre stability after MDS failover" - $DSH2 $CLIENTS "df $MOUNTPT" - $DSH2 $LIVE_CLIENT "ls -l $MOUNTPT" - $DSH2 $LIVE_CLIENT "rm -f $MOUNTPT/*_testfile" + client_df + $PDSH $LIVE_CLIENT "ls -l $MOUNT" + $PDSH $LIVE_CLIENT "rm -f $MOUNT/*_testfile" #Reintegration echo "Reintegrating CLIENTs" - $DSH2 $CLIENT1 "hostname" - sleep 2 - $DSH2 $CLIENT1 "$LCONF $CLIENTOPTS $XMLCONFIG" - $DSH2 $CLIENT2 "hostname" - sleep 2 - $DSH2 $CLIENT2 "$LCONF $CLIENTOPTS $XMLCONFIG" - $DSH2 $CLIENTS "hostname" - sleep 2 - $DSH2 $CLIENTS "df $MOUNTPT" + reintegrate_clients + client_df #Sleep echo "wait 1 minutes" @@ -461,23 +455,18 @@ test_8() { #Create files echo "Verify Lustre filesystem is up and running" - $DSH2 $CLIENTS "df $MOUNTPT" - $DSH2 $CLIENTS "/bin/touch $MOUNTPT/\`hostname\`_testfile" + client_df + $PDSH $CLIENTS "/bin/touch $MOUNT/\`hostname\`_testfile" #CLIENT Portion echo "Failing CLIENTs" - shutdown_node $CLIENT1 - shutdown_node $CLIENT2 - restart_node $CLIENT1 - restart_node $CLIENT2 - check_network $CLIENT1 900 - check_network $CLIENT2 900 + fail_clients 2 #Check FS echo "Test Lustre stability after CLIENTs failure" - $DSH2 $CLIENTS "df $MOUNTPT" - $DSH2 $LIVE_CLIENT "ls -l $MOUNTPT" - $DSH2 $LIVE_CLIENT "rm -f $MOUNTPT/*_testfile" + client_df + $PDSH $LIVE_CLIENT "ls -l $MOUNT" + $PDSH $LIVE_CLIENT "rm -f $MOUNT/*_testfile" #Sleep echo "Wait 1 minutes" @@ -485,35 +474,26 @@ test_8() { #Create files echo "Verify Lustre filesystem is up and running" - $DSH2 $CLIENTS "df $MOUNTPT" - $DSH2 $CLIENTS "/bin/touch $MOUNTPT/\`hostname\`_testfile" + client_df + $PDSH $CLIENTS "/bin/touch $MOUNT/\`hostname\`_testfile" #OST Portion echo "Failing OST" - shutdown_node $OST1 - restart_node $OST1 + shutdown_node ost1 + reboot_node ost1 #Check FS echo "Test Lustre stability after OST failure" - $DSH2 $CLIENTS "df $MOUNTPT" - $DSH2 $LIVE_CLIENT "ls -l $MOUNTPT" - $DSH2 $LIVE_CLIENT "rm -f $MOUNTPT/*_testfile" + client_df + $PDSH $LIVE_CLIENT "ls -l $MOUNT" + $PDSH $LIVE_CLIENT "rm -f $MOUNT/*_testfile" #Reintegration echo "Reintegrating CLIENTs/OST" - $DSH2 $CLIENT1 "hostname" - sleep 2 - $DSH2 $CLIENT1 "$LCONF $CLIENTOPTS $XMLCONFIG" - $DSH2 $CLIENT2 "hostname" - sleep 2 - $DSH2 $CLIENT2 "$LCONF $CLIENTOPTS $XMLCONFIG" - check_network $OST1 900 - check_port $OST1 988 - $DSH2 $OST1 "hostname" - sleep 2 - $DSH2 $OST1 "$LCONF $XMLCONFIG" - $DSH2 $CLIENTS "df $MOUNTPT" - $DSH2 $CLIENTS "/bin/touch $MOUNTPT/CLIENT_OST_2\`hostname\`_testfile" + reintegrate_clients + start ost1 + client_df + $PDSH $CLIENTS "/bin/touch $MOUNT/CLIENT_OST_2\`hostname\`_testfile" #Sleep echo "Wait 1 minutes" @@ -529,23 +509,18 @@ test_9() { #Create files echo "Verify Lustre filesystem is up and running" - $DSH2 $CLIENTS "df $MOUNTPT" - $DSH2 $CLIENTS "/bin/touch $MOUNTPT/\`hostname\`_testfile" + client_df + $PDSH $CLIENTS "/bin/touch $MOUNT/\`hostname\`_testfile" #CLIENT Portion echo "Failing CLIENTs" - shutdown_node $CLIENT1 - shutdown_node $CLIENT2 - restart_node $CLIENT1 - restart_node $CLIENT2 - check_network $CLIENT1 900 - check_network $CLIENT2 900 + fail_clients 2 #Check FS echo "Test Lustre stability after CLIENTs failure" - $DSH2 $CLIENTS "df $MOUNTPT" - $DSH2 $LIVE_CLIENT "ls -l $MOUNTPT" - $DSH2 $LIVE_CLIENT "rm -f $MOUNTPT/*_testfile" + client_df + $PDSH $LIVE_CLIENT "ls -l $MOUNT" + $PDSH $LIVE_CLIENT "rm -f $MOUNT/*_testfile" #Sleep echo "Wait 1 minutes" @@ -553,41 +528,23 @@ test_9() { #Create files echo "Verify Lustre filesystem is up and running" - $DSH2 $CLIENTS "df $MOUNTPT" - $DSH2 $CLIENTS "/bin/touch $MOUNTPT/\`hostname\`_testfile" + client_df + $PDSH $CLIENTS "/bin/touch $MOUNT/\`hostname\`_testfile" #CLIENT Portion echo "Failing CLIENTs" - shutdown_node $CLIENT3 - shutdown_node $CLIENT4 - restart_node $CLIENT3 - restart_node $CLIENT4 - check_network $CLIENT3 900 - check_network $CLIENT4 900 - + fail_clients 2 + #Check FS echo "Test Lustre stability after CLIENTs failure" - $DSH2 $CLIENTS "df $MOUNTPT" - $DSH2 $LIVE_CLIENT "ls -l $MOUNTPT" - $DSH2 $LIVE_CLIENT "rm -f $MOUNTPT/*_testfile" + client_df + $PDSH $LIVE_CLIENT "ls -l $MOUNT" + $PDSH $LIVE_CLIENT "rm -f $MOUNT/*_testfile" #Reintegration echo "Reintegrating CLIENTs/CLIENTs" - $DSH2 $CLIENT1 "hostname" - sleep 2 - $DSH2 $CLIENT1 "$LCONF $CLIENTOPTS $XMLCONFIG" - $DSH2 $CLIENT2 "hostname" - sleep 2 - $DSH2 $CLIENT2 "$LCONF $CLIENTOPTS $XMLCONFIG" - $DSH2 $CLIENT3 "hostname" - sleep 2 - $DSH2 $CLIENT3 "$LCONF $CLIENTOPTS $XMLCONFIG" - $DSH2 $CLIENT4 "hostname" - sleep 2 - $DSH2 $CLIENT4 "$LCONF $CLIENTOPTS $XMLCONFIG" - sleep 5 - $DSH2 $CLIENTS "hostname" - $DSH2 $CLIENTS "df $MOUNTPT" + reintegrate_clients + client_df #Sleep echo "Wait 1 minutes" @@ -596,8 +553,11 @@ test_9() { run_test 9 "Ninth Failure Mode: CLIENT/CLIENT `date`" ################################################### -test_10 { +test_10() { #Run availability after all failures ./availability.sh 21600 } run_test 10 "Running Availability for 6 hours..." + +equals_msg "Done, cleaning up" +cleanup diff --git a/lustre/tests/replay-dual.sh b/lustre/tests/replay-dual.sh index c61dc9e..62bd1ca 100755 --- a/lustre/tests/replay-dual.sh +++ b/lustre/tests/replay-dual.sh @@ -5,48 +5,29 @@ set -e LUSTRE=${LUSTRE:-`dirname $0`/..} . $LUSTRE/tests/test-framework.sh -init_test_env - -# XXX I wish all this stuff was in some default-config.sh somewhere -mds_HOST=${mds_HOST:-`hostname`} -mdsfailover_HOST=${mdsfailover_HOST} -ost_HOST=${ost_HOST:-`hostname`} -client_HOST=${client_HOST:-`hostname`} - -NETTYPE=${NETTYPE:-tcp} - -PDSH=${PDSH:-no_dsh} -MOUNT=${MOUNT:-/mnt/lustre} -MDSDEV=${MDSDEV:-/tmp/mds-`hostname`} -MDSSIZE=${MDSSIZE:-100000} -OSTDEV=${OSTDEV:-/tmp/ost-`hostname`} -OSTSIZE=${OSTSIZE:-100000} -MOUNT1=${MOUNT1:-${MOUNT}1} -MOUNT2=${MOUNT2:-${MOUNT}2} -UPCALL=${UPCALL:-$PWD/replay-single-upcall.sh} -FSTYPE=${FSTYPE:-ext3} -TIMEOUT=${TIMEOUT:-5} - -STRIPE_BYTES=65536 -STRIPES_PER_OBJ=1 +init_test_env $@ + +. ${CONFIG:=$LUSTRE/tests/cfg/local.sh} gen_config() { - rm -f replay-dual.xml - add_facet mds - add_facet ost - add_facet client --lustre_upcall $UPCALL - do_lmc --add mds --node mds_facet --mds mds1 --dev $MDSDEV --size $MDSSIZE - do_lmc --add lov --mds mds1 --lov lov1 --stripe_sz $STRIPE_BYTES --stripe_cnt $STRIPES_PER_OBJ --stripe_pattern 0 - do_lmc --add ost --lov lov1 --node ost_facet --ost ost1 --dev $OSTDEV --size $OSTSIZE - do_lmc --add ost --lov lov1 --node ost_facet --ost ost2 --dev ${OSTDEV}-2 --size $OSTSIZE - do_lmc --add mtpt --node client_facet --path $MOUNT --mds mds1 --ost lov1 + rm -f $XMLCONFIG + add_mds mds --dev $MDSDEV --size $MDSSIZE + if [ ! -z "$mdsfailover_HOST" ]; then + add_mdsfailover mds --dev $MDSDEV --size $MDSSIZE + fi + + add_lov lov1 mds --stripe_sz $STRIPE_BYTES\ + --stripe_cnt $STRIPES_PER_OBJ --stripe_pattern 0 + add_ost ost --lov lov1 --dev $OSTDEV --size $OSTSIZE + add_ost ost2 --lov lov1 --dev ${OSTDEV}-2 --size $OSTSIZE + add_client client mds --lov lov1 --path $MOUNT } + build_test_filter cleanup() { - [ "$DAEMONFILE" ] && $LCTL debug_daemon stop # make sure we are using the primary MDS, so the config log will # be able to clean up properly. activemds=`facet_active mds` @@ -54,12 +35,12 @@ cleanup() { fail mds fi - $LCONF --cleanup --zeroconf --mds_uuid mds1_UUID --mds_nid $mds_HOST \ - --local_nid $client_HOST --profile client_facet --mount $MOUNT - $LCONF --cleanup --zeroconf --mds_uuid mds1_UUID --mds_nid $mds_HOST \ - --local_nid $client_HOST --profile client_facet --mount $MOUNT2 - stop mds ${FORCE} --dump cleanup-dual.log - stop ost ${FORCE} + umount $MOUNT2 + umount $MOUNT + rmmod llite + stop mds ${FORCE} + stop ost2 ${FORCE} + stop ost ${FORCE} --dump cleanup-dual.log } if [ "$ONLY" == "cleanup" ]; then @@ -69,9 +50,7 @@ if [ "$ONLY" == "cleanup" ]; then fi gen_config -start mds --write_conf --reformat -start ost --reformat -start mds +start ost --reformat $OSTLCONFARGS PINGER=`cat /proc/fs/lustre/pinger` if [ "$PINGER" != "on" ]; then @@ -80,11 +59,11 @@ if [ "$PINGER" != "on" ]; then exit 1 fi -# 0-conf client -$LCONF --zeroconf --mds_uuid mds1_UUID --mds_nid `h2$NETTYPE $mds_HOST` \ - --local_nid `h2$NETTYPE $client_HOST` --profile client_facet --mount $MOUNT -$LCONF --zeroconf --mds_uuid mds1_UUID --mds_nid `h2$NETTYPE $mds_HOST` \ - --local_nid `h2$NETTYPE $client_HOST` --profile client_facet --mount $MOUNT2 +start ost2 --reformat $OSTLCONFARGS +[ "$DAEMONFILE" ] && $LCTL debug_daemon start $DAEMONFILE $DAEMONSIZE +start mds $MDSLCONFARGS --reformat +zconf_mount $MOUNT +zconf_mount $MOUNT2 echo $TIMEOUT > /proc/sys/lustre/timeout echo $UPCALL > /proc/sys/lustre/upcall @@ -198,3 +177,4 @@ run_test 6 "open1, open2, unlink |X| close1 [fail mds] close2" equals_msg test complete, cleaning up +cleanup diff --git a/lustre/tests/replay-single-upcall.sh b/lustre/tests/replay-single-upcall.sh index 193ef07..b71b8ea 100755 --- a/lustre/tests/replay-single-upcall.sh +++ b/lustre/tests/replay-single-upcall.sh @@ -4,21 +4,36 @@ TESTDIR=`dirname $0` LUSTRE=$TESTDIR/.. -exec >> $TESTDIR/recovery-`hostname`.log +exec >> $TESTDIR/logs/recovery-`hostname`.log exec 2>&1 set -xv failed_import() { - if [ -f $LUSTRE/tests/ostactive ] ; then - source $LUSTRE/tests/mdsactive + if [ -f $TESTDIR/XMLCONFIG ] ; then + source $TESTDIR/XMLCONFIG + if [ ! -f $TESTDIR/XMLCONFIG ]; then + echo "config file not found: $XMLCONFIG" + exit 1 + fi else - mdsactive=mds + echo "$TESTDIR/XMLCONFIG: not found" + exit 1 + fi + + if [ -f $TESTDIR/mdsactive ] ; then + source $TESTDIR/mdsactive + MDSSELECT="--select mds_svc=${mdsactive}_facet" + fi + + if [ -f $TESTDIR/ostactive ] ; then + source $TESTDIR/ostactive + OSTSELECT="--select ost_svc=${ostactive}_facet" fi $LUSTRE/utils/lconf --verbose --recover --node client_facet \ - --select mds1=${mdsactive}_facet\ - --tgt_uuid $2 --client_uuid $3 --conn_uuid $4 $TESTDIR/replay-single.xml + $MDSSELECT $OSTSELECT \ + --tgt_uuid $2 --client_uuid $3 --conn_uuid $4 $XMLCONFIG } diff --git a/lustre/tests/replay-single.sh b/lustre/tests/replay-single.sh index 0442588..65f2804 100755 --- a/lustre/tests/replay-single.sh +++ b/lustre/tests/replay-single.sh @@ -9,55 +9,27 @@ set -e LUSTRE=${LUSTRE:-`dirname $0`/..} . $LUSTRE/tests/test-framework.sh -init_test_env +init_test_env $@ +set -xv + +. ${CONFIG:=$LUSTRE/tests/cfg/local.sh} # Skip these tests ALWAYS_EXCEPT="" -# XXX I wish all this stuff was in some default-config.sh somewhere -mds_HOST=${mds_HOST:-`hostname`} -mdsfailover_HOST=${mdsfailover_HOST} -ost_HOST=${ost_HOST:-`hostname`} -client_HOST=${client_HOST:-`hostname`} - -NETTYPE=${NETTYPE:-tcp} - -PDSH=${PDSH:-no_dsh} - -MOUNT=${MOUNT:-/mnt/lustre} -DIR=${DIR:-$MOUNT} -MDSDEV=${MDSDEV:-$ROOT/tmp/mds-`hostname`} -MDSFAILOVERDEV=${MDSFAILOVERDEV:-$MDSDEV} -MDSSIZE=${MDSSIZE:-10000} -OSTDEV=${OSTDEV:-$ROOT/tmp/ost-`hostname`} -OSTSIZE=${OSTSIZE:-10000} -UPCALL=${UPCALL:-$PWD/replay-single-upcall.sh} -FSTYPE=${FSTYPE:-ext3} -TIMEOUT=${TIMEOUT:-10} - -STRIPE_BYTES=65536 -STRIPES_PER_OBJ=0 - gen_config() { rm -f $XMLCONFIG - add_facet mds --timeout=${TIMEOUT} - add_facet ost --timeout=${TIMEOUT} - add_facet client --lustre_upcall $UPCALL --timeout=${TIMEOUT} - do_lmc --add mds --node mds_facet --mds mds1 --dev $MDSDEV --size $MDSSIZE + add_mds mds --dev $MDSDEV --size $MDSSIZE if [ ! -z "$mdsfailover_HOST" ]; then - add_facet mdsfailover - do_lmc --add mds --failover --node mdsfailover_facet \ - --mds mds1 --dev $MDSFAILOVERDEV --size $MDSSIZE + add_mdsfailover mds --dev $MDSDEV --size $MDSSIZE fi - do_lmc --add lov --mds mds1 --lov lov1 --stripe_sz $STRIPE_BYTES --stripe_cnt $STRIPES_PER_OBJ --stripe_pattern 0 - do_lmc --add ost --lov lov1 --node ost_facet --ost ost1 --dev $OSTDEV --size $OSTSIZE - do_lmc --add ost --lov lov1 --node ost_facet --ost ost2 --dev ${OSTDEV}-2 --size $OSTSIZE - do_lmc --add mtpt --node client_facet --path $MOUNT --mds mds1 --ost lov1 - -# do_lmc --add ost --node ost_facet --ost ost1 --dev $OSTDEV --size $OSTSIZE -# do_lmc --add mtpt --node client_facet --path $MOUNT --mds mds1 --ost ost1 + add_lov lov1 mds --stripe_sz $STRIPE_BYTES\ + --stripe_cnt $STRIPES_PER_OBJ --stripe_pattern 0 + add_ost ost --lov lov1 --dev $OSTDEV --size $OSTSIZE + add_ost ost2 --lov lov1 --dev ${OSTDEV}-2 --size $OSTSIZE + add_client client mds --lov lov1 --path $MOUNT } build_test_filter @@ -71,6 +43,7 @@ cleanup() { fi zconf_umount $MOUNT stop mds ${FORCE} $MDSLCONFARGS + stop ost2 ${FORCE} --dump cleanup.log stop ost ${FORCE} --dump cleanup.log } @@ -83,6 +56,7 @@ fi gen_config start ost --reformat $OSTLCONFARGS +start ost2 --reformat $OSTLCONFARGS [ "$DAEMONFILE" ] && $LCTL debug_daemon start $DAEMONFILE $DAEMONSIZE start mds $MDSLCONFARGS --reformat zconf_mount $MOUNT -- 1.8.3.1