8 # eg, assert_env LUSTRE MDSNODES OSTNODES CLIENTS
12 if [ -z "${!name}" ]; then
13 echo "$0: $name must be set"
17 [ $failed ] && exit 1 || true
21 echo "usage: $0 [-r] [-f cfgfile]"
28 export LUSTRE=`absolute_path $LUSTRE`
29 export TESTSUITE=`basename $0 .sh`
30 export XMLCONFIG="${TESTSUITE}.xml"
31 export LTESTDIR=${LTESTDIR:-$LUSTRE/../ltest}
33 [ -d /r ] && export ROOT=/r
35 export PATH=:$PATH:$LUSTRE/utils:$LUSTRE/tests
36 export LLMOUNT=${LLMOUNT:-"llmount"}
37 export LCONF=${LCONF:-"lconf"}
38 export LMC=${LMC:-"lmc"}
39 export LCTL=${LCTL:-"$LUSTRE/utils/lctl"}
40 export CHECKSTAT="${CHECKSTAT:-checkstat} "
42 # Paths on remote nodes, if different
43 export RLUSTRE=${RLUSTRE:-$LUSTRE}
44 export RPWD=${RPWD:-$PWD}
48 while getopts "rvf:" opt $*; do
51 r) REFORMAT=--reformat;;
57 # save the name of the config file for the upcall
58 echo "XMLCONFIG=$LUSTRE/tests/$XMLCONFIG" > $LUSTRE/tests/XMLCONFIG
59 # echo "CONFIG=`canonical_path $CONFIG`" > $LUSTRE/tests/CONFIG
66 active=`facet_active $facet`
67 do_facet $facet $LCONF --select ${facet}_svc=${active}_facet \
68 --node ${active}_facet --ptldebug $PTLDEBUG --subsystem $SUBSYSTEM \
74 active=`facet_active $facet`
76 do_facet $facet $LCONF --select ${facet}_svc=${active}_facet \
77 --node ${active}_facet --ptldebug $PTLDEBUG --subsystem $SUBSYSTEM \
78 $@ --cleanup $XMLCONFIG
85 do_node $client mkdir $mnt 2> /dev/null || :
87 if [ -x /sbin/mount.lustre ] ; then
88 do_node $client mount -t lustre -o nettype=$NETTYPE `facet_active_host mds1`:/mds1_svc/client_facet $mnt || return 1
91 do_node $client $LCONF --nosetup --node client_facet $XMLCONFIG > /dev/null || return 2
92 $LCONF --nosetup --node client_facet $XMLCONFIG
93 do_node $client $LLMOUNT `facet_active_host mds1`:/mds1_svc/client_facet $mnt -o nettype=$NETTYPE|| return 4
96 [ -d /r ] && $LCTL modules > /r/tmp/ogdb-`hostname`
104 do_node $client umount $force $mnt || :
105 do_node $client $LCONF --cleanup --nosetup --node client_facet $XMLCONFIG > /dev/null || :
110 if [ "$FAILURE_MODE" = HARD ]; then
111 $POWER_DOWN `facet_active_host $facet`
113 elif [ "$FAILURE_MODE" = SOFT ]; then
114 stop $facet --force --failover --nomod
120 if [ "$FAILURE_MODE" = HARD ]; then
121 $POWER_UP `facet_active_host $facet`
127 check_network $HOST 900
128 while ! do_node $HOST "ls -d $LUSTRE " > /dev/null; do sleep 5; done
133 HOST=`facet_active_host $facet`
138 # not every config has many clients
139 if [ ! -z "$CLIENTS" ]; then
140 $PDSH $CLIENTS "df $MOUNT" > /dev/null
146 echo "Failing $facet node `facet_active_host $facet`"
147 shutdown_facet $facet
151 echo "df pid is $DFPID"
153 TO=`facet_active_host $facet`
154 echo "Failover $facet to $TO"
163 do_facet $facet $LCTL --device %${facet}_svc readonly
164 do_facet $facet $LCTL --device %${facet}_svc notransno
165 do_facet $facet $LCTL mark "REPLAY BARRIER"
166 $LCTL mark "REPLAY BARRIER"
170 UUID=`cat /proc/fs/lustre/mdc/*_MNT_*/uuid`
171 do_facet mds "echo $UUID > /proc/fs/lustre/mds/mds1_svc/evict_client"
176 facet_failover $facet
177 df $MOUNT || error "post-failover df: $?"
182 stop $facet --force --failover --nomod
185 do_facet $facet lctl --device %${facet}_svc abort_recovery
186 df $MOUNT || echo "first df failed: $?"
187 df $MOUNT || error "post-failover df: $?"
191 $LMC -m ${XMLCONFIG} $@
195 if [ "$1" = "client" ]; then echo \'*\'; else
196 $PDSH $1 $GMNALNID -l | cut -d\ -f2
201 if [ "$1" = "client" ]; then echo \'*\'; else
208 if [ "$1" = "client" ]; then echo \'*\'; else
209 echo $1 | sed 's/[^0-9]*//g'
216 varname=${facet}_HOST
222 HOST=`facet_host $facet`
223 if [ -z "$HOST" ]; then
224 echo "The env variable ${facet}_HOST must be set."
227 echo `h2$NETTYPE $HOST`
232 local activevar=${facet}active
234 if [ -z "$active" ] ; then
241 facet_active_host() {
243 local active=`facet_active $facet`
244 if [ "$facet" == client ]; then
247 echo `facet_host $active`
253 failover=${facet}failover
254 host=`facet_host $failover`
255 [ -z "$host" ] && return
256 curactive=`facet_active $facet`
257 if [ -z "${curactive}" -o "$curactive" == "$failover" ] ; then
258 eval export ${facet}active=$facet
260 eval export ${facet}active=$failover
262 # save the active host for this facet
263 activevar=${facet}active
264 echo "$activevar=${!activevar}" > ./$activevar
273 $PDSH $HOST $LCTL mark "$@" > /dev/null 2>&1 || :
275 $PDSH $HOST "(PATH=\$PATH:$RLUSTRE/utils:$RLUSTRE/tests; cd $RPWD; sh -c \"$@\")"
281 if [ "$facet" == "mds" ]; then
282 if [ "$MDSCOUNT" -gt 1 ]; then
283 for num in `seq $MDSCOUNT`; do
284 HOST=`facet_active_host $facet$num`
290 HOST=`facet_active_host $facet`
297 echo "add facet $facet: `facet_host $facet`"
298 do_lmc --add node --node ${facet}_facet $@ --timeout $TIMEOUT \
299 --lustre_upcall $UPCALL --ptldebug $PTLDEBUG --subsystem $SUBSYSTEM
300 do_lmc --add net --node ${facet}_facet --nid `facet_nid $facet` \
309 do_lmc --add mds --node ${facet}_facet --mds ${facet}_svc $*
315 add_facet ${facet}failover --lustre_upcall $UPCALL
316 do_lmc --add mds --node ${facet}failover_facet --mds ${facet}_svc $*
324 do_lmc --add ost --node ${facet}_facet --ost ${facet}_svc $*
330 add_facet ${facet}failover
331 do_lmc --add ost --failover --node ${facet}failover_facet --ost ${facet}_svc $*
338 do_lmc --add lov --mds ${mds_facet}_svc --lov $lov $*
345 do_lmc --add lov --lmv $lmv --lov $lov $*
351 do_lmc --add lmv --lmv $lmv $*
357 add_facet $facet --lustre_upcall $UPCALL
358 do_lmc --add mtpt --node ${facet}_facet $*
369 while [ $NETWORK -eq 0 ]; do
370 ping -c 1 -w 3 $1 > /dev/null
371 if [ $? -eq 0 ]; then
375 echo "waiting for $1, $((MAX - WAIT)) secs left"
378 if [ $WAIT -gt $MAX ]; then
379 echo "Network not available"
385 while( !($DSH2 $1 "netstat -tna | grep -q $2") ) ; do
396 # the sed converts spaces to commas, but leaves the last space
397 # alone, so the line doesn't end with a comma.
398 echo "$*" | tr -s " " "\n" | sort -b -u | tr "\n" " " | sed 's/ \([^$]\)/,\1/g'
402 (cd `dirname $1`; echo $PWD/`basename $1`)
405 ##################################
409 # OBD_FAIL_MDS_ALL_REQUEST_NET
411 do_facet mds "echo 0x123 > /proc/sys/lustre/fail_loc"
412 do_facet client "$1" || RC=$?
413 do_facet mds "echo 0 > /proc/sys/lustre/fail_loc"
418 # OBD_FAIL_MDS_ALL_REPLY_NET
420 do_facet mds "echo 0x122 > /proc/sys/lustre/fail_loc"
421 do_facet client "$@" || RC=$?
422 do_facet mds "echo 0 > /proc/sys/lustre/fail_loc"
427 # OBD_FAIL_MDS_REINT_NET_REP
429 do_facet mds "echo 0x119 > /proc/sys/lustre/fail_loc"
430 do_facet client "$@" || RC=$?
431 do_facet mds "echo 0 > /proc/sys/lustre/fail_loc"
436 #define OBD_FAIL_OST_BRW_PAUSE_BULK 0x214
438 do_facet ost "echo 0x214 > /proc/sys/lustre/fail_loc"
439 do_facet client "$1" || RC=$?
440 do_facet client "sync"
441 do_facet ost "echo 0 > /proc/sys/lustre/fail_loc"
446 #define OBD_FAIL_LDLM_CANCEL 0x304
448 do_facet client "echo 0x304 > /proc/sys/lustre/fail_loc"
449 do_facet client "$@" || RC=$?
450 do_facet client "echo 0 > /proc/sys/lustre/fail_loc"
455 #define OBD_FAIL_LDLM_BL_CALLBACK 0x305
457 do_facet client "echo 0x305 > /proc/sys/lustre/fail_loc"
458 do_facet client "$@" || RC=$?
459 do_facet client "echo 0 > /proc/sys/lustre/fail_loc"
467 echo "clearing fail_loc on $facet"
468 do_facet $facet "sysctl -w lustre.fail_loc=0"
472 $LCTL mark cancel_lru_locks
473 for d in /proc/fs/lustre/ldlm/namespaces/$1*; do
474 if [ -f $d/lru_size ]; then
475 echo clear > $d/lru_size
476 grep [0-9] $d/lock_unused_count
481 ##################################
484 echo "${TESTSUITE}: **** FAIL:" $@
489 build_test_filter() {
493 for E in $EXCEPT $ALWAYS_EXCEPT; do
494 eval EXCEPT_${E}=true
503 IFS=abcdefghijklmnopqrstuvwxyz _basetest $1
507 export base=`basetest $1`
508 if [ ! -z "$ONLY" ]; then
510 if [ ${!testname}x != x ]; then
515 if [ ${!testname}x != x ]; then
523 if [ ${!testname}x != x ]; then
524 echo "skipping excluded test $1"
527 testname=EXCEPT_$base
528 if [ ${!testname}x != x ]; then
529 echo "skipping excluded test $1 (base $base)"
537 EQUALS="======================================================================"
541 local suffixlen=$((${#EQUALS} - ${#msg}))
542 [ $suffixlen -lt 5 ] && suffixlen=5
543 printf '===== %s %.*s\n' "$msg" $suffixlen $EQUALS
548 lctl mark "$*" 2> /dev/null || true
557 # Pretty tests run faster.
558 equals_msg $testnum: $message
561 test_${testnum} || error "test_$testnum failed with $?"
565 (cd `dirname $1`; echo $PWD/`basename $1`)