5 # eg, assert_env LUSTRE MDSNODES OSTNODES CLIENTS
9 if [ -z "${!name}" ]; then
10 echo "$0: $name must be set"
14 [ $failed ] && exit 1 || true
18 echo "usage: $0 [-r] [-f cfgfile]"
25 export LUSTRE=`absolute_path $LUSTRE`
26 export TESTSUITE=`basename $0 .sh`
27 export XMLCONFIG="${TESTSUITE}.xml"
28 export LTESTDIR=${LTESTDIR:-$LUSTRE/../ltest}
30 [ -d /r ] && export ROOT=/r
32 export PATH=:$PATH:$LUSTRE/utils:$LUSTRE/tests
33 export LCONF=${LCONF:-"lconf"}
34 export LMC=${LMC:-"lmc"}
35 export LCTL=${LCTL:-"lctl"}
36 export CHECKSTAT="${CHECKSTAT:-checkstat} "
38 # Paths on remote nodes, if different
39 export RLUSTRE=${RLUSTRE:-$LUSTRE}
40 export RPWD=${RPWD:-$PWD}
44 while getopts "rvf:" opt $*; do
47 r) REFORMAT=--reformat;;
53 # save the name of the config file for the upcall
54 echo "XMLCONFIG=$LUSTRE/tests/$XMLCONFIG" > $LUSTRE/tests/XMLCONFIG
61 active=`facet_active $facet`
62 do_facet $facet $LCONF --select ${facet}_svc=${active}_facet --node ${active}_facet --ptldebug $PTLDEBUG $@ $XMLCONFIG
67 active=`facet_active $facet`
69 do_facet $facet $LCONF --select ${facet}_svc=${active}_facet --node ${active}_facet --ptldebug $PTLDEBUG $@ --cleanup $XMLCONFIG
75 [ -d $mnt ] || mkdir $mnt
77 if [ -x /sbin/mount.lustre ] ; then
78 mount -t lustre -o nettype=$NETTYPE \
79 `facet_host mds`:/mds_svc/client_facet $mnt
82 $LCONF --nosetup --node client_facet $XMLCONFIG
83 $LUSTRE/utils/llmount `facet_host mds`:/mds_svc/client_facet $mnt \
87 [ -d /r ] && $LCTL modules > /r/tmp/ogdb-`hostname`
94 $LCONF --cleanup --nosetup --node client_facet $XMLCONFIG || :
99 if [ "$FAILURE_MODE" = HARD ]; then
100 $POWER_DOWN `facet_active_host $facet`
102 elif [ "$FAILURE_MODE" = SOFT ]; then
103 stop $facet --force --failover --nomod
109 if [ "$FAILURE_MODE" = HARD ]; then
110 $POWER_UP `facet_active_host $facet`
116 check_network $HOST 900
117 while ! do_node $HOST "$CHECKSTAT -t dir $LUSTRE"; do sleep 5; done
122 HOST=`facet_active_host $facet`
127 # not every config has many clients
128 if [ ! -z "$CLIENTS" ]; then
129 $PDSH $CLIENTS "df $MOUNT" > /dev/null
135 echo "Failing $facet node `facet_active_host $facet`"
136 shutdown_facet $facet
141 TO=`facet_active_host $facet`
142 echo "Failover MDS to $TO"
151 do_facet $facet $LCTL --device %${facet}_svc readonly
152 do_facet $facet $LCTL --device %${facet}_svc notransno
153 do_facet $facet $LCTL mark "REPLAY BARRIER"
154 $LCTL mark "REPLAY BARRIER"
158 UUID=`cat /proc/fs/lustre/mdc/*_MNT_*/uuid`
159 do_facet mds "echo $UUID > /proc/fs/lustre/mds/mds_svc/evict_client"
164 facet_failover $facet
165 df $MOUNT || error "post-failover df: $?"
170 stop $facet --force --failover --nomod
173 do_facet $facet lctl --device %${facet}_svc abort_recovery
174 df $MOUNT || echo "first df failed: $?"
175 df $MOUNT || error "post-failover df: $?"
179 $LMC -m ${XMLCONFIG} $@
183 if [ "$1" = "client" ]; then echo \'*\'; else
190 if [ "$1" = "client" ]; then echo \'*\'; else
191 echo $1 | sed 's/[^0-9]*//g'
198 varname=${facet}_HOST
204 HOST=`facet_host $facet`
205 if [ -z "$HOST" ]; then
206 echo "The env variable ${facet}_HOST must be set."
209 echo `h2$NETTYPE $HOST`
214 local activevar=${facet}active
216 if [ -z "$active" ] ; then
223 facet_active_host() {
225 local active=`facet_active $facet`
226 echo `facet_host $active`
231 failover=${facet}failover
232 host=`facet_host $failover`
233 [ -z "$host" ] && return
234 curactive=`facet_active $facet`
235 if [ -z "${curactive}" -o "$curactive" == "$failover" ] ; then
236 eval export ${facet}active=$facet
238 eval export ${facet}active=$failover
240 # save the active host for this facet
241 activevar=${facet}active
242 echo "$activevar=${!activevar}" > ./$activevar
252 $PDSH $HOST "(PATH=\$PATH:$RLUSTRE/utils:$RLUSTRE/tests; cd $RPWD; sh -c \"$@\")"
257 HOST=`facet_active_host $facet`
264 echo "add facet $facet: `facet_host $facet`"
265 do_lmc --add node --node ${facet}_facet $@ --timeout $TIMEOUT
266 do_lmc --add net --node ${facet}_facet --nid `facet_nid $facet` \
274 add_facet $facet --lustre_upcall $UPCALL
275 do_lmc --add mds --node ${facet}_facet --mds ${facet}_svc $*
281 add_facet ${facet}failover --lustre_upcall $UPCALL
282 do_lmc --add mds --node ${facet}failover_facet --mds ${facet}_svc $*
290 do_lmc --add ost --node ${facet}_facet --ost ${facet}_svc $*
296 add_facet ${facet}failover
297 do_lmc --add ost --failover --node ${facet}failover_facet --ost ${facet}_svc $*
304 do_lmc --add lov --mds ${mds_facet}_svc --lov $lov $*
312 add_facet $facet --lustre_upcall $UPCALL
313 do_lmc --add mtpt --node ${facet}_facet --mds ${mds}_svc $*
325 while [ $NETWORK -eq 0 ]; do
326 ping -c 1 -w 3 $1 > /dev/null
327 if [ $? -eq 0 ]; then
331 echo "waiting for $1, $((MAX - WAIT)) secs left"
334 if [ $WAIT -gt $MAX ]; then
335 echo "Network not available"
341 while( !($DSH2 $1 "netstat -tna | grep -q $2") ) ; do
352 # the sed converts spaces to commas, but leaves the last space
353 # alone, so the line doesn't end with a comma.
354 echo "$*" | tr -s " " "\n" | sort -b -u | tr "\n" " " | sed 's/ \([^$]\)/,\1/g'
358 (cd `dirname $1`; echo $PWD/`basename $1`)
361 ##################################
365 # OBD_FAIL_MDS_ALL_REQUEST_NET
367 do_facet mds "echo 0x123 > /proc/sys/lustre/fail_loc"
368 do_facet client "$1" || RC=$?
369 do_facet mds "echo 0 > /proc/sys/lustre/fail_loc"
374 # OBD_FAIL_MDS_ALL_REPLY_NET
376 do_facet mds "echo 0x122 > /proc/sys/lustre/fail_loc"
377 do_facet client "$@" || RC=$?
378 do_facet mds "echo 0 > /proc/sys/lustre/fail_loc"
383 #define OBD_FAIL_OST_BRW_PAUSE_BULK 0x214
385 do_facet ost "echo 0x214 > /proc/sys/lustre/fail_loc"
386 do_facet client "$1" || RC=$?
387 do_facet client "sync"
388 do_facet ost "echo 0 > /proc/sys/lustre/fail_loc"
393 #define OBD_FAIL_LDLM_BL_CALLBACK 0x305
395 do_facet client "echo 0x305 > /proc/sys/lustre/fail_loc"
396 do_facet client "$@" || RC=$?
397 do_facet client "echo 0 > /proc/sys/lustre/fail_loc"
401 ##################################
404 echo "${TESTSUITE}: **** FAIL:" $@
408 build_test_filter() {
412 for E in $EXCEPT $ALWAYS_EXCEPT; do
413 eval EXCEPT_${E}=true
422 IFS=abcdefghijklmnopqrstuvwxyz _basetest $1
426 export base=`basetest $1`
427 if [ ! -z "$ONLY" ]; then
429 if [ ${!testname}x != x ]; then
434 if [ ${!testname}x != x ]; then
442 if [ ${!testname}x != x ]; then
443 echo "skipping excluded test $1"
446 testname=EXCEPT_$base
447 if [ ${!testname}x != x ]; then
448 echo "skipping excluded test $1 (base $base)"
456 EQUALS="======================================================================"
460 local suffixlen=$((${#EQUALS} - ${#msg}))
461 [ $suffixlen -lt 5 ] && suffixlen=5
462 printf '===== %s %.*s\n' "$msg" $suffixlen $EQUALS
471 # Pretty tests run faster.
472 equals_msg $testnum: $message
474 test_${testnum} || error "test_$testnum failed with $?"