7 # eg, assert_env LUSTRE MDSNODES OSTNODES CLIENTS
11 if [ -z "${!name}" ]; then
12 echo "$0: $name must be set"
16 [ $failed ] && exit 1 || true
20 echo "usage: $0 [-r] [-f cfgfile]"
27 export LUSTRE=`absolute_path $LUSTRE`
28 export TESTSUITE=`basename $0 .sh`
29 export XMLCONFIG=${XMLCONFIG:-${TESTSUITE}.xml}
30 export LTESTDIR=${LTESTDIR:-$LUSTRE/../ltest}
32 [ -d /r ] && test "x$ROOT" = "x" && export ROOT=/r
33 export TMP=${TMP:-$ROOT/tmp}
35 export PATH=:$PATH:$LUSTRE/utils:$LUSTRE/tests
36 export LLMOUNT=${LLMOUNT:-"llmount"}
37 export LCONF=${LCONF:-"lconf"}
38 export LMC=${LMC:-"lmc"}
39 export LCTL=${LCTL:-"$LUSTRE/utils/lctl"}
40 export CHECKSTAT="${CHECKSTAT:-checkstat} "
41 DEF_FSTYPE=`test "x$(uname -r | grep -o '2.6')" = "x2.6" && echo "ldiskfs" || echo "ext3"`
42 export FSTYPE=${FSTYPE:-$DEF_FSTYPE}
43 #used only if FSTYPE == smfs, otherwise ignored by lconf
44 MDS_BACKFSTYPE=${MDS_BACKFSTYPE:-$DEF_FSTYPE}
45 OST_BACKFSTYPE=${OST_BACKFSTYPE:-$DEF_FSTYPE}
47 export SECURITY=${SECURITY:-"null"}
49 # Paths on remote nodes, if different
50 export RLUSTRE=${RLUSTRE:-$LUSTRE}
51 export RPWD=${RPWD:-$PWD}
55 while getopts "rvf:" opt $*; do
58 r) REFORMAT=--reformat;;
67 # save the name of the config file for the upcall
68 echo "XMLCONFIG=$LUSTRE/tests/$XMLCONFIG" > $LUSTRE/tests/XMLCONFIG
69 # echo "CONFIG=`canonical_path $CONFIG`" > $LUSTRE/tests/CONFIG
78 active=`facet_active $facet`
79 do_facet $facet $LCONF --select ${facet}_svc=${active}_facet -v \
80 --node ${active}_facet --ptldebug $PTLDEBUG --subsystem $SUBSYSTEM \
81 --mds_sec $SECURITY $@ $XMLCONFIG
86 active=`facet_active $facet`
88 do_facet $facet $LCONF --select ${facet}_svc=${active}_facet \
89 --node ${active}_facet --ptldebug $PTLDEBUG --subsystem $SUBSYSTEM \
90 $@ --cleanup $XMLCONFIG
97 do_node $client mkdir $mnt 2> /dev/null || :
99 if [ -x /sbin/mount.lustre ] ; then
100 do_node $client mount -t lustre -o mds_sec=$SECURITY,nettype=$NETTYPE \
101 `facet_active_host mds1`:/mds1_svc/client_facet $mnt || return 2
103 # this is so cheating
104 do_node $client $LCONF --nosetup --node client_facet $XMLCONFIG > /dev/null || return 2
105 do_node $client $LLMOUNT `facet_active_host mds1`:/mds1_svc/client_facet $mnt \
106 -o mds_sec=$SECURITY,nettype=$NETTYPE|| return 4
109 [ -d /r ] && $LCTL modules > /r/tmp/ogdb-`hostname`
117 do_node $client umount $force $mnt || :
118 do_node $client $LCONF --cleanup --nosetup --node client_facet $XMLCONFIG > /dev/null || :
123 if [ "$FAILURE_MODE" = HARD ]; then
124 $POWER_DOWN `facet_active_host $facet`
126 elif [ "$FAILURE_MODE" = SOFT ]; then
127 stop $facet --force --failover --nomod
133 if [ "$FAILURE_MODE" = HARD ]; then
134 $POWER_UP `facet_active_host $facet`
140 check_network $HOST 900
141 while ! do_node $HOST "ls -d $LUSTRE " > /dev/null; do sleep 5; done
146 HOST=`facet_active_host $facet`
151 # not every config has many clients
152 if [ ! -z "$CLIENTS" ]; then
153 $PDSH $CLIENTS "df $MOUNT" > /dev/null
159 echo "Failing $facet node `facet_active_host $facet`"
160 shutdown_facet $facet
164 echo "df pid is $DFPID"
166 TO=`facet_active_host $facet`
167 echo "Failover $facet to $TO"
176 do_facet $facet $LCTL --device %${facet}_svc readonly
177 do_facet $facet $LCTL --device %${facet}_svc notransno
178 do_facet $facet $LCTL mark "REPLAY BARRIER"
179 $LCTL mark "REPLAY BARRIER"
182 replay_barrier_nodf() {
185 do_facet $facet $LCTL --device %${facet}_svc readonly
186 do_facet $facet $LCTL --device %${facet}_svc notransno
187 do_facet $facet $LCTL mark "REPLAY BARRIER"
188 $LCTL mark "REPLAY BARRIER"
192 UUID=`cat /proc/fs/lustre/mdc/*_MNT_*/uuid`
193 do_facet mds "echo $UUID > /proc/fs/lustre/mds/mds1_svc/evict_client"
198 facet_failover $facet
199 df $MOUNT || error "post-failover df: $?"
205 facet_failover $facet
206 do_facet mds "echo $failcode > /proc/sys/lustre/fail_loc"
207 df $MOUNT || error "post-failover df: $?"
208 do_facet mds "echo 0 > /proc/sys/lustre/fail_loc"
213 stop $facet --force --failover --nomod
216 do_facet $facet lctl --device %${facet}_svc abort_recovery
217 df $MOUNT || echo "first df failed: $?"
219 df $MOUNT || error "post-failover df: $?"
223 $LMC -m ${XMLCONFIG} $@
227 if [ "$1" = "client" -o "$1" = "'*'" ]; then echo \'*\'; else
228 $PDSH $1 $GMNALNID -l | cut -d\ -f2
233 if [ "$1" = "client" -o "$1" = "'*'" ]; then echo \'*\'; else
240 if [ "$1" = "client" -o "$1" = "'*'" ]; then echo \'*\'; else
241 echo $1 | sed 's/[^0-9]*//g'
247 if [ "$1" = "client" -o "$1" = "'*'" ]; then echo \'*\'; else
248 echo $1 | sed 's/[^0-9]*//g'
256 varname=${facet}_HOST
262 HOST=`facet_host $facet`
263 if [ -z "$HOST" ]; then
264 echo "The env variable ${facet}_HOST must be set."
267 echo `h2$NETTYPE $HOST`
272 local activevar=${facet}active
274 if [ -f ./${facet}active ] ; then
275 source ./${facet}active
279 if [ -z "$active" ] ; then
286 facet_active_host() {
288 local active=`facet_active $facet`
289 if [ "$facet" == client ]; then
292 echo `facet_host $active`
298 failover=${facet}failover
299 host=`facet_host $failover`
300 [ -z "$host" ] && return
301 curactive=`facet_active $facet`
302 if [ -z "${curactive}" -o "$curactive" == "$failover" ] ; then
303 eval export ${facet}active=$facet
305 eval export ${facet}active=$failover
307 # save the active host for this facet
308 activevar=${facet}active
309 echo "$activevar=${!activevar}" > ./$activevar
318 $PDSH $HOST $LCTL mark "$@" > /dev/null 2>&1 || :
320 $PDSH $HOST "(PATH=$RLUSTRE/utils:$RLUSTRE/tests:/usr/sbin:/sbin:\$PATH; cd $RPWD; sh -c \"$@\")"
324 seq -f mds%g $MDSCOUNT
331 if [ "$facet" == "mds" ]; then
332 for mds in `mds_list`; do
333 HOST=`facet_active_host $mds`
337 HOST=`facet_active_host $facet`
345 echo "add facet $facet: `facet_host $facet`"
346 do_lmc --add node --node ${facet}_facet $@ --timeout $TIMEOUT \
347 --lustre_upcall $UPCALL --ptldebug $PTLDEBUG --subsystem $SUBSYSTEM
348 do_lmc --add net --node ${facet}_facet --nid `facet_nid $facet` \
357 do_lmc --add mds --node ${facet}_facet --mds ${facet}_svc --fstype $FSTYPE --backfstype $MDS_BACKFSTYPE $*
363 add_facet ${facet}failover --lustre_upcall $UPCALL
364 do_lmc --add mds --node ${facet}failover_facet --mds ${facet}_svc --fstype $FSTYPE --backfstype $MDS_BACKFSTYPE $*
372 do_lmc --add ost --node ${facet}_facet --ost ${facet}_svc --fstype $FSTYPE --backfstype $OST_BACKFSTYPE $*
378 do_lmc --delete ost --node ${facet}_facet --ost ${facet}_svc $*
385 do_facet $facet $LCONF --node ${facet}_facet --ptldebug $PTLDEBUG $* $XMLCONFIG
390 do_facet $facet $LCONF --node ${facet}_facet --cleanup $* $XMLCONFIG
398 do_lmc --add gss --gss ${facet}_svc --node ${facet}_facet $*
405 do_lmc --add cmobd --node mds1_facet --cmobd ${facet}_svc --cache_obd ${cache_facet} --master_obd ${master_facet}
413 do_lmc --add cobd --node mds1_facet --cmobd ${facet}_svc --cache_obd ${cache_facet} --master_obd ${master_facet}
419 do_lmc --deactivate ost --node ${facet}_facet --ost ${facet}_svc $*
425 add_facet ${facet}failover
426 do_lmc --add ost --failover --node ${facet}failover_facet --ost ${facet}_svc --fstype $FSTYPE --backfstype $OST_BACKFSTYPE $*
433 do_lmc --add lov --mds ${mds_facet}_svc --lov $lov $*
440 do_lmc --add lov --lmv $lmv --lov $lov $*
443 add_lov_to_cache_master_lmv() {
448 do_lmc --add lov --cachelmv $cache_lmv --masterlmv $master_lmv --lov $lov $*
454 do_lmc --add lmv --lmv $lmv $*
461 add_facet $facet --lustre_upcall $UPCALL
462 do_lmc --add mtpt --node ${facet}_facet --clientoptions async --mds ${mds}_svc $*
476 while [ $NETWORK -eq 0 ]; do
477 ping -c 1 -w 3 $1 > /dev/null
478 if [ $? -eq 0 ]; then
482 echo "waiting for $1, $((MAX - WAIT)) secs left"
485 if [ $WAIT -gt $MAX ]; then
486 echo "Network not available"
492 while( !($DSH2 $1 "netstat -tna | grep -q $2") ) ; do
503 # the sed converts spaces to commas, but leaves the last space
504 # alone, so the line doesn't end with a comma.
505 echo "$*" | tr -s " " "\n" | sort -b -u | tr "\n" " " | sed 's/ \([^$]\)/,\1/g'
509 (cd `dirname $1`; echo $PWD/`basename $1`)
512 ##################################
516 # OBD_FAIL_MDS_ALL_REQUEST_NET
518 do_facet mds "echo 0x123 > /proc/sys/lustre/fail_loc"
519 do_facet client "$1" || RC=$?
520 do_facet mds "echo 0 > /proc/sys/lustre/fail_loc"
525 # OBD_FAIL_MDS_ALL_REPLY_NET
527 do_facet mds "echo 0x122 > /proc/sys/lustre/fail_loc"
528 do_facet client "$@" || RC=$?
529 do_facet mds "echo 0 > /proc/sys/lustre/fail_loc"
534 # OBD_FAIL_MDS_REINT_NET_REP
536 do_facet mds "echo 0x119 > /proc/sys/lustre/fail_loc"
537 do_facet client "$@" || RC=$?
538 do_facet mds "echo 0 > /proc/sys/lustre/fail_loc"
543 #define OBD_FAIL_OST_BRW_PAUSE_BULK 0x214
545 do_facet ost "echo 0x214 > /proc/sys/lustre/fail_loc"
546 do_facet client "$1" || RC=$?
547 do_facet client "sync"
548 do_facet ost "echo 0 > /proc/sys/lustre/fail_loc"
553 #define OBD_FAIL_LDLM_CANCEL 0x304
555 do_facet client "echo 0x304 > /proc/sys/lustre/fail_loc"
556 do_facet client "$@" || RC=$?
557 do_facet client "echo 0 > /proc/sys/lustre/fail_loc"
562 #define OBD_FAIL_LDLM_BL_CALLBACK 0x305
564 do_facet client "echo 0x305 > /proc/sys/lustre/fail_loc"
565 do_facet client "$@" || RC=$?
566 do_facet client "echo 0 > /proc/sys/lustre/fail_loc"
574 echo "clearing fail_loc on $facet"
575 do_facet $facet "sysctl -w lustre.fail_loc=0"
579 $LCTL mark cancel_lru_locks
580 for d in /proc/fs/lustre/ldlm/namespaces/$1*; do
581 if [ -f $d/lru_size ]; then
582 echo clear > $d/lru_size
583 grep [0-9] $d/lock_unused_count
590 for a in /proc/fs/lustre/llite/*/dump_page_cache; do
591 if [ `wc -l $a | awk '{print $1}'` -gt 1 ]; then
592 echo there is still data in page cache $a ?
600 ##################################
603 echo "${TESTSUITE}: **** FAIL:" $@
608 build_test_filter() {
609 [ "$ONLY" ] && log "only running $ONLY"
613 [ "$EXCEPT$ALWAYS_EXCEPT" ] && log "skipping $EXCEPT $ALWAYS_EXCEPT"
614 for E in $EXCEPT $ALWAYS_EXCEPT; do
615 eval EXCEPT_${E}=true
624 IFS=abcdefghijklmnopqrstuvwxyz _basetest $1
628 export base=`basetest $1`
629 if [ ! -z "$ONLY" ]; then
631 if [ ${!testname}x != x ]; then
636 if [ ${!testname}x != x ]; then
644 if [ ${!testname}x != x ]; then
645 echo "skipping excluded test $1"
648 testname=EXCEPT_$base
649 if [ ${!testname}x != x ]; then
650 echo "skipping excluded test $1 (base $base)"
658 EQUALS="======================================================================"
662 local suffixlen=$((${#EQUALS} - ${#msg}))
663 [ $suffixlen -lt 5 ] && suffixlen=5
664 printf '===== %s %.*s\n' "$msg" $suffixlen $EQUALS
669 lctl mark "$*" 2> /dev/null || true
682 # Pretty tests run faster.
683 equals_msg $testnum: $message
686 log "== test $testnum: $message ============ `date +%H:%M:%S` ($BEFORE)"
687 test_${testnum} || error "test_$testnum failed with $?"
688 pass "($((`date +%s` - $BEFORE))s)"
692 (cd `dirname $1`; echo $PWD/`basename $1`)