2 # vim:expandtab:shiftwidth=4:softtabstop=4:tabstop=4:
4 trap 'echo "test-framework exiting on error"' ERR
9 export REFORMAT=${REFORMAT:-""}
11 export GMNALNID=${GMNALNID:-/usr/sbin/gmlndnid}
12 export CATASTROPHE=${CATASTROPHE:-/proc/sys/lnet/catastrophe}
13 #export PDSH="pdsh -S -Rssh -w"
15 # eg, assert_env LUSTRE MDSNODES OSTNODES CLIENTS
19 if [ -z "${!name}" ]; then
20 echo "$0: $name must be set"
24 [ $failed ] && exit 1 || true
28 echo "usage: $0 [-r] [-f cfgfile]"
35 export LUSTRE=`absolute_path $LUSTRE`
36 export TESTSUITE=`basename $0 .sh`
37 export LTESTDIR=${LTESTDIR:-$LUSTRE/../ltest}
39 [ -d /r ] && export ROOT=${ROOT:-/r}
40 export TMP=${TMP:-$ROOT/tmp}
41 export TESTSUITELOG=${TMP}/${TESTSUITE}.log
43 export PATH=:$PATH:$LUSTRE/utils:$LUSTRE/tests
44 export LCTL=${LCTL:-"$LUSTRE/utils/lctl"}
45 export LFS=${LFS:-"$LUSTRE/utils/lfs"}
46 [ ! -f "$LCTL" ] && export LCTL=$(which lctl)
47 export LFS=${LFS:-"$LUSTRE/utils/lfs"}
48 [ ! -f "$LFS" ] && export LFS=$(which lfs)
49 export MKFS=${MKFS:-"$LUSTRE/utils/mkfs.lustre"}
50 [ ! -f "$MKFS" ] && export MKFS=$(which mkfs.lustre)
51 export TUNEFS=${TUNEFS:-"$LUSTRE/utils/tunefs.lustre"}
52 [ ! -f "$TUNEFS" ] && export TUNEFS=$(which tunefs.lustre)
53 export CHECKSTAT="${CHECKSTAT:-"checkstat -v"} "
54 export FSYTPE=${FSTYPE:-"ldiskfs"}
55 export NAME=${NAME:-local}
56 export LPROC=/proc/fs/lustre
59 if [ "$ACCEPTOR_PORT" ]; then
60 export PORT_OPT="--port $ACCEPTOR_PORT"
63 # Paths on remote nodes, if different
64 export RLUSTRE=${RLUSTRE:-$LUSTRE}
65 export RPWD=${RPWD:-$PWD}
66 export I_MOUNTED=${I_MOUNTED:-"no"}
70 while getopts "rvf:" opt $*; do
73 r) REFORMAT=--reformat;;
82 [ "$TESTSUITELOG" ] && rm -f $TESTSUITELOG || true
87 2.4.*) EXT=".o"; USE_QUOTA=no; [ ! "$CLIENTONLY" ] && FSTYPE=ext3;;
88 *) EXT=".ko"; USE_QUOTA=yes;;
94 BASE=`basename $module $EXT`
95 lsmod | grep -q ${BASE} || \
96 if [ -f ${LUSTRE}/${module}${EXT} ]; then
97 insmod ${LUSTRE}/${module}${EXT} $@
99 # must be testing a "make install" or "rpm" installation
105 if [ -n "$MODPROBE" ]; then
109 if [ "$HAVE_MODULES" = true ]; then
115 echo Loading modules from $LUSTRE
116 load_module ../lnet/libcfs/libcfs
117 [ -f /etc/modprobe.conf ] && MODPROBECONF=/etc/modprobe.conf
118 [ -f /etc/modprobe.d/Lustre ] && MODPROBECONF=/etc/modprobe.d/Lustre
119 [ -z "$LNETOPTS" -a -n "$MODPROBECONF" ] && \
120 LNETOPTS=$(awk '/^options lnet/ { print $0}' $MODPROBECONF | sed 's/^options lnet //g')
121 echo "lnet options: '$LNETOPTS'"
122 # note that insmod will ignore anything in modprobe.conf
123 load_module ../lnet/lnet/lnet $LNETOPTS
124 LNETLND=${LNETLND:-"socklnd/ksocklnd"}
125 load_module ../lnet/klnds/$LNETLND
126 load_module lvfs/lvfs
127 load_module obdclass/obdclass
128 load_module ptlrpc/ptlrpc
129 [ "$USE_QUOTA" = "yes" ] && load_module quota/lquota
134 if [ -z "$CLIENTONLY" ]; then
137 [ "$FSTYPE" = "ldiskfs" ] && load_module ../ldiskfs/ldiskfs/ldiskfs
138 load_module lvfs/fsfilt_$FSTYPE
140 load_module obdfilter/obdfilter
143 load_module llite/lustre
144 load_module llite/llite_lloop
145 rm -f $TMP/ogdb-`hostname`
147 [ -d /r ] && OGDB="/r/tmp"
148 $LCTL modules > $OGDB/ogdb-`hostname`
149 # 'mount' doesn't look in $PATH, just sbin
150 [ -f $LUSTRE/utils/mount.lustre ] && cp $LUSTRE/utils/mount.lustre /sbin/. || true
154 if [ `uname -r | cut -c 3` -eq 4 ]; then
162 MODULES=$($LCTL modules | awk '{ print $2 }')
163 while [ -n "$MODULES" ]; do
165 $RMMOD $MODULES >/dev/null 2>&1 || true
166 MODULES=$($LCTL modules | awk '{ print $2 }')
167 if [ -z "$MODULES" ]; then
171 echo "waiting, $((MAX - WAIT)) secs left"
173 if [ $WAIT -eq $MAX ]; then
174 echo "LNET modules $MODULES will not unload"
182 lsmod | grep lnet > /dev/null && $LCTL dl && $LCTL dk $TMP/debug
183 local MODULES=$($LCTL modules | awk '{ print $2 }')
184 $RMMOD $MODULES > /dev/null 2>&1 || true
185 # do it again, in case we tried to unload ksocklnd too early
186 MODULES=$($LCTL modules | awk '{ print $2 }')
187 [ -n "$MODULES" ] && $RMMOD $MODULES > /dev/null 2>&1 || true
188 MODULES=$($LCTL modules | awk '{ print $2 }')
189 if [ -n "$MODULES" ]; then
190 echo "Modules still loaded: "
192 if [ -e $LPROC ]; then
193 echo "Lustre still loaded"
194 cat $LPROC/devices || true
198 echo "Lustre stopped but LNET is still loaded, waiting..."
199 wait_for_lnet || return 3
204 LEAK_LUSTRE=$(dmesg | tail -n 30 | grep "obd mem.*leaked" || true)
205 LEAK_PORTALS=$(dmesg | tail -n 20 | grep "Portals memory leaked" || true)
206 if [ "$LEAK_LUSTRE" -o "$LEAK_PORTALS" ]; then
207 echo "$LEAK_LUSTRE" 1>&2
208 echo "$LEAK_PORTALS" 1>&2
209 mv $TMP/debug $TMP/debug-leak.`date +%s` || true
210 echo "Memory leaks detected"
211 [ -n "$IGNORE_LEAK" ] && echo "ignoring leaks" && return 0
214 echo "modules unloaded."
219 # start facet device options
225 echo "Starting ${facet}: $@ ${device} ${MOUNT%/*}/${facet}"
226 do_facet ${facet} mkdir -p ${MOUNT%/*}/${facet}
227 do_facet ${facet} mount -t lustre $@ ${device} ${MOUNT%/*}/${facet}
229 if [ $RC -ne 0 ]; then
230 echo "mount -t lustre $@ ${device} ${MOUNT%/*}/${facet}"
231 echo "Start of ${device} on ${facet} failed ${RC}"
233 do_facet ${facet} sync
234 label=$(do_facet ${facet} "e2label ${device}")
235 [ -z "$label" ] && echo no label for ${device} && exit 1
236 eval export ${facet}_svc=${label}
237 eval export ${facet}_dev=${device}
238 eval export ${facet}_opt=\"$@\"
239 echo Started ${label}
248 HOST=`facet_active_host $facet`
249 [ -z $HOST ] && echo stop: no host for $facet && return 0
251 running=$(do_facet ${facet} "grep -c ${MOUNT%/*}/${facet}' ' /proc/mounts") || true
252 if [ ${running} -ne 0 ]; then
253 echo "Stopping ${MOUNT%/*}/${facet} (opts:$@)"
254 do_facet ${facet} umount -d $@ ${MOUNT%/*}/${facet}
257 # umount should block, but we should wait for unrelated obd's
258 # like the MGS or MGC to also stop.
261 # conf-sanity 31 takes a long time cleanup
262 while [ $WAIT -lt 300 ]; do
263 running=$(do_facet ${facet} "[ -e $LPROC ] && grep ST' ' $LPROC/devices") || true
264 if [ -z "${running}" ]; then
267 echo "waited $WAIT for${running}"
268 if [ $INTERVAL -lt 64 ]; then
269 INTERVAL=$((INTERVAL + INTERVAL))
272 WAIT=$((WAIT + INTERVAL))
274 echo "service didn't stop after $WAIT seconds. Still running:"
283 # Only supply -o to mount if we have options
284 if [ -n "$MOUNTOPT" ]; then
285 OPTIONS="-o $MOUNTOPT"
287 local device=$MGSNID:/$FSNAME
288 if [ -z "$mnt" -o -z "$FSNAME" ]; then
289 echo Bad zconf mount command: opt=$OPTIONS dev=$device mnt=$mnt
293 echo "Starting client: $OPTIONS $device $mnt"
294 do_node $client mkdir -p $mnt
295 do_node $client mount -t lustre $OPTIONS $device $mnt || return 1
297 do_node $client "sysctl -w lnet.debug=$PTLDEBUG; sysctl -w lnet.subsystem_debug=${SUBSYSTEM# }"
298 [ -d /r ] && $LCTL modules > /r/tmp/ogdb-`hostname`
306 local running=$(do_node $client "grep -c $mnt' ' /proc/mounts") || true
307 if [ $running -ne 0 ]; then
308 echo "Stopping client $mnt (opts:$force)"
309 do_node $client umount $force $mnt
315 if [ "$FAILURE_MODE" = HARD ]; then
316 $POWER_DOWN `facet_active_host $facet`
318 elif [ "$FAILURE_MODE" = SOFT ]; then
325 if [ "$FAILURE_MODE" = HARD ]; then
326 $POWER_UP `facet_active_host $facet`
332 # verify that lustre actually cleaned up properly
334 [ -f $CATASTROPHE ] && [ `cat $CATASTROPHE` -ne 0 ] && \
335 error "LBUG/LASSERT detected"
336 BUSY=`dmesg | grep -i destruct || true`
339 [ -e $TMP/debug ] && mv $TMP/debug $TMP/debug-busy.`date +%s`
342 LEAK_LUSTRE=`dmesg | tail -n 30 | grep "obd mem.*leaked" || true`
343 LEAK_PORTALS=`dmesg | tail -n 20 | grep "Portals memory leaked" || true`
344 if [ "$LEAK_LUSTRE" -o "$LEAK_PORTALS" ]; then
345 echo "$0: $LEAK_LUSTRE" 1>&2
346 echo "$0: $LEAK_PORTALS" 1>&2
347 echo "$0: Memory leak(s) detected..." 1>&2
348 mv $TMP/debug $TMP/debug-leak.`date +%s`
352 [ "`lctl dl 2> /dev/null | wc -l`" -gt 0 ] && lctl dl && \
353 echo "$0: lustre didn't clean up..." 1>&2 && return 202 || true
355 if [ "`/sbin/lsmod 2>&1 | egrep 'lnet|libcfs'`" ]; then
356 echo "$0: modules still loaded..." 1>&2
363 wait_delete_completed () {
364 local TOTALPREV=`awk 'BEGIN{total=0}; {total+=$1}; END{print total}' \
365 $LPROC/osc/*/kbytesavail`
369 while [ "$WAIT" -ne "$MAX_WAIT" ]; do
371 TOTAL=`awk 'BEGIN{total=0}; {total+=$1}; END{print total}' \
372 $LPROC/osc/*/kbytesavail`
373 [ "$TOTAL" -eq "$TOTALPREV" ] && break
374 echo "Waiting delete completed ... prev: $TOTALPREV current: $TOTAL "
378 echo "Delete completed."
383 check_network "$HOST" 900
384 while ! do_node $HOST "ls -d $LUSTRE " > /dev/null; do sleep 5; done
389 HOST=`facet_active_host $facet`
393 wait_mds_recovery_done () {
394 local timeout=`do_facet mds cat /proc/sys/lustre/timeout`
395 #define OBD_RECOVERY_TIMEOUT (obd_timeout * 5 / 2)
396 # as we are in process of changing obd_timeout in different ways
397 # let's set MAX longer than that
398 MAX=$(( timeout * 4 ))
400 while [ $WAIT -lt $MAX ]; do
401 STATUS=`do_facet mds grep status /proc/fs/lustre/mds/*-MDT*/recovery_status`
402 echo $STATUS | grep COMPLETE && return 0
405 echo "Waiting $(($MAX - $WAIT)) secs for MDS recovery done"
407 echo "MDS recovery not done in $MAX sec"
412 # not every config has many clients
413 if [ -n "$CLIENTS" ]; then
414 $PDSH $CLIENTS "df $MOUNT" > /dev/null
416 df $MOUNT > /dev/null
421 uname -n >> $MOUNT/recon
422 if [ ! -z "$CLIENTS" ]; then
423 $PDSH $CLIENTS "df $MOUNT; uname -n >> $MOUNT/recon" > /dev/null
425 echo Connected clients:
427 ls -l $MOUNT/recon > /dev/null
433 echo "Failing $facet on node `facet_active_host $facet`"
434 shutdown_facet $facet
438 echo "df pid is $DFPID"
440 TO=`facet_active_host $facet`
441 echo "Failover $facet to $TO"
443 local dev=${facet}_dev
444 local opt=${facet}_opt
445 start $facet ${!dev} ${!opt} || error "Restart of $facet failed"
456 local svc=${facet}_svc
457 do_facet $facet $LCTL --device %${!svc} readonly
458 do_facet $facet $LCTL --device %${!svc} notransno
459 do_facet $facet $LCTL mark "$facet REPLAY BARRIER on ${!svc}"
460 $LCTL mark "local REPLAY BARRIER on ${!svc}"
463 replay_barrier_nodf() {
464 local facet=$1 echo running=${running}
466 local svc=${facet}_svc
467 echo Replay barrier on ${!svc}
468 do_facet $facet $LCTL --device %${!svc} readonly
469 do_facet $facet $LCTL --device %${!svc} notransno
470 do_facet $facet $LCTL mark "$facet REPLAY BARRIER on ${!svc}"
471 $LCTL mark "local REPLAY BARRIER on ${!svc}"
475 UUID=`cat /proc/fs/lustre/mdc/${mds_svc}-mdc-*/uuid`
476 do_facet mds "echo $UUID > /proc/fs/lustre/mds/${mds_svc}/evict_client"
480 UUID=`cat /proc/fs/lustre/osc/${ost1_svc}-osc-*/uuid`
481 do_facet ost1 "echo $UUID > /proc/fs/lustre/obdfilter/${ost1_svc}/evict_client"
485 facet_failover $* || error "failover: $?"
486 df $MOUNT || error "post-failover df: $?"
493 local svc=${facet}_svc
494 local dev=${facet}_dev
495 local opt=${facet}_opt
496 start $facet ${!dev} ${!opt}
497 do_facet $facet lctl --device %${!svc} abort_recovery
498 df $MOUNT || echo "first df failed: $?"
500 df $MOUNT || error "post-failover df: $?"
504 echo There is no lmc. This is mountconf, baby.
509 if [ "$1" = "client" -o "$1" = "'*'" ]; then echo \'*\'; else
510 ID=`$PDSH $1 $GMNALNID -l | cut -d\ -f2`
516 if [ "$1" = "client" -o "$1" = "'*'" ]; then echo \'*\'; else
517 ID=`xtprocadmin -n $1 2>/dev/null | egrep -v 'NID' | awk '{print $1}'`
518 if [ -z "$ID" ]; then
519 echo "Could not get a ptl id for $1..."
528 if [ "$1" = "client" -o "$1" = "'*'" ]; then echo \'*\'; else
535 if [ "$1" = "client" -o "$1" = "'*'" ]; then echo \'*\'; else
536 if type __h2elan >/dev/null 2>&1; then
539 ID=`echo $1 | sed 's/[^0-9]*//g'`
547 if [ "$1" = "client" -o "$1" = "'*'" ]; then echo \'*\'; else
548 ID=`echo $1 | sed 's/[^0-9]*//g'`
556 varname=${facet}_HOST
557 if [ -z "${!varname}" ]; then
558 if [ "${facet:0:3}" == "ost" ]; then
559 eval ${facet}_HOST=${ost_HOST}
567 local activevar=${facet}active
569 if [ -f ./${facet}active ] ; then
570 source ./${facet}active
574 if [ -z "$active" ] ; then
581 facet_active_host() {
583 local active=`facet_active $facet`
584 if [ "$facet" == client ]; then
587 echo `facet_host $active`
593 failover=${facet}failover
594 host=`facet_host $failover`
595 [ -z "$host" ] && return
596 curactive=`facet_active $facet`
597 if [ -z "${curactive}" -o "$curactive" == "$failover" ] ; then
598 eval export ${facet}active=$facet
600 eval export ${facet}active=$failover
602 # save the active host for this facet
603 activevar=${facet}active
604 echo "$activevar=${!activevar}" > ./$activevar
611 if [ "$HOST" = "$(hostname)" ]; then
615 echo "CMD: $HOST $@" >&2
616 $myPDSH $HOST $LCTL mark "$@" > /dev/null 2>&1 || :
618 $myPDSH $HOST "(PATH=\$PATH:$RLUSTRE/utils:$RLUSTRE/tests:/sbin:/usr/sbin; cd $RPWD; sh -c \"$@\")" | sed "s/^${HOST}: //"
619 return ${PIPESTATUS[0]}
625 HOST=`facet_active_host $facet`
626 [ -z $HOST ] && echo No host defined for facet ${facet} && exit 1
633 # make sure its not already running
636 do_facet ${facet} $MKFS $*
642 #if $OSTDEVn isn't defined, default is $OSTDEVBASE + num
643 eval DEVPTR=${!DEVNAME:=${OSTDEVBASE}${num}}
651 # make sure we are using the primary server, so test-framework will
652 # be able to clean up properly.
653 activemds=`facet_active mds`
654 if [ $activemds != "mds" ]; then
658 # assume client mount is local
659 grep " $MOUNT " /proc/mounts && zconf_umount `hostname` $MOUNT $*
660 grep " $MOUNT2 " /proc/mounts && zconf_umount `hostname` $MOUNT2 $*
661 [ "$CLIENTONLY" ] && return
663 for num in `seq $OSTCOUNT`; do
675 [ "$FSTYPE" ] && FSTYPE_OPT="--backfstype $FSTYPE"
678 # We need ldiskfs here, may as well load them all
680 [ "$CLIENTONLY" ] && return
681 echo Formatting mds, osts
683 add mds $MDS_MKFS_OPTS $FSTYPE_OPT --reformat $MDSDEV || exit 10
685 add mds $MDS_MKFS_OPTS $FSTYPE_OPT --reformat $MDSDEV > /dev/null || exit 10
688 for num in `seq $OSTCOUNT`; do
690 add ost$num $OST_MKFS_OPTS $FSTYPE_OPT --reformat `ostdevname $num` || exit 10
692 add ost$num $OST_MKFS_OPTS $FSTYPE_OPT --reformat `ostdevname $num` > /dev/null || exit 10
698 grep " $1 " /proc/mounts || zconf_mount `hostname` $*
703 if [ -z "$CLIENTONLY" ]; then
705 echo $REFORMAT | grep -q "reformat" \
706 || do_facet mds "$TUNEFS --writeconf $MDSDEV"
707 start mds $MDSDEV $MDS_MOUNT_OPTS
708 for num in `seq $OSTCOUNT`; do
709 DEVNAME=`ostdevname $num`
710 start ost$num $DEVNAME $OST_MOUNT_OPTS
713 [ "$DAEMONFILE" ] && $LCTL debug_daemon start $DAEMONFILE $DAEMONSIZE
715 if [ "$MOUNT_2" ]; then
721 mounted_lustre_filesystems() {
722 awk '($3 ~ "lustre" && $1 ~ ":") { print $2 }' /proc/mounts
725 check_and_setup_lustre() {
726 MOUNTED="`mounted_lustre_filesystems`"
727 if [ -z "$MOUNTED" ]; then
728 [ "$REFORMAT" ] && formatall
730 MOUNTED="`mounted_lustre_filesystems`"
731 [ -z "$MOUNTED" ] && error "NAME=$NAME not mounted"
734 if [ "$ONLY" == "setup" ]; then
739 cleanup_and_setup_lustre() {
740 if [ "$ONLY" == "cleanup" -o "`mount | grep $MOUNT`" ]; then
741 sysctl -w lnet.debug=0 || true
743 if [ "$ONLY" == "cleanup" ]; then
747 check_and_setup_lustre
750 check_and_cleanup_lustre() {
751 if [ "`mount | grep $MOUNT`" ]; then
752 rm -rf $DIR/[Rdfs][1-9]*
754 if [ "$I_MOUNTED" = "yes" ]; then
755 cleanupall -f || error "cleanup failed"
767 while [ $NETWORK -eq 0 ]; do
768 ping -c 1 -w 3 $1 > /dev/null
769 if [ $? -eq 0 ]; then
773 echo "waiting for $1, $((MAX - WAIT)) secs left"
776 if [ $WAIT -gt $MAX ]; then
777 echo "Network not available"
783 while( !($DSH2 $1 "netstat -tna | grep -q $2") ) ; do
794 # the sed converts spaces to commas, but leaves the last space
795 # alone, so the line doesn't end with a comma.
796 echo "$*" | tr -s " " "\n" | sort -b -u | tr "\n" " " | sed 's/ \([^$]\)/,\1/g'
800 (cd `dirname $1`; echo $PWD/`basename $1`)
803 ##################################
807 # OBD_FAIL_MDS_ALL_REQUEST_NET
809 do_facet mds sysctl -w lustre.fail_loc=0x123
810 do_facet client "$1" || RC=$?
811 do_facet mds sysctl -w lustre.fail_loc=0
816 # OBD_FAIL_MDS_ALL_REPLY_NET
818 do_facet mds sysctl -w lustre.fail_loc=0x122
819 do_facet client "$@" || RC=$?
820 do_facet mds sysctl -w lustre.fail_loc=0
825 # OBD_FAIL_MDS_REINT_NET_REP
827 do_facet mds sysctl -w lustre.fail_loc=0x119
828 do_facet client "$@" || RC=$?
829 do_facet mds sysctl -w lustre.fail_loc=0
834 #define OBD_FAIL_OST_BRW_PAUSE_BULK 0x214
836 do_facet ost1 sysctl -w lustre.fail_loc=0x214
837 do_facet client "$1" || RC=$?
838 do_facet client "sync"
839 do_facet ost1 sysctl -w lustre.fail_loc=0
844 #define OBD_FAIL_LDLM_CANCEL 0x304
846 do_facet client sysctl -w lustre.fail_loc=0x304
847 do_facet client "$@" || RC=$?
848 do_facet client sysctl -w lustre.fail_loc=0
853 #define OBD_FAIL_LDLM_BL_CALLBACK 0x305
855 do_facet client sysctl -w lustre.fail_loc=0x305
856 do_facet client "$@" || RC=$?
857 do_facet client sysctl -w lustre.fail_loc=0
862 #define OBD_FAIL_LDLM_REPLY 0x30c
864 do_facet mds sysctl -w lustre.fail_loc=0x30c
865 do_facet client "$@" || RC=$?
866 do_facet mds sysctl -w lustre.fail_loc=0
874 echo "clearing fail_loc on $facet"
875 do_facet $facet "sysctl -w lustre.fail_loc=0"
879 $LCTL mark "cancel_lru_locks $1 start"
880 for d in `find $LPROC/ldlm/namespaces | egrep -i $1`; do
881 [ -f $d/lru_size ] && echo clear > $d/lru_size
882 [ -f $d/lock_unused_count ] && grep [1-9] $d/lock_unused_count /dev/null
884 $LCTL mark "cancel_lru_locks $1 stop"
889 for a in /proc/fs/lustre/llite/*/dump_page_cache; do
890 if [ `wc -l $a | awk '{print $1}'` -gt 1 ]; then
891 echo there is still data in page cache $a ?
900 DEBUGSAVE="$(sysctl -n lnet.debug)"
904 [ -n "$DEBUGSAVE" ] && sysctl -w lnet.debug="${DEBUGSAVE}"
908 ##################################
910 ##################################
913 local FAIL_ON_ERROR=${FAIL_ON_ERROR:-true}
914 local TYPE=${TYPE:-"FAIL"}
916 sysctl -w lustre.fail_loc=0 2> /dev/null || true
917 log "${TESTSUITE} ${TESTNAME}: **** ${TYPE}:" $@
918 ERRLOG=$TMP/lustre_${TESTSUITE}_${TESTNAME}.$(date +%s)
919 echo "Dumping lctl log to $ERRLOG"
920 # We need to dump the logs on all nodes
922 [ ! "$mds_HOST" = "$(hostname)" ] && do_node $mds_HOST $LCTL dk $ERRLOG
923 [ ! "$ost_HOST" = "$(hostname)" -a ! "$ost_HOST" = "$mds_HOST" ] && do_node $ost_HOST $LCTL dk $ERRLOG
925 [ "$TESTSUITELOG" ] && echo "$0: ${TYPE}: $TESTNAME $@" >> $TESTSUITELOG
926 if $FAIL_ON_ERROR; then
931 # use only if we are ignoring failures for this test, bugno required.
932 # (like ALWAYS_EXCEPT, but run the test and ignore the results.)
933 # e.g. error_ignore 5494 "your message"
935 FAIL_ON_ERROR=false TYPE="IGNORE (bz$1)" error $2
939 log " SKIP: ${TESTSUITE} ${TESTNAME} $@"
940 [ "$TESTSUITELOG" ] && echo "${TESTSUITE}: SKIP: $TESTNAME $@" >> $TESTSUITELOG
943 build_test_filter() {
944 [ "$ONLY" ] && log "only running test `echo $ONLY`"
948 [ "$EXCEPT$ALWAYS_EXCEPT" ] && \
949 log "skipping tests: `echo $EXCEPT $ALWAYS_EXCEPT`"
950 for E in $EXCEPT $ALWAYS_EXCEPT; do
951 eval EXCEPT_${E}=true
953 for G in $GRANT_CHECK_LIST; do
954 eval GCHECK_ONLY_${G}=true
963 IFS=abcdefghijklmnopqrstuvwxyz _basetest $1
967 export base=`basetest $1`
968 if [ ! -z "$ONLY" ]; then
970 if [ ${!testname}x != x ]; then
975 if [ ${!testname}x != x ]; then
983 if [ ${!testname}x != x ]; then
984 TESTNAME=test_$1 skip "skipping excluded test $1"
987 testname=EXCEPT_$base
988 if [ ${!testname}x != x ]; then
989 TESTNAME=test_$1 skip "skipping excluded test $1 (base $base)"
997 EQUALS="======================================================================"
1001 local suffixlen=$((${#EQUALS} - ${#msg}))
1002 [ $suffixlen -lt 5 ] && suffixlen=5
1003 printf '===== %s %.*s\n' "$msg" $suffixlen $EQUALS
1008 lsmod | grep lnet > /dev/null || load_modules
1009 $LCTL mark "$*" 2> /dev/null || true
1014 strace -o $TMP/$1.strace -ttt $*
1016 log "FINISHED: $*: rc $RC"
1025 FFREE=`cat /proc/fs/lustre/mds/*/filesfree`
1026 FTOTAL=`cat /proc/fs/lustre/mds/*/filestotal`
1027 [ $FFREE -ge $FTOTAL ] && error "files free $FFREE > total $FTOTAL" || true
1034 export tdir=d${base}
1037 log "== test $testnum: $message ============ `date +%H:%M:%S` ($BEFORE)"
1039 export TESTNAME=test_$testnum
1040 test_${testnum} || error "test_$testnum failed with $?"
1042 check_grant ${testnum} || error "check_grant $testnum failed with $?"
1043 [ -f $CATASTROPHE ] && [ `cat $CATASTROPHE` -ne 0 ] && \
1044 error "LBUG/LASSERT detected"
1045 pass "($((`date +%s` - $BEFORE))s)"
1053 (cd `dirname $1`; echo $PWD/`basename $1`)
1057 [ -d $DIR1 ] && cd $DIR1 && sync; sleep 1; sync
1058 [ -d $DIR2 ] && cd $DIR2 && sync; sleep 1; sync
1063 export base=`basetest $1`
1064 [ "$CHECK_GRANT" == "no" ] && return 0
1066 testname=GCHECK_ONLY_${base}
1067 [ ${!testname}x == x ] && return 0
1069 echo -n "checking grant......"
1071 # write some data to sync client lost_grant
1072 rm -f $DIR1/${tfile}_check_grant_* 2>&1
1073 for i in `seq $OSTCOUNT`; do
1074 $LFS setstripe $DIR1/${tfile}_check_grant_$i 0 $(($i -1)) 1
1075 dd if=/dev/zero of=$DIR1/${tfile}_check_grant_$i bs=4k \
1076 count=1 > /dev/null 2>&1
1078 # sync all the data and make sure no pending data on server
1081 #get client grant and server grant
1083 for d in ${LPROC}/osc/*/cur_grant_bytes; do
1084 client_grant=$((client_grant + `cat $d`))
1087 for d in ${LPROC}/obdfilter/*/tot_granted; do
1088 server_grant=$((server_grant + `cat $d`))
1091 # cleanup the check_grant file
1092 for i in `seq $OSTCOUNT`; do
1093 rm $DIR1/${tfile}_check_grant_$i
1096 #check whether client grant == server grant
1097 if [ $client_grant != $server_grant ]; then
1098 echo "failed: client:${client_grant} server: ${server_grant}"
1105 ########################
1111 ost=`echo $1 | awk -F_ '{print $3}'`
1112 if [ -z $ost ]; then
1113 ost=`echo $1 | sed 's/-osc.*//'`
1120 [ ! -e /proc/fs/lustre/mds/*MDT* ]
1125 [ $(grep -c obdfilter $LPROC/devices) -eq 0 ]
1130 grep -q patchless $LPROC/version