+
+ shift $((OPTIND - 1))
+ ONLY=${ONLY:-$*}
+
+ [ "$TESTSUITELOG" ] && rm -f $TESTSUITELOG || true
+
+}
+
+load_module() {
+ EXT=".ko"
+ module=$1
+ shift
+ BASE=`basename $module $EXT`
+ lsmod | grep -q ${BASE} || \
+ if [ -f ${LUSTRE}/${module}${EXT} ]; then
+ insmod ${LUSTRE}/${module}${EXT} $@
+ else
+ # must be testing a "make install" or "rpm" installation
+ # note failed to load ptlrpc_gss is considered not fatal
+ if [ "$BASE" == "ptlrpc_gss" ]; then
+ modprobe $BASE $@ 2>/dev/null || echo "gss/krb5 is not supported"
+ else
+ modprobe $BASE $@
+ fi
+ fi
+}
+
+load_modules() {
+ if [ -n "$MODPROBE" ]; then
+ # use modprobe
+ return 0
+ fi
+ if [ "$HAVE_MODULES" = true ]; then
+ # we already loaded
+ return 0
+ fi
+ HAVE_MODULES=true
+
+ echo Loading modules from $LUSTRE
+ load_module ../lnet/libcfs/libcfs
+ [ -z "$LNETOPTS" ] && \
+ LNETOPTS=$(awk '/^options lnet/ { print $0}' /etc/modprobe.conf | sed 's/^options lnet //g')
+ echo "lnet options: '$LNETOPTS'"
+ # note that insmod will ignore anything in modprobe.conf
+ load_module ../lnet/lnet/lnet $LNETOPTS
+ LNETLND=${LNETLND:-"socklnd/ksocklnd"}
+ load_module ../lnet/klnds/$LNETLND
+ [ "$FSTYPE" = "ldiskfs" ] && load_module ../ldiskfs/ldiskfs/ldiskfs
+ load_module lvfs/lvfs
+ load_module obdclass/obdclass
+ load_module lvfs/fsfilt_$FSTYPE
+ load_module ptlrpc/ptlrpc
+ load_module ptlrpc/gss/ptlrpc_gss
+ # Now, some modules depend on lquota without USE_QUOTA check,
+ # will fix later. Disable check "$USE_QUOTA" = "yes" temporary.
+ #[ "$USE_QUOTA" = "yes" ] && load_module quota/lquota
+ load_module quota/lquota
+ load_module fid/fid
+ load_module fld/fld
+ load_module lmv/lmv
+ load_module mdc/mdc
+ load_module osc/osc
+ load_module lov/lov
+ load_module mds/mds
+ load_module mdd/mdd
+ load_module mdt/mdt
+ load_module cmm/cmm
+ load_module osd/osd
+ load_module ost/ost
+ load_module obdfilter/obdfilter
+ load_module llite/lustre
+ load_module llite/llite_lloop
+ load_module mgc/mgc
+ load_module mgs/mgs
+ rm -f $TMP/ogdb-`hostname`
+ $LCTL modules > $TMP/ogdb-`hostname`
+ # 'mount' doesn't look in $PATH, just sbin
+ [ -f $LUSTRE/utils/mount.lustre ] && cp $LUSTRE/utils/mount.lustre /sbin/. || true
+}
+
+RMMOD=rmmod
+if [ `uname -r | cut -c 3` -eq 4 ]; then
+ RMMOD="modprobe -r"
+fi
+
+wait_for_lnet() {
+ local UNLOADED=0
+ local WAIT=0
+ local MAX=60
+ MODULES=$($LCTL modules | awk '{ print $2 }')
+ while [ -n "$MODULES" ]; do
+ sleep 5
+ $RMMOD $MODULES > /dev/null 2>&1 || true
+ MODULES=$($LCTL modules | awk '{ print $2 }')
+ if [ -z "$MODULES" ]; then
+ return 0
+ else
+ WAIT=$((WAIT + 5))
+ echo "waiting, $((MAX - WAIT)) secs left"
+ fi
+ if [ $WAIT -eq $MAX ]; then
+ echo "LNET modules $MODULES will not unload"
+ lsmod
+ return 3
+ fi
+ done
+}
+
+unload_modules() {
+ lsmod | grep lnet > /dev/null && $LCTL dl && $LCTL dk $TMP/debug
+ local MODULES=$($LCTL modules | awk '{ print $2 }')
+ $RMMOD $MODULES > /dev/null 2>&1 || true
+ # do it again, in case we tried to unload ksocklnd too early
+ MODULES=$($LCTL modules | awk '{ print $2 }')
+ [ -n "$MODULES" ] && $RMMOD $MODULES > /dev/null 2>&1 || true
+ MODULES=$($LCTL modules | awk '{ print $2 }')
+ if [ -n "$MODULES" ]; then
+ echo "Modules still loaded: "
+ echo $MODULES
+ if [ -e $LPROC ]; then
+ echo "Lustre still loaded"
+ cat $LPROC/devices || true
+ lsmod
+ return 2
+ else
+ echo "Lustre stopped but LNET is still loaded, waiting..."
+ wait_for_lnet || return 3
+ fi
+ fi
+ HAVE_MODULES=false
+
+ LEAK_LUSTRE=$(dmesg | tail -n 30 | grep "obd mem.*leaked" || true)
+ LEAK_PORTALS=$(dmesg | tail -n 20 | grep "Portals memory leaked" || true)
+ if [ "$LEAK_LUSTRE" -o "$LEAK_PORTALS" ]; then
+ echo "$LEAK_LUSTRE" 1>&2
+ echo "$LEAK_PORTALS" 1>&2
+ mv $TMP/debug $TMP/debug-leak.`date +%s` || true
+ echo "Memory leaks detected"
+ return 254
+ fi
+ echo "modules unloaded."
+ return 0
+}
+
+check_gss_daemon_facet() {
+ facet=$1
+ dname=$2
+
+ num=`do_facet $facet ps -o cmd -C $dname | grep $dname | wc -l`
+ if [ $num -ne 1 ]; then
+ echo "$num instance of $dname on $facet"
+ return 1
+ fi
+ return 0
+}
+
+send_sigint() {
+ local facet=$1
+ shift
+ do_facet $facet "killall -2 $@ 2>/dev/null || true"
+}
+
+start_gss_daemons() {
+ # starting on MDT
+ for num in `seq $MDSCOUNT`; do
+ do_facet mds$num "$LSVCGSSD -v"
+ if $GSS_PIPEFS; then
+ do_facet mds$num "$LGSSD -v"
+ fi
+ done
+ # starting on OSTs
+ for num in `seq $OSTCOUNT`; do
+ do_facet ost$num "$LSVCGSSD -v"
+ done
+ # starting on client
+ # FIXME: is "client" the right facet name?
+ if $GSS_PIPEFS; then
+ do_facet client "$LGSSD -v"
+ fi
+
+ # wait daemons entering "stable" status
+ sleep 5
+
+ #
+ # check daemons are running
+ #
+ for num in `seq $MDSCOUNT`; do
+ check_gss_daemon_facet mds$num lsvcgssd
+ if $GSS_PIPEFS; then
+ check_gss_daemon_facet mds$num lgssd
+ fi
+ done
+ for num in `seq $OSTCOUNT`; do
+ check_gss_daemon_facet ost$num lsvcgssd
+ done
+ if $GSS_PIPEFS; then
+ check_gss_daemon_facet client lgssd
+ fi
+}
+
+stop_gss_daemons() {
+ for num in `seq $MDSCOUNT`; do
+ send_sigint mds$num lsvcgssd lgssd
+ done
+ for num in `seq $OSTCOUNT`; do
+ send_sigint ost$num lsvcgssd
+ done
+ send_sigint client lgssd
+}
+
+init_krb5_env() {
+ if [ ! -z $SEC ]; then
+ MDS_MOUNT_OPTS=$MDS_MOUNT_OPTS,sec=$SEC
+ OST_MOUNT_OPTS=$OST_MOUNT_OPTS,sec=$SEC
+ fi
+
+ if $GSS; then
+ start_gss_daemons
+ fi
+}
+
+cleanup_krb5_env() {
+ if $GSS; then
+ stop_gss_daemons
+ # maybe cleanup credential cache?
+ fi
+}
+
+mdsdevlabel() {
+ local num=$1
+ local device=`mdsdevname $num`
+ local label=`do_facet mds$num "e2label ${device}" | grep -v "CMD: "`
+ echo -n $label
+}
+
+ostdevlabel() {
+ local num=$1
+ local device=`ostdevname $num`
+ local label=`do_facet ost$num "e2label ${device}" | grep -v "CMD: "`
+ echo -n $label