X-Git-Url: https://git.whamcloud.com/?a=blobdiff_plain;f=lustre%2Ftests%2Ftest-framework.sh;h=df2293684d015fe0bef6895d150c1e4033ae4647;hb=ddd0d41173eeb500a4a16495fefe5568782252cf;hp=2803ee169e5bd3a333b3b7e2910a493973bcb8c1;hpb=113303973ec9f8484eb2355a1a6ef3c4c7fd6a56;p=fs%2Flustre-release.git diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh index 2803ee1..df22936 100644 --- a/lustre/tests/test-framework.sh +++ b/lustre/tests/test-framework.sh @@ -6,10 +6,14 @@ set -e #set -x -export REFORMAT="" +export REFORMAT=${REFORMAT:-""} export VERBOSE=false export GMNALNID=${GMNALNID:-/usr/sbin/gmlndnid} export CATASTROPHE=${CATASTROPHE:-/proc/sys/lnet/catastrophe} +export GSS=false +export GSS_KRB5=false +export GSS_PIPEFS=false +#export PDSH="pdsh -S -Rssh -w" # eg, assert_env LUSTRE MDSNODES OSTNODES CLIENTS assert_env() { @@ -37,23 +41,43 @@ init_test_env() { [ -d /r ] && export ROOT=${ROOT:-/r} export TMP=${TMP:-$ROOT/tmp} + export TESTSUITELOG=${TMP}/${TESTSUITE}.log - export PATH=:$PATH:$LUSTRE/utils:$LUSTRE/tests + export PATH=:$PATH:$LUSTRE/utils:$LUSTRE/utils/gss:$LUSTRE/tests export LCTL=${LCTL:-"$LUSTRE/utils/lctl"} + export LFS=${LFS:-"$LUSTRE/utils/lfs"} + [ ! -f "$LCTL" ] && export LCTL=$(which lctl) + export LFS=${LFS:-"$LUSTRE/utils/lfs"} + [ ! -f "$LFS" ] && export LFS=$(which lfs) export MKFS=${MKFS:-"$LUSTRE/utils/mkfs.lustre"} + [ ! -f "$MKFS" ] && export MKFS=$(which mkfs.lustre) export TUNEFS=${TUNEFS:-"$LUSTRE/utils/tunefs.lustre"} - export CHECKSTAT="${CHECKSTAT:-checkstat} " - export FSYTPE=${FSTYPE:-"ext3"} + [ ! -f "$TUNEFS" ] && export TUNEFS=$(which tunefs.lustre) + export CHECKSTAT="${CHECKSTAT:-"checkstat -v"} " + export FSYTPE=${FSTYPE:-"ldiskfs"} export NAME=${NAME:-local} export LPROC=/proc/fs/lustre + export LGSSD=${LGSSD:-"$LUSTRE/utils/gss/lgssd"} + export LSVCGSSD=${LSVCGSSD:-"$LUSTRE/utils/gss/lsvcgssd"} + export KRB5DIR=${KRB5DIR:-"/usr/kerberos"} + export DIR2 if [ "$ACCEPTOR_PORT" ]; then export PORT_OPT="--port $ACCEPTOR_PORT" fi + case "x$SEC" in + xkrb5*) + echo "Using GSS/krb5 ptlrpc security flavor" + GSS=true + GSS_KRB5=true + ;; + esac + # Paths on remote nodes, if different export RLUSTRE=${RLUSTRE:-$LUSTRE} export RPWD=${RPWD:-$PWD} + export I_MOUNTED=${I_MOUNTED:-"no"} # command line @@ -68,14 +92,13 @@ init_test_env() { shift $((OPTIND - 1)) ONLY=${ONLY:-$*} -} -case `uname -r` in -2.4.*) EXT=".o"; USE_QUOTA=no; FSTYPE=ext3 ;; - *) EXT=".ko"; USE_QUOTA=yes; FSTYPE=ldiskfs ;; -esac + [ "$TESTSUITELOG" ] && rm -f $TESTSUITELOG || true + +} load_module() { + EXT=".ko" module=$1 shift BASE=`basename $module $EXT` @@ -84,7 +107,12 @@ load_module() { insmod ${LUSTRE}/${module}${EXT} $@ else # must be testing a "make install" or "rpm" installation - modprobe $BASE $@ + # note failed to load ptlrpc_gss is considered not fatal + if [ "$BASE" == "ptlrpc_gss" ]; then + modprobe $BASE $@ 2>/dev/null || echo "gss/krb5 is not supported" + else + modprobe $BASE $@ + fi fi } @@ -101,20 +129,34 @@ load_modules() { echo Loading modules from $LUSTRE load_module ../lnet/libcfs/libcfs + [ -z "$LNETOPTS" ] && \ + LNETOPTS=$(awk '/^options lnet/ { print $0}' /etc/modprobe.conf | sed 's/^options lnet //g') + echo "lnet options: '$LNETOPTS'" # note that insmod will ignore anything in modprobe.conf load_module ../lnet/lnet/lnet $LNETOPTS LNETLND=${LNETLND:-"socklnd/ksocklnd"} load_module ../lnet/klnds/$LNETLND + [ "$FSTYPE" = "ldiskfs" ] && load_module ../ldiskfs/ldiskfs/ldiskfs load_module lvfs/lvfs load_module obdclass/obdclass + load_module lvfs/fsfilt_$FSTYPE load_module ptlrpc/ptlrpc - [ "$USE_QUOTA" = "yes" ] && load_module quota/lquota + load_module ptlrpc/gss/ptlrpc_gss + # Now, some modules depend on lquota without USE_QUOTA check, + # will fix later. Disable check "$USE_QUOTA" = "yes" temporary. + #[ "$USE_QUOTA" = "yes" ] && load_module quota/lquota + load_module quota/lquota + load_module fid/fid + load_module fld/fld + load_module lmv/lmv load_module mdc/mdc load_module osc/osc load_module lov/lov load_module mds/mds - [ "$FSTYPE" = "ldiskfs" ] && load_module ldiskfs/ldiskfs - load_module lvfs/fsfilt_$FSTYPE + load_module mdd/mdd + load_module mdt/mdt + load_module cmm/cmm + load_module osd/osd load_module ost/ost load_module obdfilter/obdfilter load_module llite/lustre @@ -138,7 +180,7 @@ wait_for_lnet() { MODULES=$($LCTL modules | awk '{ print $2 }') while [ -n "$MODULES" ]; do sleep 5 - $RMMOD $MODULES >/dev/null 2>&1 || true + $RMMOD $MODULES > /dev/null 2>&1 || true MODULES=$($LCTL modules | awk '{ print $2 }') if [ -z "$MODULES" ]; then return 0 @@ -157,10 +199,10 @@ wait_for_lnet() { unload_modules() { lsmod | grep lnet > /dev/null && $LCTL dl && $LCTL dk $TMP/debug local MODULES=$($LCTL modules | awk '{ print $2 }') - $RMMOD $MODULES >/dev/null 2>&1 || true + $RMMOD $MODULES > /dev/null 2>&1 || true # do it again, in case we tried to unload ksocklnd too early MODULES=$($LCTL modules | awk '{ print $2 }') - [ -n "$MODULES" ] && $RMMOD $MODULES >/dev/null || true + [ -n "$MODULES" ] && $RMMOD $MODULES > /dev/null 2>&1 || true MODULES=$($LCTL modules | awk '{ print $2 }') if [ -n "$MODULES" ]; then echo "Modules still loaded: " @@ -171,7 +213,7 @@ unload_modules() { lsmod return 2 else - echo "Lustre stopped, but LNET is still loaded" + echo "Lustre stopped but LNET is still loaded, waiting..." wait_for_lnet || return 3 fi fi @@ -184,13 +226,110 @@ unload_modules() { echo "$LEAK_PORTALS" 1>&2 mv $TMP/debug $TMP/debug-leak.`date +%s` || true echo "Memory leaks detected" - [ "$LEAK_LUSTRE" -a $(echo $LEAK_LUSTRE | awk 'leaked=$8 {print leaked % 56}') == 0 ] && echo "ignoring known bug 10818" && return 0 return 254 fi echo "modules unloaded." return 0 } +check_gss_daemon_facet() { + facet=$1 + dname=$2 + + num=`do_facet $facet ps -o cmd -C $dname | grep $dname | wc -l` + if [ $num -ne 1 ]; then + echo "$num instance of $dname on $facet" + return 1 + fi + return 0 +} + +send_sigint() { + local facet=$1 + shift + do_facet $facet "killall -2 $@ 2>/dev/null || true" +} + +start_gss_daemons() { + # starting on MDT + for num in `seq $MDSCOUNT`; do + do_facet mds$num "$LSVCGSSD -v" + if $GSS_PIPEFS; then + do_facet mds$num "$LGSSD -v" + fi + done + # starting on OSTs + for num in `seq $OSTCOUNT`; do + do_facet ost$num "$LSVCGSSD -v" + done + # starting on client + # FIXME: is "client" the right facet name? + if $GSS_PIPEFS; then + do_facet client "$LGSSD -v" + fi + + # wait daemons entering "stable" status + sleep 5 + + # + # check daemons are running + # + for num in `seq $MDSCOUNT`; do + check_gss_daemon_facet mds$num lsvcgssd + if $GSS_PIPEFS; then + check_gss_daemon_facet mds$num lgssd + fi + done + for num in `seq $OSTCOUNT`; do + check_gss_daemon_facet ost$num lsvcgssd + done + if $GSS_PIPEFS; then + check_gss_daemon_facet client lgssd + fi +} + +stop_gss_daemons() { + for num in `seq $MDSCOUNT`; do + send_sigint mds$num lsvcgssd lgssd + done + for num in `seq $OSTCOUNT`; do + send_sigint ost$num lsvcgssd + done + send_sigint client lgssd +} + +init_krb5_env() { + if [ ! -z $SEC ]; then + MDS_MOUNT_OPTS=$MDS_MOUNT_OPTS,sec=$SEC + OST_MOUNT_OPTS=$OST_MOUNT_OPTS,sec=$SEC + fi + + if $GSS; then + start_gss_daemons + fi +} + +cleanup_krb5_env() { + if $GSS; then + stop_gss_daemons + # maybe cleanup credential cache? + fi +} + +mdsdevlabel() { + local num=$1 + local device=`mdsdevname $num` + local label=`do_facet mds$num "e2label ${device}" | grep -v "CMD: "` + echo -n $label +} + +ostdevlabel() { + local num=$1 + local device=`ostdevname $num` + local label=`do_facet ost$num "e2label ${device}" | grep -v "CMD: "` + echo -n $label +} + # Facet functions # start facet device options start() { @@ -207,7 +346,7 @@ start() { echo Start of ${device} on ${facet} failed ${RC} else do_facet ${facet} sync - label=`do_facet ${facet} "e2label ${device}" | grep -v "CMD: "` + label=$(do_facet ${facet} "e2label ${device}") [ -z "$label" ] && echo no label for ${device} && exit 1 eval export ${facet}_svc=${label} eval export ${facet}_dev=${device} @@ -224,7 +363,7 @@ stop() { HOST=`facet_active_host $facet` [ -z $HOST ] && echo stop: no host for $facet && return 0 - running=`do_facet ${facet} "grep -c ${MOUNT%/*}/${facet}' ' /proc/mounts" | grep -v "CMD: "` + running=$(do_facet ${facet} "grep -c ${MOUNT%/*}/${facet}' ' /proc/mounts") || true if [ ${running} -ne 0 ]; then echo "Stopping ${MOUNT%/*}/${facet} (opts:$@)" do_facet ${facet} umount -d $@ ${MOUNT%/*}/${facet} @@ -236,16 +375,16 @@ stop() { local INTERVAL=1 # conf-sanity 31 takes a long time cleanup while [ $WAIT -lt 300 ]; do - running=$(do_facet ${facet} "[ -e $LPROC ] && grep ST' ' $LPROC/devices" | grep -v "CMD: ") || true - if [ -z "${running}" ]; then - return 0 - fi - echo "waited $WAIT for${running}" - if [ $INTERVAL -lt 64 ]; then - INTERVAL=$((INTERVAL + INTERVAL)) - fi - sleep $INTERVAL - WAIT=$((WAIT + INTERVAL)) + running=$(do_facet ${facet} "[ -e $LPROC ] && grep ST' ' $LPROC/devices") || true + if [ -z "${running}" ]; then + return 0 + fi + echo "waited $WAIT for${running}" + if [ $INTERVAL -lt 64 ]; then + INTERVAL=$((INTERVAL + INTERVAL)) + fi + sleep $INTERVAL + WAIT=$((WAIT + INTERVAL)) done echo "service didn't stop after $WAIT seconds. Still running:" echo ${running} @@ -279,7 +418,7 @@ zconf_umount() { client=$1 mnt=$2 [ "$3" ] && force=-f - local running=`do_node $client "grep -c $mnt' ' /proc/mounts" | grep -v "CMD: "` + local running=$(do_node $client "grep -c $mnt' ' /proc/mounts") || true if [ $running -ne 0 ]; then echo "Stopping client $mnt (opts:$force)" do_node $client umount $force $mnt @@ -348,6 +487,24 @@ wait_for() { wait_for_host $HOST } +wait_mds_recovery_done () { + local timeout=`do_facet mds cat /proc/sys/lustre/timeout` +#define OBD_RECOVERY_TIMEOUT (obd_timeout * 5 / 2) +# as we are in process of changing obd_timeout in different ways +# let's set MAX longer than that + MAX=$(( timeout * 4 )) + WAIT=0 + while [ $WAIT -lt $MAX ]; do + STATUS=`do_facet mds grep status /proc/fs/lustre/mdt/*-MDT*/recovery_status` + echo $STATUS | grep COMPLETE && return 0 + sleep 5 + WAIT=$((WAIT + 5)) + echo "Waiting $(($MAX - $WAIT)) secs for MDS recovery done" + done + echo "MDS recovery not done in $MAX sec" + return 1 +} + client_df() { # not every config has many clients if [ ! -z "$CLIENTS" ]; then @@ -380,7 +537,7 @@ facet_failover() { wait_for $facet local dev=${facet}_dev local opt=${facet}_opt - start $facet ${!dev} ${!opt} + start $facet ${!dev} ${!opt} || error "Restart of $facet failed" } obd_name() { @@ -410,12 +567,12 @@ replay_barrier_nodf() { } mds_evict_client() { - UUID=`cat /proc/fs/lustre/mdc/${mds_svc}-mdc-*/uuid` - do_facet mds "echo $UUID > /proc/fs/lustre/mds/${mds_svc}/evict_client" + UUID=`cat /proc/fs/lustre/mdc/${mds1_svc}-mdc-*/uuid` + do_facet mds1 "echo $UUID > /proc/fs/lustre/mdt/${mds1_svc}/evict_client" } ost_evict_client() { - UUID=`cat /proc/fs/lustre/osc/${ost1_svc}-osc-*/uuid` + UUID=`grep ${ost1_svc}-osc- $LPROC/devices | egrep -v 'MDT' | awk '{print $5}'` do_facet ost1 "echo $UUID > /proc/fs/lustre/obdfilter/${ost1_svc}/evict_client" } @@ -450,11 +607,27 @@ h2gm () { fi } -h2tcp() { +h2name_or_ip() { if [ "$1" = "client" -o "$1" = "'*'" ]; then echo \'*\'; else - echo $1"@tcp" + echo $1"@$2" fi } + +h2ptl() { + if [ "$1" = "client" -o "$1" = "'*'" ]; then echo \'*\'; else + ID=`xtprocadmin -n $1 2>/dev/null | egrep -v 'NID' | awk '{print $1}'` + if [ -z "$ID" ]; then + echo "Could not get a ptl id for $1..." + exit 1 + fi + echo $ID"@ptl" + fi +} +declare -fx h2ptl + +h2tcp() { + h2name_or_ip "$1" "tcp" +} declare -fx h2tcp h2elan() { @@ -470,13 +643,15 @@ h2elan() { declare -fx h2elan h2openib() { - if [ "$1" = "client" -o "$1" = "'*'" ]; then echo \'*\'; else - ID=`echo $1 | sed 's/[^0-9]*//g'` - echo $ID"@openib" - fi + h2name_or_ip "$1" "openib" } declare -fx h2openib +h2o2ib() { + h2name_or_ip "$1" "o2ib" +} +declare -fx h2o2ib + facet_host() { local facet=$1 varname=${facet}_HOST @@ -538,10 +713,11 @@ do_node() { myPDSH="no_dsh" fi if $VERBOSE; then - echo "CMD: $HOST $@" + echo "CMD: $HOST $@" >&2 $myPDSH $HOST $LCTL mark "$@" > /dev/null 2>&1 || : fi - $myPDSH $HOST "(PATH=\$PATH:$RLUSTRE/utils:$RLUSTRE/tests:/sbin:/usr/sbin; cd $RPWD; sh -c \"$@\")" + $myPDSH $HOST "(PATH=\$PATH:$RLUSTRE/utils:$RLUSTRE/tests:/sbin:/usr/sbin; cd $RPWD; sh -c \"$@\")" | sed "s/^${HOST}: //" + return ${PIPESTATUS[0]} } do_facet() { @@ -569,21 +745,31 @@ ostdevname() { echo -n $DEVPTR } +mdsdevname() { + num=$1 + DEVNAME=MDSDEV$num + #if $MDSDEVn isn't defined, default is $MDSDEVBASE + num + eval DEVPTR=${!DEVNAME:=${MDSDEVBASE}${num}} + echo -n $DEVPTR +} + ######## ## MountConf setup stopall() { # make sure we are using the primary server, so test-framework will # be able to clean up properly. - activemds=`facet_active mds` - if [ $activemds != "mds" ]; then - fail mds + activemds=`facet_active mds1` + if [ $activemds != "mds1" ]; then + fail mds1 fi # assume client mount is local grep " $MOUNT " /proc/mounts && zconf_umount `hostname` $MOUNT $* grep " $MOUNT2 " /proc/mounts && zconf_umount `hostname` $MOUNT2 $* - stop mds -f + for num in `seq $MDSCOUNT`; do + stop mds$num -f + done for num in `seq $OSTCOUNT`; do stop ost$num -f done @@ -593,20 +779,34 @@ stopall() { cleanupall() { stopall $* unload_modules + cleanup_krb5_env +} + +mdsmkfsopts() +{ + local nr=$1 + test $nr = 1 && echo -n $MDS_MKFS_OPTS || echo -n $MDSn_MKFS_OPTS } formatall() { + [ "$FSTYPE" ] && FSTYPE_OPT="--backfstype $FSTYPE" + stopall # We need ldiskfs here, may as well load them all load_modules - echo Formatting mds, osts - if $VERBOSE; then - add mds $MDS_MKFS_OPTS --reformat $MDSDEV || exit 10 - else - add mds $MDS_MKFS_OPTS --reformat $MDSDEV > /dev/null || exit 10 - fi + [ "$CLIENTONLY" ] && return + echo "Formatting mdts, osts" + for num in `seq $MDSCOUNT`; do + echo "Format mds$num: $(mdsdevname $num)" + if $VERBOSE; then + add mds$num `mdsmkfsopts $num` $FSTYPE_OPT --reformat `mdsdevname $num` || exit 9 + else + add mds$num `mdsmkfsopts $num` $FSTYPE_OPT --reformat `mdsdevname $num` > /dev/null || exit 9 + fi + done for num in `seq $OSTCOUNT`; do + echo "Format ost$num: $(ostdevname $num)" if $VERBOSE; then add ost$num $OST_MKFS_OPTS --reformat `ostdevname $num` || exit 10 else @@ -621,17 +821,66 @@ mount_client() { setupall() { load_modules - echo Setup mdt, osts - start mds $MDSDEV $MDS_MOUNT_OPTS - for num in `seq $OSTCOUNT`; do - DEVNAME=`ostdevname $num` - start ost$num $DEVNAME $OST_MOUNT_OPTS - done + init_krb5_env + if [ -z "$CLIENTONLY" ]; then + echo "Setup mdts, osts" + for num in `seq $MDSCOUNT`; do + DEVNAME=$(mdsdevname $num) + echo $REFORMAT | grep -q "reformat" \ + || do_facet mds$num "$TUNEFS --writeconf $DEVNAME" + start mds$num $DEVNAME $MDS_MOUNT_OPTS + done + for num in `seq $OSTCOUNT`; do + DEVNAME=$(ostdevname $num) + start ost$num $DEVNAME $OST_MOUNT_OPTS + done + fi [ "$DAEMONFILE" ] && $LCTL debug_daemon start $DAEMONFILE $DAEMONSIZE mount_client $MOUNT + if [ "$MOUNT_2" ]; then + mount_client $MOUNT2 + fi sleep 5 } +mounted_lustre_filesystems() { + awk '($3 ~ "lustre" && $1 ~ ":") { print $2 }' /proc/mounts +} + +check_and_setup_lustre() { + MOUNTED="`mounted_lustre_filesystems`" + if [ -z "$MOUNTED" ]; then + [ "$REFORMAT" ] && formatall + setupall + MOUNTED="`mounted_lustre_filesystems`" + [ -z "$MOUNTED" ] && error "NAME=$NAME not mounted" + export I_MOUNTED=yes + fi + if [ "$ONLY" == "setup" ]; then + exit 0 + fi +} + +cleanup_and_setup_lustre() { + if [ "$ONLY" == "cleanup" -o "`mount | grep $MOUNT`" ]; then + sysctl -w lnet.debug=0 || true + cleanupall + if [ "$ONLY" == "cleanup" ]; then + exit 0 + fi + fi + check_and_setup_lustre +} + +check_and_cleanup_lustre() { + if [ "`mount | grep $MOUNT`" ]; then + rm -rf $DIR/[Rdfs][1-9]* + fi + if [ "$I_MOUNTED" = "yes" ]; then + cleanupall -f || error "cleanup failed" + fi + unset I_MOUNTED +} ####### # General functions @@ -772,13 +1021,38 @@ pgcache_empty() { return 0 } +debugsave() { + DEBUGSAVE="$(sysctl -n lnet.debug)" +} + +debugrestore() { + [ -n "$DEBUGSAVE" ] && sysctl -w lnet.debug="${DEBUGSAVE}" + DEBUGSAVE="" +} + +FAIL_ON_ERROR=true ################################## # Test interface error() { + local ERRLOG sysctl -w lustre.fail_loc=0 2> /dev/null || true - echo "${TESTSUITE}: **** FAIL:" $@ - log "FAIL: $TESTNAME $@" - exit 1 + log "${TESTSUITE} ${TESTNAME}: **** FAIL:" $@ + ERRLOG=$TMP/lustre_${TESTSUITE}_${TESTNAME}.$(date +%s) + echo "Dumping lctl log to $ERRLOG" + # We need to dump the logs on all nodes + $LCTL dk $ERRLOG + [ ! "$mds_HOST" = "$(hostname)" ] && do_node $mds_HOST $LCTL dk $ERRLOG + [ ! "$ost_HOST" = "$(hostname)" -a ! "$ost_HOST" = "$mds_HOST" ] && do_node $ost_HOST $LCTL dk $ERRLOG + debugrestore + [ "$TESTSUITELOG" ] && echo "$0: FAIL: $TESTNAME $@" >> $TESTSUITELOG + if $FAIL_ON_ERROR; then + exit 1 + fi +} + +skip () { + log " SKIP: ${TESTSUITE} ${TESTNAME} $@" + [ "$TESTSUITELOG" ] && echo "${TESTSUITE}: SKIP: $TESTNAME $@" >> $TESTSUITELOG } build_test_filter() { @@ -791,6 +1065,9 @@ build_test_filter() { for E in $EXCEPT $ALWAYS_EXCEPT; do eval EXCEPT_${E}=true done + for G in $GRANT_CHECK_LIST; do + eval GCHECK_ONLY_${G}=true + done } _basetest() { @@ -819,12 +1096,12 @@ run_test() { fi testname=EXCEPT_$1 if [ ${!testname}x != x ]; then - log "skipping excluded test $1" + TESTNAME=test_$1 skip "skipping excluded test $1" return 0 fi testname=EXCEPT_$base if [ ${!testname}x != x ]; then - log "skipping excluded test $1 (base $base)" + TESTNAME=test_$1 skip "skipping excluded test $1 (base $base)" return 0 fi run_one $1 "$2" @@ -847,6 +1124,14 @@ log() { $LCTL mark "$*" 2> /dev/null || true } +trace() { + log "STARTING: $*" + strace -o $TMP/$1.strace -ttt $* + RC=$? + log "FINISHED: $*: rc $RC" + return 1 +} + pass() { echo PASS $@ } @@ -861,10 +1146,7 @@ run_one() { testnum=$1 message=$2 tfile=f${testnum} - tdir=d${base} - - # Pretty tests run faster. - equals_msg $testnum: $message + export tdir=d${base} BEFORE=`date +%s` log "== test $testnum: $message ============ `date +%H:%M:%S` ($BEFORE)" @@ -872,13 +1154,93 @@ run_one() { export TESTNAME=test_$testnum test_${testnum} || error "test_$testnum failed with $?" #check_mds + check_grant ${testnum} || error "check_grant $testnum failed with $?" [ -f $CATASTROPHE ] && [ `cat $CATASTROPHE` -ne 0 ] && \ error "LBUG/LASSERT detected" pass "($((`date +%s` - $BEFORE))s)" unset TESTNAME + unset tdir + cd $SAVE_PWD + $CLEANUP } canonical_path() { (cd `dirname $1`; echo $PWD/`basename $1`) } +sync_clients() { + [ -d $DIR1 ] && cd $DIR1 && sync; sleep 1; sync + [ -d $DIR2 ] && cd $DIR2 && sync; sleep 1; sync + cd $SAVE_PWD +} + +check_grant() { + export base=`basetest $1` + [ "$CHECK_GRANT" == "no" ] && return 0 + + testname=GCHECK_ONLY_${base} + [ ${!testname}x == x ] && return 0 + + echo -n "checking grant......" + cd $SAVE_PWD + # write some data to sync client lost_grant + rm -f $DIR1/${tfile}_check_grant_* 2>&1 + for i in `seq $OSTCOUNT`; do + $LFS setstripe $DIR1/${tfile}_check_grant_$i 0 $(($i -1)) 1 + dd if=/dev/zero of=$DIR1/${tfile}_check_grant_$i bs=4k \ + count=1 > /dev/null 2>&1 + done + # sync all the data and make sure no pending data on server + sync_clients + + #get client grant and server grant + client_grant=0 + for d in ${LPROC}/osc/*/cur_grant_bytes; do + client_grant=$((client_grant + `cat $d`)) + done + server_grant=0 + for d in ${LPROC}/obdfilter/*/tot_granted; do + server_grant=$((server_grant + `cat $d`)) + done + + # cleanup the check_grant file + for i in `seq $OSTCOUNT`; do + rm $DIR1/${tfile}_check_grant_$i + done + + #check whether client grant == server grant + if [ $client_grant != $server_grant ]; then + echo "failed: client:${client_grant} server: ${server_grant}" + return 1 + else + echo "pass" + fi +} + +######################## +# helper functions + +osc_to_ost() +{ + osc=$1 + ost=`echo $1 | awk -F_ '{print $3}'` + if [ -z $ost ]; then + ost=`echo $1 | sed 's/-osc.*//'` + fi + echo $ost +} + +remote_mds () +{ + [ ! -e /proc/fs/lustre/mdt/*MDT* ] +} + +remote_ost () +{ + [ $(grep -c obdfilter $LPROC/devices) -eq 0 ] +} + +is_patchless () +{ + grep -q patchless $LPROC/version +}