Whamcloud - gitweb
Branch:b1_6
authorwangdi <wangdi>
Mon, 27 Aug 2007 03:14:45 +0000 (03:14 +0000)
committerwangdi <wangdi>
Mon, 27 Aug 2007 03:14:45 +0000 (03:14 +0000)
cleanup sanity test scripts and move some common funcs to test-frame.sh.
b=12599
i=adilger
i=brian
i=mjmac

lustre/ChangeLog
lustre/tests/recovery-small.sh
lustre/tests/replay-dual.sh
lustre/tests/replay-ost-single.sh
lustre/tests/replay-single.sh
lustre/tests/sanity.sh
lustre/tests/sanityN.sh
lustre/tests/test-framework.sh

index 8e3a273..79a87db 100644 (file)
@@ -86,6 +86,12 @@ Description: Add EXPORT_SYMBOL check for node_to_cpumask symbol.
 Details    : This allows the patchless client to be loaded on architectures
              without this export.
 
+Severity   : normal
+Bugzilla   : 12599
+Description: move common funcs to test-frame.sh
+Details    : cleanup those test-scripts and move some common funcs to
+             test-frame.sh 
+
 --------------------------------------------------------------------------------
 
 2007-08-27         Cluster File Systems, Inc. <info@clusterfs.com>
index 7470691..8adc969 100755 (executable)
@@ -15,32 +15,10 @@ build_test_filter
 
 # Allow us to override the setup if we already have a mounted system by
 # setting SETUP=" " and CLEANUP=" "
-SETUP=${SETUP:-"setup"}
-CLEANUP=${CLEANUP:-"cleanup"}
+SETUP=${SETUP:-""}
+CLEANUP=${CLEANUP:-""}
 
-setup() {
-    [ "$REFORMAT" ] && formatall
-    setupall
-}
-
-cleanup() {
-       cleanupall || { echo "FAILed to clean up"; exit 20; }
-}
-
-if [ ! -z "$EVAL" ]; then
-    eval "$EVAL"
-    exit $?
-fi
-
-if [ "$ONLY" == "cleanup" ]; then
-    sysctl -w lnet.debug=0 || true
-    cleanup
-    exit
-fi
-
-$SETUP
-
-[ "$ONLY" == "setup" ] && exit
+cleanup_and_setup_lustre
 
 test_1() {
     drop_request "mcreate $MOUNT/1"  || return 1
@@ -899,5 +877,5 @@ test_59() { # bug 10589
 }
 run_test 59 "Read cancel race on client eviction"
 
-$CLEANUP
+check_and_cleanup_lustre
 echo "$0: completed"
index 0a8a773..bbd2687 100755 (executable)
@@ -5,37 +5,23 @@ set -e
 # bug number:  10124 
 ALWAYS_EXCEPT="15c   $REPLAY_DUAL_EXCEPT"
 
+SAVE_PWD=$PWD
 PTLDEBUG=${PTLDEBUG:--1}
 LUSTRE=${LUSTRE:-`dirname $0`/..}
+SETUP=${SETUP:-""}
+CLEANUP=${CLEANUP:-""}
+MOUNT_2=${MOUNT_2:-"yes"}
 . $LUSTRE/tests/test-framework.sh
 
 init_test_env $@
 
 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
 
-SETUP=${SETUP:-"setup"}
-CLEANUP=${CLEANUP:-"cleanup"}
 
 build_test_filter
 
-cleanup() {
-    stopall
-}
-
-if [ "$ONLY" == "cleanup" ]; then
-    sysctl -w lnet.debug=0
-    cleanup
-    exit
-fi
-
-setup() {
-    cleanup
-    [ "$REFORMAT" ] && formatall
-    setupall
-    mount_client $MOUNT2
-}
+cleanup_and_setup_lustre
 
-$SETUP
 [ "$DAEMONFILE" ] && $LCTL debug_daemon start $DAEMONFILE $DAEMONSIZE
 
 test_1() {
@@ -448,10 +434,9 @@ test_19() { # Bug 10991 - resend of open request does not fail assertion.
 }
 run_test 19 "resend of open request"
 
-if [ "$ONLY" != "setup" ]; then
-   equals_msg test complete, cleaning up
-   SLEEP=$((`date +%s` - $NOW))
-   [ $SLEEP -lt $TIMEOUT ] && sleep $SLEEP
-   $CLEANUP
-fi
+equals_msg test complete, cleaning up
+SLEEP=$((`date +%s` - $NOW))
+[ $SLEEP -lt $TIMEOUT ] && sleep $SLEEP
+check_and_cleanup_lustre
+
 echo "$0: completed"
index e81bd12..68cf1fe 100755 (executable)
@@ -67,7 +67,6 @@ setup() {
 mkdir -p $DIR
 
 $SETUP
-
 test_0b() {
     fail ost1
     cp /etc/profile  $DIR/$tfile
index a20b50a..c8b02e0 100755 (executable)
@@ -6,52 +6,23 @@ set -e
 #
 # This test needs to be run on the client
 #
-
+SAVE_PWD=$PWD
 LUSTRE=${LUSTRE:-`dirname $0`/..}
+SETUP=${SETUP:-}
+CLEANUP=${CLEANUP:-}
 . $LUSTRE/tests/test-framework.sh
 init_test_env $@
 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
-
+CHECK_GRANT=${CHECK_GRANT:-"yes"}
+GRANT_CHECK_LIST=${GRANT_CHECK_LIST:-""}
 
 # Skip these tests
 # bug number: 
 ALWAYS_EXCEPT="$REPLAY_SINGLE_EXCEPT"
 
-gen_config() {
-    rm -f $XMLCONFIG
-    add_mds mds --dev $MDSDEV --size $MDSSIZE
-    if [ ! -z "$mdsfailover_HOST" ]; then
-        add_mdsfailover mds --dev $MDSDEV --size $MDSSIZE
-    fi
-    
-    add_lov lov1 mds --stripe_sz $STRIPE_BYTES \
-       --stripe_cnt $STRIPES_PER_OBJ --stripe_pattern 0
-    add_ost ost --lov lov1 --dev `ostdevname 1` --size $OSTSIZE
-    add_ost ost2 --lov lov1 --dev `ostdevname 2` --size $OSTSIZE
-    add_client client mds --lov lov1 --path $MOUNT
-}
-
 build_test_filter
 
-SETUP=${SETUP:-"setup"}
-CLEANUP=${CLEANUP:-"cleanupall"}
-
-if [ "$ONLY" == "cleanup" ]; then
-    sysctl -w lnet.debug=0 || true
-    $CLEANUP
-    exit 0
-fi
-
-setup() {
-    [ "$REFORMAT" ] && formatall
-    setupall
-}
-
-$SETUP
-
-if [ "$ONLY" == "setup" ]; then
-    exit 0
-fi
+cleanup_and_setup_lustre
 
 mkdir -p $DIR
 
@@ -1199,4 +1170,5 @@ test_61c() {
 run_test 61c "test race mds llog sync vs llog cleanup"
 
 equals_msg `basename $0`: test complete, cleaning up
-$CLEANUP
+
+check_and_cleanup_lustre
index e6173cc..7d4f2ff 100644 (file)
@@ -55,6 +55,8 @@ MEMHOG=${MEMHOG:-memhog}
 DIRECTIO=${DIRECTIO:-directio}
 ACCEPTOR_PORT=${ACCEPTOR_PORT:-988}
 UMOUNT=${UMOUNT:-"umount -d"}
+CHECK_GRANT=${CHECK_GRANT:-"yes"}
+GRANT_CHECK_LIST=${GRANT_CHECK_LIST:-""}
 
 if [ $UID -ne 0 ]; then
        echo "Warning: running as non-root uid $UID"
@@ -77,6 +79,9 @@ export NAME=${NAME:-local}
 
 SAVE_PWD=$PWD
 
+CLEANUP=${CLEANUP:-:}
+SETUP=${SETUP:-:}
+TRACE=${TRACE:-""}
 LUSTRE=${LUSTRE:-`dirname $0`/..}
 . $LUSTRE/tests/test-framework.sh
 init_test_env $@
@@ -86,29 +91,12 @@ cleanup() {
        echo -n "cln.."
        cleanupall ${FORCE} $* || { echo "FAILed to clean up"; exit 20; }
 }
-CLEANUP=${CLEANUP:-:}
-
 setup() {
        echo -n "mnt.."
         load_modules
        setupall || exit 10
        echo "done"
 }
-SETUP=${SETUP:-:}
-
-log() {
-       echo "$*"
-       $LCTL mark "$*" 2> /dev/null || true
-}
-
-trace() {
-       log "STARTING: $*"
-       strace -o $TMP/$1.strace -ttt $*
-       RC=$?
-       log "FINISHED: $*: rc $RC"
-       return 1
-}
-TRACE=${TRACE:-""}
 
 check_kernel_version() {
        VERSION_FILE=$LPROC/version
@@ -121,117 +109,14 @@ check_kernel_version() {
        return 1
 }
 
-_basetest() {
-    echo $*
-}
-
-basetest() {
-    IFS=abcdefghijklmnopqrstuvwxyz _basetest $1
-}
-
-run_one() {
-       if ! grep -q $DIR /proc/mounts; then
-               $SETUP
-       fi
-       testnum=$1
-       message=$2
-       BEFORE=`date +%s`
-       log "== test $testnum: $message= `date +%H:%M:%S` ($BEFORE)"
-       export TESTNAME=test_$testnum
-       export tfile=f${testnum}
-       export tdir=d${base}
-       test_${testnum} || error "exit with rc=$?"
-       unset TESTNAME
-       pass "($((`date +%s` - $BEFORE))s)"
-       cd $SAVE_PWD
-       $CLEANUP
-}
-
-build_test_filter() {
-       [ "$ALWAYS_EXCEPT$EXCEPT$SANITY_EXCEPT" ] && \
-           echo "Skipping tests: `echo $ALWAYS_EXCEPT $EXCEPT $SANITY_EXCEPT`"
-
-        for O in $ONLY; do
-            eval ONLY_${O}=true
-        done
-        for E in $EXCEPT $ALWAYS_EXCEPT $SANITY_EXCEPT; do
-            eval EXCEPT_${E}=true
-        done
-}
-
-_basetest() {
-       echo $*
-}
-
-basetest() {
-       IFS=abcdefghijklmnopqrstuvwxyz _basetest $1
-}
-
-run_test() {
-         export base=`basetest $1`
-         if [ "$ONLY" ]; then
-                 testname=ONLY_$1
-                 if [ ${!testname}x != x ]; then
-                       run_one $1 "$2"
-                       return $?
-                 fi
-                 testname=ONLY_$base
-                 if [ ${!testname}x != x ]; then
-                         run_one $1 "$2"
-                         return $?
-                 fi
-                 echo -n "."
-                 return 0
-       fi
-        testname=EXCEPT_$1
-        if [ ${!testname}x != x ]; then
-                 TESTNAME=test_$1 skip "skipping excluded test $1"
-                 return 0
-        fi
-        testname=EXCEPT_$base
-        if [ ${!testname}x != x ]; then
-                 TESTNAME=test_$1 skip "skipping excluded test $1 (base $base)"
-                 return 0
-        fi
-        run_one $1 "$2"
-       return $?
-}
+if [ "$ONLY" == "cleanup" ]; then
+       sh llmountcleanup.sh
+       exit 0
+fi
 
 [ "$SANITYLOG" ] && rm -f $SANITYLOG || true
 
-error() { 
-       sysctl -w lustre.fail_loc=0
-       log "$0: FAIL: $TESTNAME $@"
-       $LCTL dk $TMP/lustre-log-$TESTNAME.log
-       if [ "$SANITYLOG" ]; then
-               echo "$0: FAIL: $TESTNAME $@" >> $SANITYLOG
-       else
-               exit 1
-       fi
-}
-
-pass() { 
-       echo PASS $@
-}
-
-skip () {
-       log "$0: SKIP: $TESTNAME $@"
-       [ "$SANITYLOG" ] && echo "$0: SKIP: $TESTNAME $@" >> $SANITYLOG
-
-}
-
-mounted_lustre_filesystems() {
-       awk '($3 ~ "lustre" && $1 ~ ":") { print $2 }' /proc/mounts
-}
-
-MOUNTED="`mounted_lustre_filesystems`"
-if [ -z "$MOUNTED" ]; then
-        formatall
-       setupall
-       MOUNTED="`mounted_lustre_filesystems`"
-       [ -z "$MOUNTED" ] && error "NAME=$NAME not mounted"
-       I_MOUNTED=yes
-fi
+check_and_setup_lustre
 
 DIR=${DIR:-$MOUNT}
 [ -z "`echo $DIR | grep $MOUNT`" ] && echo "$DIR not in $MOUNT" && exit 99
@@ -4132,16 +4017,11 @@ TMP=$OLDTMP
 HOME=$OLDHOME
 
 log "cleanup: ======================================================"
-if [ "`mount | grep $MOUNT`" ]; then
-       rm -rf $DIR/[Rdfs][1-9]*
-fi
-if [ "$I_MOUNTED" = "yes" ]; then
-       cleanupall -f || error "cleanup failed"
-else
+check_and_cleanup_lustre
+if [ "$I_MOUNTED" != "yes" ]; then
        sysctl -w lnet.debug="$OLDDEBUG" 2> /dev/null || true
 fi
 
-
 echo '=========================== finished ==============================='
 [ -f "$SANITYLOG" ] && cat $SANITYLOG && grep -q FAIL $SANITYLOG && exit 1 || true
 echo "$0: completed"
index ad6f172..0f5209e 100644 (file)
@@ -4,7 +4,7 @@ set -e
 
 ONLY=${ONLY:-"$*"}
 # bug number for skipped test:  3192
-ALWAYS_EXCEPT=${ALWAYS_EXCEPT:-"14b"}
+-ALWAYS_EXCEPT=${ALWAYS_EXCEPT:-"14b"}
 # UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT!
 
 [ "$SLOW" = "no" ] && EXCEPT="$EXCEPT 16"
@@ -20,16 +20,15 @@ CHECKSTAT=${CHECKSTAT:-"checkstat -v"}
 CREATETEST=${CREATETEST:-createtest}
 GETSTRIPE=${GETSTRIPE:-lfs getstripe}
 SETSTRIPE=${SETSTRIPE:-lstripe}
-LFS=${LFS:-lfs}
-LCTL=${LCTL:-lctl}
 MCREATE=${MCREATE:-mcreate}
 OPENFILE=${OPENFILE:-openfile}
 OPENUNLINK=${OPENUNLINK:-openunlink}
 TOEXCL=${TOEXCL:-toexcl}
 TRUNCATE=${TRUNCATE:-truncate}
 export TMP=${TMP:-/tmp}
-CHECK_GRANT=${CHECK_GRANT:-"no"}
-
+MOUNT_2=${MOUNT_2:-"yes"}
+CHECK_GRANT=${CHECK_GRANT:-"yes"}
+GRANT_CHECK_LIST=${GRANT_CHECK_LIST:-""}
 
 if [ $UID -ne 0 ]; then
        RUNAS_ID="$UID"
@@ -45,216 +44,18 @@ export NAME=${NAME:-local}
 
 LUSTRE=${LUSTRE:-`dirname $0`/..}
 . $LUSTRE/tests/test-framework.sh
-init_test_env $@
-. ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
-
-cleanup() {
-       echo -n "cln.."
-       grep " $MOUNT2 " /proc/mounts && zconf_umount `hostname` $MOUNT2 ${FORCE}
-       cleanupall ${FORCE} > /dev/null || { echo "FAILed to clean up"; exit 20; }
-}
 CLEANUP=${CLEANUP:-:}
-
-setup() {
-       echo -n "mnt.."
-       setupall || exit 10
-       echo "done"
-}
 SETUP=${SETUP:-:}
+init_test_env $@
+. ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
 
-log() {
-       echo "$*"
-       lctl mark "$*" 2> /dev/null || true
-}
-
-trace() {
-       log "STARTING: $*"
-       strace -o $TMP/$1.strace -ttt $*
-       RC=$?
-       log "FINISHED: $*: rc $RC"
-       return 1
-}
 TRACE=${TRACE:-""}
 
 LPROC=/proc/fs/lustre
 
-run_one() {
-       if ! grep -q $DIR /proc/mounts; then
-               $SETUP
-       fi
-       testnum=$1
-       message=$2
-       BEFORE=`date +%s`
-       log "== test $testnum: $message= `date +%H:%M:%S` ($BEFORE)"
-       export TESTNAME=test_$testnum
-       export tfile=f${testnum}
-       export tdir=d${base}
-       test_$1 || error "exit with rc=$?"
-       check_grant || error "check grant fail"
-       unset TESTNAME
-       pass "($((`date +%s` - $BEFORE))s)"
-       cd $SAVE_PWD
-       $CLEANUP
-}
-
-build_test_filter() {
-       [ "$ALWAYS_EXCEPT$EXCEPT$SANITYN_EXCEPT" ] && \
-           echo "Skipping tests: `echo $ALWAYS_EXCEPT $EXCEPT $SANITYN_EXCEPT`"
-
-        for O in $ONLY; do
-            eval ONLY_${O}=true
-        done
-        for E in $EXCEPT $ALWAYS_EXCEPT $SANITY_EXCEPT; do
-            eval EXCEPT_${E}=true
-        done
-}
-
-_basetest() {
-    echo $*
-}
-
-basetest() {
-    IFS=abcdefghijklmnopqrstuvwxyz _basetest $1
-}
-
-build_test_filter() {
-       [ "$ALWAYS_EXCEPT$EXCEPT$SANITYN_EXCEPT" ] && \
-           echo "Skipping tests: `echo $ALWAYS_EXCEPT $EXCEPT $SANITYN_EXCEPT`"
-
-        for O in $ONLY; do
-            eval ONLY_${O}=true
-        done
-        for E in $EXCEPT $ALWAYS_EXCEPT $SANITYN_EXCEPT; do
-            eval EXCEPT_${E}=true
-        done
-}
-
-_basetest() {
-    echo $*
-}
-
-basetest() {
-    IFS=abcdefghijklmnopqrstuvwxyz _basetest $1
-}
-
-sync_clients() {
-       cd $DIR1
-       sync; sleep 1; sync
-       cd $DIR2
-       sync; sleep 1; sync
-
-       cd $SAVE_PWD
-}
-
-check_grant() {
-       [ "$CHECK_GRANT" == "no" ] && return 0
-
-       echo -n "checking grant......"
-       cd $SAVE_PWD
-       # write some data to sync client lost_grant
-       rm -f $DIR1/${tfile}_check_grant_* 2>&1
-       for i in `seq $OSTCOUNT`; do
-               $LFS setstripe $DIR1/${tfile}_check_grant_$i 0 $(($i -1)) 1
-               dd if=/dev/zero of=$DIR1/${tfile}_check_grant_$i bs=4k \
-                                             count=1 > /dev/null 2>&1 
-       done
-       # sync all the data and make sure no pending data on server
-       sync_clients
-       
-       #get client grant and server grant 
-       client_grant=0
-               for d in /proc/fs/lustre/osc/*/cur_grant_bytes; do 
-               client_grant=$(($client_grant + `cat $d`))
-       done
-       server_grant=0
-       for d in /proc/fs/lustre/obdfilter/*/tot_granted; do
-               server_grant=$(($server_grant + `cat $d`))
-       done
-
-       # cleanup the check_grant file
-       for i in `seq $OSTCOUNT`; do
-               rm $DIR1/${tfile}_check_grant_$i
-       done
-
-       #check whether client grant == server grant 
-       if [ $client_grant != $server_grant ]; then
-               echo "failed: client:${client_grant} server: ${server_grant}"
-               return 1
-       else
-               echo "pass"
-       fi
-}
-
-run_test() {
-         export base=`basetest $1`
-         if [ "$ONLY" ]; then
-                 testname=ONLY_$1
-                 if [ ${!testname}x != x ]; then
-                       run_one $1 "$2"
-                       return $?
-                 fi
-                 testname=ONLY_$base
-                 if [ ${!testname}x != x ]; then
-                         run_one $1 "$2"
-                         return $?
-                 fi
-                 echo -n "."
-                 return 0
-       fi
-        testname=EXCEPT_$1
-        if [ ${!testname}x != x ]; then
-                 echo "skipping excluded test $1"
-                 return 0
-        fi
-        testname=EXCEPT_$base
-        if [ ${!testname}x != x ]; then
-                 echo "skipping excluded test $1 (base $base)"
-                 return 0
-        fi
-        run_one $1 "$2"
-       return $?
-}
-
 [ "$SANITYLOG" ] && rm -f $SANITYLOG || true
 
-error () {
-       sysctl -w lustre.fail_loc=0 2> /dev/null || true
-       log "$0: FAIL: $TESTNAME $@"
-       $LCTL dk $TMP/lustre-log-$TESTNAME.log
-       if [ "$SANITYLOG" ]; then
-               echo "$0: FAIL: $TESTNAME $@" >> $SANITYLOG
-       else
-               exit 1
-       fi
-}
-
-pass() {
-       echo PASS $@
-}
-
-mounted_lustre_filesystems() {
-       awk '($3 ~ "lustre" && $1 ~ ":") { print $2 }' /proc/mounts
-}
-MOUNTED="`mounted_lustre_filesystems`"
-if [ -z "$MOUNTED" ]; then
-    formatall
-    setupall
-    mount_client $MOUNT2
-    MOUNTED="`mounted_lustre_filesystems`"
-    [ -z "$MOUNTED" ] && error "NAME=$NAME not mounted"
-    I_MOUNTED=yes
-fi
-export MOUNT1=`mounted_lustre_filesystems | head -n 1`
-[ -z "$MOUNT1" ] && error "NAME=$NAME not mounted once"
-export MOUNT2=`mounted_lustre_filesystems | tail -n 1`
-[ "$MOUNT1" = "$MOUNT2" ] && error "NAME=$NAME not mounted twice"
-[ `mounted_lustre_filesystems | wc -l` -ne 2 ] && \
-       error "NAME=$NAME mounted more than twice"
-
-export DIR1=${DIR1:-$MOUNT1}
-export DIR2=${DIR2:-$MOUNT2}
-[ -z "`echo $DIR1 | grep $MOUNT1`" ] && echo "$DIR1 not in $MOUNT1" && exit 96
-[ -z "`echo $DIR2 | grep $MOUNT2`" ] && echo "$DIR2 not in $MOUNT2" && exit 95
+check_and_setup_lustre
 
 LPROC=/proc/fs/lustre
 LOVNAME=`cat $LPROC/llite/*/lov/common_name | tail -n 1`
@@ -778,10 +579,8 @@ test_30() { #bug #11110
 run_test 30 "recreate file race ========="
 
 log "cleanup: ======================================================"
-rm -rf $DIR1/[df][0-9]* $DIR1/lnk || true
-if [ "$I_MOUNTED" = "yes" ]; then
-    cleanup
-fi
+
+check_and_cleanup_lustre
 
 echo '=========================== finished ==============================='
 [ -f "$SANITYLOG" ] && cat $SANITYLOG && exit 1 || true
index 57bd529..728f2c5 100644 (file)
@@ -41,6 +41,7 @@ init_test_env() {
 
     export PATH=:$PATH:$LUSTRE/utils:$LUSTRE/tests
     export LCTL=${LCTL:-"$LUSTRE/utils/lctl"}
+    export LFS=${LFS:-"$LUSTRE/utils/lfs"}
     [ ! -f "$LCTL" ] && export LCTL=$(which lctl) 
     export LFS=${LFS:-"$LUSTRE/utils/lfs"}
     [ ! -f "$LFS" ] && export LFS=$(which lfs) 
@@ -60,6 +61,7 @@ init_test_env() {
     # Paths on remote nodes, if different 
     export RLUSTRE=${RLUSTRE:-$LUSTRE}
     export RPWD=${RPWD:-$PWD}
+    export I_MOUNTED=${I_MOUNTED:-"no"}
 
     # command line
     
@@ -669,11 +671,49 @@ setupall() {
     [ "$DAEMONFILE" ] && $LCTL debug_daemon start $DAEMONFILE $DAEMONSIZE
     mount_client $MOUNT
     if [ "$MOUNT_2" ]; then
-       mount_client $MOUNT2
+        mount_client $MOUNT2
     fi
     sleep 5
 }
 
+mounted_lustre_filesystems() {
+       awk '($3 ~ "lustre" && $1 ~ ":") { print $2 }' /proc/mounts
+}
+
+check_and_setup_lustre() {
+    MOUNTED="`mounted_lustre_filesystems`"
+    if [ -z "$MOUNTED" ]; then
+        [ "$REFORMAT" ] && formatall
+        setupall
+        MOUNTED="`mounted_lustre_filesystems`"
+        [ -z "$MOUNTED" ] && error "NAME=$NAME not mounted"
+        export I_MOUNTED=yes
+    fi
+    if [ "$ONLY" == "setup" ]; then
+        exit 0
+    fi
+}
+
+cleanup_and_setup_lustre() {
+    if [ "$ONLY" == "cleanup" -o "`mount | grep $MOUNT`" ]; then
+        sysctl -w lnet.debug=0 || true
+        cleanupall
+        if [ "$ONLY" == "cleanup" ]; then 
+           exit 0
+        fi
+    fi
+    check_and_setup_lustre
+}
+
+check_and_cleanup_lustre() {
+    if [ "`mount | grep $MOUNT`" ]; then
+        rm -rf $DIR/[Rdfs][1-9]*
+    fi
+    if [ "$I_MOUNTED" = "yes" ]; then
+        cleanupall -f || error "cleanup failed"
+    fi
+    unset I_MOUNTED
+}
 
 ####### 
 # General functions
@@ -846,6 +886,9 @@ build_test_filter() {
     for E in $EXCEPT $ALWAYS_EXCEPT; do
         eval EXCEPT_${E}=true
     done
+    for G in $GRANT_CHECK_LIST; do
+        eval GCHECK_ONLY_${G}=true
+       done
 }
 
 _basetest() {
@@ -902,6 +945,14 @@ log() {
     $LCTL mark "$*" 2> /dev/null || true
 }
 
+trace() {
+       log "STARTING: $*"
+       strace -o $TMP/$1.strace -ttt $*
+       RC=$?
+       log "FINISHED: $*: rc $RC"
+       return 1
+}
+
 pass() {
     echo PASS $@
 }
@@ -918,25 +969,74 @@ run_one() {
     tfile=f${testnum}
     tdir=d${base}
 
-    # Pretty tests run faster.
-    equals_msg $testnum: $message
-
     BEFORE=`date +%s`
     log "== test $testnum: $message ============ `date +%H:%M:%S` ($BEFORE)"
     #check_mds
     export TESTNAME=test_$testnum
     test_${testnum} || error "test_$testnum failed with $?"
     #check_mds
+    check_grant ${testnum} || error "check_grant $testnum failed with $?"
     [ -f $CATASTROPHE ] && [ `cat $CATASTROPHE` -ne 0 ] && \
         error "LBUG/LASSERT detected"
     pass "($((`date +%s` - $BEFORE))s)"
     unset TESTNAME
+    cd $SAVE_PWD
+    $CLEANUP
 }
 
 canonical_path() {
     (cd `dirname $1`; echo $PWD/`basename $1`)
 }
 
+sync_clients() {
+    [ -d $DIR1 ] && cd $DIR1 && sync; sleep 1; sync 
+    [ -d $DIR2 ] && cd $DIR2 && sync; sleep 1; sync 
+       cd $SAVE_PWD
+}
+
+check_grant() {
+    export base=`basetest $1`
+    [ "$CHECK_GRANT" == "no" ] && return 0
+
+       testname=GCHECK_ONLY_${base}
+        [ ${!testname}x == x ] && return 0
+
+       echo -n "checking grant......"
+       cd $SAVE_PWD
+       # write some data to sync client lost_grant
+       rm -f $DIR1/${tfile}_check_grant_* 2>&1
+       for i in `seq $OSTCOUNT`; do
+               $LFS setstripe $DIR1/${tfile}_check_grant_$i 0 $(($i -1)) 1
+               dd if=/dev/zero of=$DIR1/${tfile}_check_grant_$i bs=4k \
+                                             count=1 > /dev/null 2>&1 
+       done
+       # sync all the data and make sure no pending data on server
+       sync_clients
+       
+       #get client grant and server grant 
+       client_grant=0
+    for d in ${LPROC}/osc/*/cur_grant_bytes; do 
+               client_grant=$((client_grant + `cat $d`))
+       done
+       server_grant=0
+       for d in ${LPROC}/obdfilter/*/tot_granted; do
+               server_grant=$((server_grant + `cat $d`))
+       done
+
+       # cleanup the check_grant file
+       for i in `seq $OSTCOUNT`; do
+               rm $DIR1/${tfile}_check_grant_$i
+       done
+
+       #check whether client grant == server grant 
+       if [ $client_grant != $server_grant ]; then
+               echo "failed: client:${client_grant} server: ${server_grant}"
+               return 1
+       else
+               echo "pass"
+       fi
+}
+
 ########################
 # helper functions