LU-1526 tests: Adapt oos to the new grant and osd-zfs behavior

This patch is backported from commit 938947f of LU-1415 to
support oos interoperating testing with 2.4 server.

The patch moves the oos checking codes from oos.sh and
oos2.sh into common function oos_full() in test-framework.sh.

The oos_full() allows 1% of total space in bavail because of
delayed allocation with ZFS which might release some free space
after txg commit.

Signed-off-by: Yu Jian <yujian@whamcloud.com>
Change-Id: I8933a6c8614a96d7bab5689c9d3e3a1f59bbc133
Reviewed-on: http://review.whamcloud.com/4986
Tested-by: Hudson
Reviewed-by: Niu Yawei <yawei.niu@intel.com>
Reviewed-by: Li Wei <wei.g.li@intel.com>
Reviewed-by: Andreas Dilger <andreas.dilger@intel.com>
Reviewed-by: wangdi <di.wang@intel.com>
Tested-by: Maloo <whamcloud.maloo@gmail.com>

diff --git a/lustre/tests/oos.sh b/lustre/tests/oos.sh
index b313a07..98d2c18 100755 (executable)
--- a/lustre/tests/oos.sh
+++ b/lustre/tests/oos.sh
@@ -5,6 +5,7 @@ set -e
 
 LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)}
 . $LUSTRE/tests/test-framework.sh
+. ${CONFIG:=$LUSTRE/tests/cfg/${NAME}.sh}
 
 export PATH=`dirname $0`/../utils:$PATH
 LFS=${LFS:-lfs}
@@ -41,16 +42,15 @@ export LANG=C LC_LANG=C # for "No space left on device" message
 [ -f $LOG ] && error "log file wasn't removed?"
 
 echo BEFORE dd started
-for OSC in `$LCTL get_param -N osc.*-osc-*.kbytesavail | cut -d"." -f1-2`; do
-       AVAIL=`$LCTL get_param -n $OSC.kbytesavail`
-       GRANT=$((`$LCTL get_param -n $OSC.cur_grant_bytes` / 1024))
-       echo -n "$(echo $OSC | cut -d"." -f2) avl=$AVAIL grnt=$GRANT diff=$(($AVAIL - $GRANT))"
-       echo " "
-done
+oos_full || true
 
 # make sure we stripe over all OSTs to avoid OOS on only a subset of OSTs
 $LFS setstripe $OOS -c $STRIPECOUNT
-if dd if=/dev/zero of=$OOS count=$(($ORIGFREE + 100)) bs=1k 2> $LOG; then
+# add 20% of margin since the metadata overhead estimated in bavail might be
+# too aggressive and we might be able to write more than reported initially
+# by statfs.
+echo dd size $((ORIGFREE * 120 / 100))kB
+if dd if=/dev/zero of=$OOS count=$((ORIGFREE * 120 / 100)) bs=1k 2> $LOG; then
        echo "ERROR: dd did not fail"
        SUCCESS=0
 fi
@@ -67,18 +67,8 @@ fi
 sync; sleep 1 ; sync
 
 echo AFTER dd
-for OSC in `$LCTL get_param -N osc.*-osc-*.kbytesavail | cut -d"." -f1-2`; do
-       AVAIL=`$LCTL get_param -n $OSC.kbytesavail`
-       GRANT=$((`$LCTL get_param -n $OSC.cur_grant_bytes` / 1024))
-       echo -n "$(echo $OSC | cut -d"." -f2) avl=$AVAIL grnt=$GRANT diff=$(($AVAIL - $GRANT))"
-       [ $(($AVAIL - $GRANT)) -lt 400 ] && OSCFULL=full && echo -n " FULL"
-       echo " "
-done
-
-if [ -z "$OSCFULL" ]; then
+if ! oos_full; then
        echo "no OSTs are close to full"
-       $LCTL get_param "osc.*-osc-*.kbytesavail"
-       $LCTL get_param "osc.*-osc-*.cur*"
        SUCCESS=0
 fi
 
@@ -95,10 +85,10 @@ fi
 
 #$LCTL debug_daemon stop
 
-echo LOG file
-cat $LOG
+[ $SUCCESS != 0 ] && echo LOG file && sed "s/^/LOG: /" $LOG
 rm -f $OOS
-sync; sleep 1; sync
+
+sync; sleep 3; sync
 
 wait_delete_completed 300
 

diff --git a/lustre/tests/oos2.sh b/lustre/tests/oos2.sh
old mode 100644 (file)
new mode 100755 (executable)
index 6437fa4..64a0f35
--- a/lustre/tests/oos2.sh
+++ b/lustre/tests/oos2.sh
@@ -4,6 +4,7 @@ set -e
 
 LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)}
 . $LUSTRE/tests/test-framework.sh
+. ${CONFIG:=$LUSTRE/tests/cfg/${NAME}.sh}
 
 export PATH=$LUSTRE/utils:$PATH
 LFS=${LFS:-lfs}
@@ -60,25 +61,8 @@ fi
 # flush cache to OST(s) so avail numbers are correct
 sync; sleep 1 ; sync
 
-for OSC in `$LCTL get_param -N osc.*-osc-*.kbytesavail | cut -d"." -f1-2`; do
-       AVAIL=`$LCTL get_param -n $OSC.kbytesavail`
-       GRANT=$((`$LCTL get_param -n $OSC.cur_grant_bytes` / 1024))
-       echo -n "$(echo $OSC | cut -d"." -f2) avl=$AVAIL grnt=$GRANT diff=$(($AVAIL - $GRANT))"
-       [ $(($AVAIL - $GRANT)) -lt 400 ] && OSCFULL=full && echo -n " FULL"
-       echo " "
-done
-
-# FIXME - This test reports false failures
-# The grants from multiple clients need to be added together and compared 
-# against the kbytesavail.
-#/proc/fs/lustre/osc/lustre-OST0001-osc-c3b04200/kbytesavail:16248
-#/proc/fs/lustre/osc/lustre-OST0001-osc-c3b04200/cur_grant_bytes:4313088
-#/proc/fs/lustre/osc/lustre-OST0001-osc-c3b04e00/cur_grant_bytes:12660736
-
-if [ -z "$OSCFULL" ]; then
+if ! oos_full; then
        echo "no OSTs are close to full"
-       $LCTL get_param "osc.*-osc-*.kbytesavail"
-       $LCTL get_param "osc.*-osc-*.cur*"
        SUCCESS=0
 fi
 

diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh
index 02ffed1..b0455e7 100644 (file)
--- a/lustre/tests/test-framework.sh
+++ b/lustre/tests/test-framework.sh
@@ -224,6 +224,33 @@ case `uname -r` in
     *) EXT=".ko"; USE_QUOTA=yes;;
 esac
 
+oos_full() {
+       local -a AVAILA
+       local -a GRANTA
+       local -a TOTALA
+       local OSCFULL=1
+       AVAILA=($(do_nodes $(comma_list $(osts_nodes)) \
+                 $LCTL get_param obdfilter.*.kbytesavail))
+       GRANTA=($(do_nodes $(comma_list $(osts_nodes)) \
+                 $LCTL get_param -n obdfilter.*.tot_granted))
+       TOTALA=($(do_nodes $(comma_list $(osts_nodes)) \
+                 $LCTL get_param -n obdfilter.*.kbytestotal))
+       for ((i=0; i<${#AVAILA[@]}; i++)); do
+               local -a AVAIL1=(${AVAILA[$i]//=/ })
+               local -a TOTAL=(${TOTALA[$i]//=/ })
+               GRANT=$((${GRANTA[$i]}/1024))
+               # allow 1% of total space in bavail because of delayed
+               # allocation with ZFS which might release some free space after
+               # txg commit.  For small devices, we set a mininum of 8MB
+               local LIMIT=$((${TOTAL} / 100 + 8000))
+               echo -n $(echo ${AVAIL1[0]} | cut -d"." -f2) avl=${AVAIL1[1]} \
+                       grnt=$GRANT diff=$((AVAIL1[1] - GRANT)) limit=${LIMIT}
+               [ $((AVAIL1[1] - GRANT)) -lt $LIMIT ] && OSCFULL=0 && \
+                       echo " FULL" || echo
+       done
+       return $OSCFULL
+}
+
 pool_list () {
    do_facet mgs lctl pool_list $1
 }
@@ -1355,7 +1382,7 @@ wait_delete_completed_mds() {
        local changes
 
        # find MDS with pending deletions
-       for node in $(mdts_nodes); do
+       for node in $(facet_host mds); do
                changes=$(do_node $node "lctl get_param -n osc.*MDT*.sync_*" \
                        2>/dev/null | calc_sum)
                if [ -z "$changes" ] || [ $changes -eq 0 ]; then