LU-11596 tests: wait for grant to stop changing

Wait for the client and server grant to stop changing in sanity
test-42d, test_64[acd] to avoid intermittent failures.

Test-Parameters: trivial testlist=sanity env=ONLY=64a,ONLY_REPEAT=100
Signed-off-by: Andreas Dilger <adilger@whamcloud.com>
Change-Id: I3ef3a93d04e73ffb0098f712373c1462c53ebbe5
Reviewed-on: https://review.whamcloud.com/37876
Reviewed-by: Sebastien Buisson <sbuisson@ddn.com>
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Wei Liu <sarah@whamcloud.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>

diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh
index 6256a56..f705935 100755 (executable)
--- a/lustre/tests/sanity.sh
+++ b/lustre/tests/sanity.sh
@@ -7613,8 +7613,8 @@ run_test 63b "async write errors should be returned to fsync ==="
 test_64a () {
        [ $PARALLEL == "yes" ] && skip "skip parallel run"
 
 test_64a () {
        [ $PARALLEL == "yes" ] && skip "skip parallel run"
 

-       df $DIR
-       lctl get_param -n osc.*[oO][sS][cC][_-]*.cur* | grep "[0-9]"
+       lfs df $DIR
+       lctl get_param osc.*[oO][sS][cC][_-]*.cur* | grep "=[1-9]"

 }
 run_test 64a "verify filter grant calculations (in kernel) ====="
 
 }
 run_test 64a "verify filter grant calculations (in kernel) ====="
 

diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh
index aa1e63b..3e7822a 100755 (executable)
--- a/lustre/tests/test-framework.sh
+++ b/lustre/tests/test-framework.sh
@@ -6219,22 +6219,22 @@ build_test_filter() {
                fi
        done
 
                fi
        done
 

-    [ "$EXCEPT$ALWAYS_EXCEPT" ] && \
-        log "excepting tests: `echo $EXCEPT $ALWAYS_EXCEPT`"
-    [ "$EXCEPT_SLOW" ] && \
-        log "skipping tests SLOW=no: `echo $EXCEPT_SLOW`"
-    for E in $EXCEPT; do
-        eval EXCEPT_${E}=true
-    done
-    for E in $ALWAYS_EXCEPT; do
-        eval EXCEPT_ALWAYS_${E}=true
-    done
-    for E in $EXCEPT_SLOW; do
-        eval EXCEPT_SLOW_${E}=true
-    done
-    for G in $GRANT_CHECK_LIST; do
-        eval GCHECK_ONLY_${G}=true
-        done
+       [ "$EXCEPT$ALWAYS_EXCEPT" ] &&
+               log "excepting tests: `echo $EXCEPT $ALWAYS_EXCEPT`"
+       [ "$EXCEPT_SLOW" ] &&
+               log "skipping tests SLOW=no: `echo $EXCEPT_SLOW`"
+       for E in $EXCEPT; do
+               eval EXCEPT_${E}=true
+       done
+       for E in $ALWAYS_EXCEPT; do
+               eval EXCEPT_ALWAYS_${E}=true
+       done
+       for E in $EXCEPT_SLOW; do
+               eval EXCEPT_SLOW_${E}=true
+       done
+       for G in $GRANT_CHECK_LIST; do
+               eval GCHECK_ONLY_${G}=true
+       done

 }
 
 basetest() {
 }
 
 basetest() {


@@ -6547,6 +6547,27 @@ canonical_path() {
        (cd $(dirname $1); echo $PWD/$(basename $1))
 }
 
        (cd $(dirname $1); echo $PWD/$(basename $1))
 }
 

+grant_from_clients() {
+       local nodes="$1"
+
+       # get client grant
+       do_nodes $nodes "$LCTL get_param -n osc.${FSNAME}-*.cur_*grant_bytes" |
+               calc_sum
+}
+
+grant_from_servers() {
+       local nodes="$1"
+
+       # get server grant
+       # which is tot_granted less grant_precreate
+       do_nodes $nodes "$LCTL get_param obdfilter.${FSNAME}-OST*.tot_granted" \
+               " obdfilter.${FSNAME}-OST*.tot_pending" \
+               " obdfilter.${FSNAME}-OST*.grant_precreate" |
+               tr '=' ' ' | awk '/tot_granted/{ total += $2 };
+                                 /tot_pending/{ total -= $2 };
+                                 /grant_precreate/{ total -= $2 };
+                                 END { printf("%0.0f", total) }'
+}

 
 check_grant() {
        export base=$(basetest $1)
 
 check_grant() {
        export base=$(basetest $1)


@@ -6558,6 +6579,7 @@ check_grant() {
 
        echo -n "checking grant......"
 
 
        echo -n "checking grant......"
 

+       local osts=$(comma_list $(osts_nodes))

        local clients=$CLIENTS
        [ -z "$clients" ] && clients=$(hostname)
 
        local clients=$CLIENTS
        [ -z "$clients" ] && clients=$(hostname)
 


@@ -6566,29 +6588,28 @@ check_grant() {
        clients_up # initiate all idling connections
 
        # get client grant
        clients_up # initiate all idling connections
 
        # get client grant

-       client_grant=$(do_nodes $clients \
-               "$LCTL get_param -n osc.${FSNAME}-*.cur_*grant_bytes" |
-               awk '{ total += $1 } END { printf("%0.0f", total) }')
+       cli_grant=$(grant_from_clients $clients)

 
        # get server grant
        # which is tot_granted less grant_precreate
 
        # get server grant
        # which is tot_granted less grant_precreate

-       server_grant=$(do_nodes $(comma_list $(osts_nodes)) \
-               "$LCTL get_param "\
-               "obdfilter.${FSNAME}-OST*.{tot_granted,tot_pending,grant_precreate}" |
-               sed 's/=/ /'| awk '/tot_granted/{ total += $2 };
-                               /tot_pending/{ total -= $2 };
-                               /grant_precreate/{ total -= $2 };
-                               END { printf("%0.0f", total) }')
+       srv_grant=$(grant_from_servers $osts)

 
 

+       count=0

        # check whether client grant == server grant
        # check whether client grant == server grant

-       if [[ $client_grant -ne $server_grant ]]; then
+       while [[ $cli_grant != $srv_grant && count++ -lt 30 ]]; do
+               echo "wait for client:$cli_grant == server:$srv_grant"
+               sleep 1
+               cli_grant=$(grant_from_clients $clients)
+               srv_grant=$(grant_from_servers $osts)
+       done
+       if [[ $cli_grant -ne $srv_grant ]]; then

                do_nodes $(comma_list $(osts_nodes)) \
                        "$LCTL get_param obdfilter.${FSNAME}-OST*.tot*" \
                do_nodes $(comma_list $(osts_nodes)) \
                        "$LCTL get_param obdfilter.${FSNAME}-OST*.tot*" \

-                       "obdfilter.${FSNAME}-OST*.grant_*"
+                       "obdfilter.${FSNAME}-OST*.grant_*"

                do_nodes $clients "$LCTL get_param osc.${FSNAME}-*.cur_*_bytes"
                do_nodes $clients "$LCTL get_param osc.${FSNAME}-*.cur_*_bytes"

-               error "failed: client:${client_grant} server: ${server_grant}."
+               error "failed grant check: client:$cli_grant server:$srv_grant"

        else
        else

-               echo "pass: client:${client_grant} server: ${server_grant}"
+               echo "pass grant check: client:$cli_grant server:$srv_grant"

        fi
 }
 
        fi
 }