From 75ebfa0b924a41d15aea026dc9a29bec39d1419a Mon Sep 17 00:00:00 2001 From: Andreas Dilger Date: Tue, 10 Mar 2020 22:52:35 -0600 Subject: [PATCH] LU-11596 tests: wait for grant to stop changing Wait for the client and server grant to stop changing in sanity test-42d, test_64[acd] to avoid intermittent failures. Test-Parameters: trivial testlist=sanity env=ONLY=64a,ONLY_REPEAT=100 Signed-off-by: Andreas Dilger Change-Id: I3ef3a93d04e73ffb0098f712373c1462c53ebbe5 Reviewed-on: https://review.whamcloud.com/37876 Reviewed-by: Sebastien Buisson Tested-by: jenkins Tested-by: Maloo Reviewed-by: Wei Liu Reviewed-by: Oleg Drokin --- lustre/tests/sanity.sh | 4 +-- lustre/tests/test-framework.sh | 81 ++++++++++++++++++++++++++---------------- 2 files changed, 53 insertions(+), 32 deletions(-) diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh index 6256a56..f705935 100755 --- a/lustre/tests/sanity.sh +++ b/lustre/tests/sanity.sh @@ -7613,8 +7613,8 @@ run_test 63b "async write errors should be returned to fsync ===" test_64a () { [ $PARALLEL == "yes" ] && skip "skip parallel run" - df $DIR - lctl get_param -n osc.*[oO][sS][cC][_-]*.cur* | grep "[0-9]" + lfs df $DIR + lctl get_param osc.*[oO][sS][cC][_-]*.cur* | grep "=[1-9]" } run_test 64a "verify filter grant calculations (in kernel) =====" diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh index aa1e63b..3e7822a 100755 --- a/lustre/tests/test-framework.sh +++ b/lustre/tests/test-framework.sh @@ -6219,22 +6219,22 @@ build_test_filter() { fi done - [ "$EXCEPT$ALWAYS_EXCEPT" ] && \ - log "excepting tests: `echo $EXCEPT $ALWAYS_EXCEPT`" - [ "$EXCEPT_SLOW" ] && \ - log "skipping tests SLOW=no: `echo $EXCEPT_SLOW`" - for E in $EXCEPT; do - eval EXCEPT_${E}=true - done - for E in $ALWAYS_EXCEPT; do - eval EXCEPT_ALWAYS_${E}=true - done - for E in $EXCEPT_SLOW; do - eval EXCEPT_SLOW_${E}=true - done - for G in $GRANT_CHECK_LIST; do - eval GCHECK_ONLY_${G}=true - done + [ "$EXCEPT$ALWAYS_EXCEPT" ] && + log "excepting tests: `echo $EXCEPT $ALWAYS_EXCEPT`" + [ "$EXCEPT_SLOW" ] && + log "skipping tests SLOW=no: `echo $EXCEPT_SLOW`" + for E in $EXCEPT; do + eval EXCEPT_${E}=true + done + for E in $ALWAYS_EXCEPT; do + eval EXCEPT_ALWAYS_${E}=true + done + for E in $EXCEPT_SLOW; do + eval EXCEPT_SLOW_${E}=true + done + for G in $GRANT_CHECK_LIST; do + eval GCHECK_ONLY_${G}=true + done } basetest() { @@ -6547,6 +6547,27 @@ canonical_path() { (cd $(dirname $1); echo $PWD/$(basename $1)) } +grant_from_clients() { + local nodes="$1" + + # get client grant + do_nodes $nodes "$LCTL get_param -n osc.${FSNAME}-*.cur_*grant_bytes" | + calc_sum +} + +grant_from_servers() { + local nodes="$1" + + # get server grant + # which is tot_granted less grant_precreate + do_nodes $nodes "$LCTL get_param obdfilter.${FSNAME}-OST*.tot_granted" \ + " obdfilter.${FSNAME}-OST*.tot_pending" \ + " obdfilter.${FSNAME}-OST*.grant_precreate" | + tr '=' ' ' | awk '/tot_granted/{ total += $2 }; + /tot_pending/{ total -= $2 }; + /grant_precreate/{ total -= $2 }; + END { printf("%0.0f", total) }' +} check_grant() { export base=$(basetest $1) @@ -6558,6 +6579,7 @@ check_grant() { echo -n "checking grant......" + local osts=$(comma_list $(osts_nodes)) local clients=$CLIENTS [ -z "$clients" ] && clients=$(hostname) @@ -6566,29 +6588,28 @@ check_grant() { clients_up # initiate all idling connections # get client grant - client_grant=$(do_nodes $clients \ - "$LCTL get_param -n osc.${FSNAME}-*.cur_*grant_bytes" | - awk '{ total += $1 } END { printf("%0.0f", total) }') + cli_grant=$(grant_from_clients $clients) # get server grant # which is tot_granted less grant_precreate - server_grant=$(do_nodes $(comma_list $(osts_nodes)) \ - "$LCTL get_param "\ - "obdfilter.${FSNAME}-OST*.{tot_granted,tot_pending,grant_precreate}" | - sed 's/=/ /'| awk '/tot_granted/{ total += $2 }; - /tot_pending/{ total -= $2 }; - /grant_precreate/{ total -= $2 }; - END { printf("%0.0f", total) }') + srv_grant=$(grant_from_servers $osts) + count=0 # check whether client grant == server grant - if [[ $client_grant -ne $server_grant ]]; then + while [[ $cli_grant != $srv_grant && count++ -lt 30 ]]; do + echo "wait for client:$cli_grant == server:$srv_grant" + sleep 1 + cli_grant=$(grant_from_clients $clients) + srv_grant=$(grant_from_servers $osts) + done + if [[ $cli_grant -ne $srv_grant ]]; then do_nodes $(comma_list $(osts_nodes)) \ "$LCTL get_param obdfilter.${FSNAME}-OST*.tot*" \ - "obdfilter.${FSNAME}-OST*.grant_*" + "obdfilter.${FSNAME}-OST*.grant_*" do_nodes $clients "$LCTL get_param osc.${FSNAME}-*.cur_*_bytes" - error "failed: client:${client_grant} server: ${server_grant}." + error "failed grant check: client:$cli_grant server:$srv_grant" else - echo "pass: client:${client_grant} server: ${server_grant}" + echo "pass grant check: client:$cli_grant server:$srv_grant" fi } -- 1.8.3.1