LU-16529 test: wait quota synced on quota slaves

Check and wait the quota setting to be synchronized on
quota slaves before running actual sanity-quota test_84.

Test-Parameters: trivial testlist=sanity-quota env=ONLY=84,ONLY_REPEAT=100
Fixes: a2fd4d3aee ("LU-15880 quota: fix insane grant quota")
Signed-off-by: Hongchao Zhang <hongchao@whamcloud.com>
Change-Id: I7752bff33f24d1d38dc340b2addbfc98d6f7c857
Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/50197
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>
Reviewed-by: Sergey Cheremencev <scherementsev@ddn.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>

diff --git a/lustre/tests/sanity-quota.sh b/lustre/tests/sanity-quota.sh
index b6dfed9..8789429 100755 (executable)
--- a/lustre/tests/sanity-quota.sh
+++ b/lustre/tests/sanity-quota.sh
@@ -495,6 +495,55 @@ reset_quota_settings() {
        sleep 1
 }
 
+get_quota_on_qsd() {
+       local facet
+       local spec
+       local qid
+       local qtype
+       local output
+
+       facet=$1
+       case "$2" in
+               usr) qtype="limit_user";;
+               grp) qtype="limit_group";;
+               *)         error "unknown quota parameter $2";;
+       esac
+
+       qid=$3
+       case "$4" in
+               hardlimit) spec=4;;
+               softlimit) spec=6;;
+               *)         error "unknown quota parameter $4";;
+       esac
+
+       do_facet $facet $LCTL get_param osd-*.*-OST0000.quota_slave.$qtype |
+               awk '($3 == '$qid') {getline; print $'$spec'; exit;}' | tr -d ,
+}
+
+wait_quota_setting_synced() {
+       local value
+       local qtype=$1
+       local qid=$2
+       local limit_type=$3
+       local limit_val=$4
+       local interval=0
+
+       value=$(get_quota_on_qsd ost1 $qtype $qid $limit_type)
+       while [[ $value != $limit_val ]]; do
+               (( interval != 0 )) ||
+                       do_facet ost1 $LCTL set_param \
+                               osd-*.*-OST0000.quota_slave.force_reint=1
+
+               (( interval <= 20 )) ||
+                       error "quota ($value) don't update on QSD, $limit_val"
+
+               interval=$((interval + 1))
+               sleep 1
+
+               value=$(get_quota_on_qsd ost1 $qtype $qid $limit_type)
+       done
+}
+
 # enable quota debug
 quota_init() {
        do_nodes $(comma_list $(nodes_list)) \
@@ -5709,6 +5758,8 @@ test_84()
                 error "set user quota failed"
        $LFS quota -gv $TSTUSR $DIR
 
+       wait_quota_setting_synced "grp" $TSTID "hardlimit" $((10*1024*1024))
+
        mkdir -p $dir1 || error "failed to mkdir"
        chown $TSTUSR.$TSTUSR $dir1 || error "chown $dir1 failed"