LU-12778 tests: give time to apply nodemap

As nodemap definitions can need time before they are taken into
account, retry several times before declaring the nodemap is not
updated.

Test-Parameters: trivial testlist=sanity-sec,sanity-sec,sanity-sec,sanity-sec
Test-Parameters: trivial testlist=sanity-sec,sanity-sec,sanity-sec,sanity-sec
Test-Parameters: trivial testlist=sanity-sec,sanity-sec,sanity-sec,sanity-sec
Signed-off-by: Sebastien Buisson <sbuisson@ddn.com>
Change-Id: I632bd100ee62e3604aed3aaabc826e7a32287234
Reviewed-on: https://review.whamcloud.com/36280
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Emoly Liu <emoly@whamcloud.com>
Reviewed-by: James Nunez <jnunez@whamcloud.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>

diff --git a/lustre/tests/sanity-sec.sh b/lustre/tests/sanity-sec.sh
index a268ab1..b77e1ab 100755 (executable)
--- a/lustre/tests/sanity-sec.sh
+++ b/lustre/tests/sanity-sec.sh
@@ -219,7 +219,6 @@ run_test 4 "set supplementary group ==============="
 
 create_nodemaps() {
        local i
-       local out
        local rc
 
        squash_id default 99 0
@@ -236,9 +235,10 @@ create_nodemaps() {
                        return $rc
                fi
 
-               out=$(do_facet mgs $LCTL get_param nodemap.$csum.id)
-               ## This needs to return zero if the following statement is 1
-               [[ $(echo $out | grep -c $csum) == 0 ]] && return 1
+               wait_update_facet --verbose mgs \
+                       "$LCTL get_param nodemap.$csum.id 2>/dev/null | \
+                       grep -c $csum || true" 1 30 ||
+                   return 1
        done
        for (( i = 0; i < NODEMAP_COUNT; i++ )); do
                local csum=${HOSTNAME_CHECKSUM}_${i}
@@ -250,7 +250,6 @@ create_nodemaps() {
 
 delete_nodemaps() {
        local i
-       local out
 
        for ((i = 0; i < NODEMAP_COUNT; i++)); do
                local csum=${HOSTNAME_CHECKSUM}_${i}
@@ -260,8 +259,10 @@ delete_nodemaps() {
                        return 3
                fi
 
-               out=$(do_facet mgs $LCTL get_param nodemap.$csum.id 2>/dev/null)
-               [[ $(echo $out | grep -c $csum) != 0 ]] && return 1
+               wait_update_facet --verbose mgs \
+                       "$LCTL get_param nodemap.$csum.id 2>/dev/null | \
+                       grep -c $csum || true" 0 30 ||
+                   return 1
        done
        for (( i = 0; i < NODEMAP_COUNT; i++ )); do
                local csum=${HOSTNAME_CHECKSUM}_${i}