LU-8627 test: wait on MDS for ost-pool proc entry to update

Many of the ost-pools test cases fail on the setup having
separate MGS and MDS.

The lctl pool commands are ran on MGS. The tests involving the
pools fail if the pool configurations are not updated on client
and/or MDS side.

This patch adds wait for pool configuration propogation on MDS(s)
on various occassions like pool creation, adding targets to the
pool, removing targets from the pool, destroying the pool, etc.

Seagate-Bug-id: MRP-2929
Signed-off-by: Bhagyesh Dudhediya <bhagyesh.dudhediya@seagate.com>
Change-Id: Ib4bde368f8068b0d32f89e905cf80a27032bd41e
Reviewed-on: http://es-gerrit.xyus.xyratex.com:8080/10132
Reviewed-by: Ashish Purkar <ashish.purkar@seagate.com>
Reviewed-by: Ujjwal Lanjewar <ujjwal.lanjewar@seagate.com>
Reviewed-by: Elena V. Gryaznova <elena.gryaznova@seagate.com>
Tested-by: Jenkins
Tested-by: Elena V. Gryaznova <elena.gryaznova@seagate.com>
Reviewed-on: https://review.whamcloud.com/22653
Tested-by: Maloo <hpdd-maloo@intel.com>
Reviewed-by: Andreas Dilger <andreas.dilger@intel.com>
Reviewed-by: James Nunez <james.a.nunez@intel.com>

diff --git a/lustre/tests/ost-pools.sh b/lustre/tests/ost-pools.sh
index 307c4fa..23f610b 100755 (executable)
--- a/lustre/tests/ost-pools.sh
+++ b/lustre/tests/ost-pools.sh
@@ -178,9 +178,18 @@ add_pool() {
        local RC=$?
        [[ $RC -ne 0 ]] && return $RC
 
+       # wait for OSTs to be added to the pool
+       for mds_id in $(seq $MDSCOUNT); do
+               local mdt_id=$((mds_id-1))
+               local lodname=$FSNAME-MDT$(printf "%04x" $mdt_id)-mdtlov
+               wait_update_facet mds$mds_id \
+                       "lctl get_param -n lod.$lodname.pools.$pool |
+                               sort -u | tr '\n' ' ' " "$tgt" >/dev/null ||
+                       error "mds$mds_id:pool add failed $1; $2"
+       done
        wait_update $HOSTNAME "lctl get_param -n lov.$FSNAME-*.pools.$pool |
-               sort -u | tr '\n' ' ' " "$tgt" >/dev/null || RC=1
-       [[ $RC -ne 0 ]] && error "pool_add failed: $1; $2"
+               sort -u | tr '\n' ' ' " "$tgt" >/dev/null ||
+               error "pool_add failed: $1; $2"
        return $RC
 }
 

diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh
index 96c242e..1a13b48 100755 (executable)
--- a/lustre/tests/test-framework.sh
+++ b/lustre/tests/test-framework.sh
@@ -6224,22 +6224,27 @@ pool_list () {
 }
 
 create_pool() {
-    local fsname=${1%%.*}
-    local poolname=${1##$fsname.}
-
-    do_facet mgs lctl pool_new $1
-    local RC=$?
-    # get param should return err unless pool is created
-    [[ $RC -ne 0 ]] && return $RC
-
-    wait_update $HOSTNAME "lctl get_param -n lov.$fsname-*.pools.$poolname \
-        2>/dev/null || echo foo" "" || RC=1
-    if [[ $RC -eq 0 ]]; then
-        add_pool_to_list $1
-    else
-        error "pool_new failed $1"
-    fi
-    return $RC
+       local fsname=${1%%.*}
+       local poolname=${1##$fsname.}
+
+       do_facet mgs lctl pool_new $1
+       local RC=$?
+       # get param should return err unless pool is created
+       [[ $RC -ne 0 ]] && return $RC
+
+       for mds_id in $(seq $MDSCOUNT); do
+               local mdt_id=$((mds_id-1))
+               local lodname=$fsname-MDT$(printf "%04x" $mdt_id)-mdtlov
+               wait_update_facet mds$mds_id \
+                       "lctl get_param -n lod.$lodname.pools.$poolname \
+                               2>/dev/null || echo foo" "" ||
+                       error "mds$mds_id: pool_new failed $1"
+       done
+       wait_update $HOSTNAME "lctl get_param -n lov.$fsname-*.pools.$poolname \
+               2>/dev/null || echo foo" "" || error "pool_new failed $1"
+
+       add_pool_to_list $1
+       return $RC
 }
 
 add_pool_to_list () {
@@ -6272,28 +6277,31 @@ destroy_pool_int() {
 
 # <fsname>.<poolname> or <poolname>
 destroy_pool() {
-    local fsname=${1%%.*}
-    local poolname=${1##$fsname.}
-
-    [[ x$fsname = x$poolname ]] && fsname=$FSNAME
+       local fsname=${1%%.*}
+       local poolname=${1##$fsname.}
 
-    local RC
+       [[ x$fsname = x$poolname ]] && fsname=$FSNAME
 
-    pool_list $fsname.$poolname || return $?
+       local RC
 
-    destroy_pool_int $fsname.$poolname
-    RC=$?
-    [[ $RC -ne 0 ]] && return $RC
+       pool_list $fsname.$poolname || return $?
 
-    wait_update $HOSTNAME "lctl get_param -n lov.$fsname-*.pools.$poolname \
-      2>/dev/null || echo foo" "foo" || RC=1
+       destroy_pool_int $fsname.$poolname
+       RC=$?
+       [[ $RC -ne 0 ]] && return $RC
+       for mds_id in $(seq $MDSCOUNT); do
+               local mdt_id=$((mds_id-1))
+               local lodname=$fsname-MDT$(printf "%04x" $mdt_id)-mdtlov
+               wait_update_facet mds$mds_id \
+                       "lctl get_param -n lod.$lodname.pools.$poolname \
+                               2>/dev/null || echo foo" "foo" ||
+                       error "mds$mds_id: destroy pool failed $1"
+       done
+       wait_update $HOSTNAME "lctl get_param -n lov.$fsname-*.pools.$poolname \
+               2>/dev/null || echo foo" "foo" || error "destroy pool failed $1"
 
-    if [[ $RC -eq 0 ]]; then
-        remove_pool_from_list $fsname.$poolname
-    else
-        error "destroy pool failed $1"
-    fi
-    return $RC
+       remove_pool_from_list $fsname.$poolname
+       return $RC
 }
 
 destroy_pools () {
@@ -7291,6 +7299,18 @@ pool_add_targets() {
        local t=$(for i in $list; do printf "$FSNAME-OST%04x_UUID " $i; done)
        do_facet mgs $LCTL pool_add \
                        $FSNAME.$pool $FSNAME-OST[$first-$last/$step]
+
+       # wait for OSTs to be added to the pool
+       for mds_id in $(seq $MDSCOUNT); do
+               local mdt_id=$((mds_id-1))
+               local lodname=$FSNAME-MDT$(printf "%04x" $mdt_id)-mdtlov
+               wait_update_facet mds$mds_id \
+                       "lctl get_param -n lod.$lodname.pools.$pool |
+                               sort -u | tr '\n' ' ' " "$t" || {
+                       error_noexit "mds$mds_id: Add to pool failed"
+                       return 3
+               }
+       done
        wait_update $HOSTNAME "lctl get_param -n lov.$FSNAME-*.pools.$pool \
                        | sort -u | tr '\n' ' ' " "$t" || {
                error_noexit "Add to pool failed"
@@ -7427,6 +7447,17 @@ pool_remove_first_target() {
        local pname="lov.$FSNAME-*.pools.$pool"
        local t=$($LCTL get_param -n $pname | head -1)
        do_facet mgs $LCTL pool_remove $FSNAME.$pool $t
+       for mds_id in $(seq $MDSCOUNT); do
+               local mdt_id=$((mds_id-1))
+               local lodname=$FSNAME-MDT$(printf "%04x" $mdt_id)-mdtlov
+               wait_update_facet mds$mds_id \
+                       "lctl get_param -n lod.$lodname.pools.$pool |
+                               grep $t" "" || {
+                       error_noexit "mds$mds_id: $t not removed from" \
+                       "$FSNAME.$pool"
+                       return 2
+               }
+       done
        wait_update $HOSTNAME "lctl get_param -n $pname | grep $t" "" || {
                error_noexit "$t not removed from $FSNAME.$pool"
                return 1
@@ -7442,6 +7473,15 @@ pool_remove_all_targets() {
        do
                do_facet mgs $LCTL pool_remove $FSNAME.$pool $t
        done
+       for mds_id in $(seq $MDSCOUNT); do
+               local mdt_id=$((mds_id-1))
+               local lodname=$FSNAME-MDT$(printf "%04x" $mdt_id)-mdtlov
+               wait_update_facet mds$mds_id "lctl get_param -n \
+                       lod.$lodname.pools.$pool" "" || {
+                       error_noexit "mds$mds_id: Pool $pool not drained"
+                       return 4
+               }
+       done
        wait_update $HOSTNAME "lctl get_param -n $pname" "" || {
                error_noexit "Pool $FSNAME.$pool cannot be drained"
                return 1