From 385dd38158e540a126207ab39af1669cca078de2 Mon Sep 17 00:00:00 2001 From: Bhagyesh Dudhediya Date: Wed, 30 Nov 2016 19:36:04 +0530 Subject: [PATCH] LU-8627 test: wait on MDS for ost-pool proc entry to update Many of the ost-pools test cases fail on the setup having separate MGS and MDS. The lctl pool commands are ran on MGS. The tests involving the pools fail if the pool configurations are not updated on client and/or MDS side. This patch adds wait for pool configuration propogation on MDS(s) on various occassions like pool creation, adding targets to the pool, removing targets from the pool, destroying the pool, etc. Seagate-Bug-id: MRP-2929 Signed-off-by: Bhagyesh Dudhediya Change-Id: Ib4bde368f8068b0d32f89e905cf80a27032bd41e Reviewed-on: http://es-gerrit.xyus.xyratex.com:8080/10132 Reviewed-by: Ashish Purkar Reviewed-by: Ujjwal Lanjewar Reviewed-by: Elena V. Gryaznova Tested-by: Jenkins Tested-by: Elena V. Gryaznova Reviewed-on: https://review.whamcloud.com/22653 Tested-by: Maloo Reviewed-by: Andreas Dilger Reviewed-by: James Nunez --- lustre/tests/ost-pools.sh | 13 ++++- lustre/tests/test-framework.sh | 106 ++++++++++++++++++++++++++++------------- 2 files changed, 84 insertions(+), 35 deletions(-) diff --git a/lustre/tests/ost-pools.sh b/lustre/tests/ost-pools.sh index 307c4fa..23f610b 100755 --- a/lustre/tests/ost-pools.sh +++ b/lustre/tests/ost-pools.sh @@ -178,9 +178,18 @@ add_pool() { local RC=$? [[ $RC -ne 0 ]] && return $RC + # wait for OSTs to be added to the pool + for mds_id in $(seq $MDSCOUNT); do + local mdt_id=$((mds_id-1)) + local lodname=$FSNAME-MDT$(printf "%04x" $mdt_id)-mdtlov + wait_update_facet mds$mds_id \ + "lctl get_param -n lod.$lodname.pools.$pool | + sort -u | tr '\n' ' ' " "$tgt" >/dev/null || + error "mds$mds_id:pool add failed $1; $2" + done wait_update $HOSTNAME "lctl get_param -n lov.$FSNAME-*.pools.$pool | - sort -u | tr '\n' ' ' " "$tgt" >/dev/null || RC=1 - [[ $RC -ne 0 ]] && error "pool_add failed: $1; $2" + sort -u | tr '\n' ' ' " "$tgt" >/dev/null || + error "pool_add failed: $1; $2" return $RC } diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh index 96c242e..1a13b48 100755 --- a/lustre/tests/test-framework.sh +++ b/lustre/tests/test-framework.sh @@ -6224,22 +6224,27 @@ pool_list () { } create_pool() { - local fsname=${1%%.*} - local poolname=${1##$fsname.} - - do_facet mgs lctl pool_new $1 - local RC=$? - # get param should return err unless pool is created - [[ $RC -ne 0 ]] && return $RC - - wait_update $HOSTNAME "lctl get_param -n lov.$fsname-*.pools.$poolname \ - 2>/dev/null || echo foo" "" || RC=1 - if [[ $RC -eq 0 ]]; then - add_pool_to_list $1 - else - error "pool_new failed $1" - fi - return $RC + local fsname=${1%%.*} + local poolname=${1##$fsname.} + + do_facet mgs lctl pool_new $1 + local RC=$? + # get param should return err unless pool is created + [[ $RC -ne 0 ]] && return $RC + + for mds_id in $(seq $MDSCOUNT); do + local mdt_id=$((mds_id-1)) + local lodname=$fsname-MDT$(printf "%04x" $mdt_id)-mdtlov + wait_update_facet mds$mds_id \ + "lctl get_param -n lod.$lodname.pools.$poolname \ + 2>/dev/null || echo foo" "" || + error "mds$mds_id: pool_new failed $1" + done + wait_update $HOSTNAME "lctl get_param -n lov.$fsname-*.pools.$poolname \ + 2>/dev/null || echo foo" "" || error "pool_new failed $1" + + add_pool_to_list $1 + return $RC } add_pool_to_list () { @@ -6272,28 +6277,31 @@ destroy_pool_int() { # . or destroy_pool() { - local fsname=${1%%.*} - local poolname=${1##$fsname.} - - [[ x$fsname = x$poolname ]] && fsname=$FSNAME + local fsname=${1%%.*} + local poolname=${1##$fsname.} - local RC + [[ x$fsname = x$poolname ]] && fsname=$FSNAME - pool_list $fsname.$poolname || return $? + local RC - destroy_pool_int $fsname.$poolname - RC=$? - [[ $RC -ne 0 ]] && return $RC + pool_list $fsname.$poolname || return $? - wait_update $HOSTNAME "lctl get_param -n lov.$fsname-*.pools.$poolname \ - 2>/dev/null || echo foo" "foo" || RC=1 + destroy_pool_int $fsname.$poolname + RC=$? + [[ $RC -ne 0 ]] && return $RC + for mds_id in $(seq $MDSCOUNT); do + local mdt_id=$((mds_id-1)) + local lodname=$fsname-MDT$(printf "%04x" $mdt_id)-mdtlov + wait_update_facet mds$mds_id \ + "lctl get_param -n lod.$lodname.pools.$poolname \ + 2>/dev/null || echo foo" "foo" || + error "mds$mds_id: destroy pool failed $1" + done + wait_update $HOSTNAME "lctl get_param -n lov.$fsname-*.pools.$poolname \ + 2>/dev/null || echo foo" "foo" || error "destroy pool failed $1" - if [[ $RC -eq 0 ]]; then - remove_pool_from_list $fsname.$poolname - else - error "destroy pool failed $1" - fi - return $RC + remove_pool_from_list $fsname.$poolname + return $RC } destroy_pools () { @@ -7291,6 +7299,18 @@ pool_add_targets() { local t=$(for i in $list; do printf "$FSNAME-OST%04x_UUID " $i; done) do_facet mgs $LCTL pool_add \ $FSNAME.$pool $FSNAME-OST[$first-$last/$step] + + # wait for OSTs to be added to the pool + for mds_id in $(seq $MDSCOUNT); do + local mdt_id=$((mds_id-1)) + local lodname=$FSNAME-MDT$(printf "%04x" $mdt_id)-mdtlov + wait_update_facet mds$mds_id \ + "lctl get_param -n lod.$lodname.pools.$pool | + sort -u | tr '\n' ' ' " "$t" || { + error_noexit "mds$mds_id: Add to pool failed" + return 3 + } + done wait_update $HOSTNAME "lctl get_param -n lov.$FSNAME-*.pools.$pool \ | sort -u | tr '\n' ' ' " "$t" || { error_noexit "Add to pool failed" @@ -7427,6 +7447,17 @@ pool_remove_first_target() { local pname="lov.$FSNAME-*.pools.$pool" local t=$($LCTL get_param -n $pname | head -1) do_facet mgs $LCTL pool_remove $FSNAME.$pool $t + for mds_id in $(seq $MDSCOUNT); do + local mdt_id=$((mds_id-1)) + local lodname=$FSNAME-MDT$(printf "%04x" $mdt_id)-mdtlov + wait_update_facet mds$mds_id \ + "lctl get_param -n lod.$lodname.pools.$pool | + grep $t" "" || { + error_noexit "mds$mds_id: $t not removed from" \ + "$FSNAME.$pool" + return 2 + } + done wait_update $HOSTNAME "lctl get_param -n $pname | grep $t" "" || { error_noexit "$t not removed from $FSNAME.$pool" return 1 @@ -7442,6 +7473,15 @@ pool_remove_all_targets() { do do_facet mgs $LCTL pool_remove $FSNAME.$pool $t done + for mds_id in $(seq $MDSCOUNT); do + local mdt_id=$((mds_id-1)) + local lodname=$FSNAME-MDT$(printf "%04x" $mdt_id)-mdtlov + wait_update_facet mds$mds_id "lctl get_param -n \ + lod.$lodname.pools.$pool" "" || { + error_noexit "mds$mds_id: Pool $pool not drained" + return 4 + } + done wait_update $HOSTNAME "lctl get_param -n $pname" "" || { error_noexit "Pool $FSNAME.$pool cannot be drained" return 1 -- 1.8.3.1