LU-8952 tests: handling test specific cleanup of ost pools

[fs/lustre-release.git] / lustre / tests / test-framework.sh

diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh
index 01779ba..39a0990 100755 (executable)
--- a/lustre/tests/test-framework.sh
+++ b/lustre/tests/test-framework.sh
@@ -530,10 +530,12 @@ load_modules_local() {
        fi
 
        load_module ../libcfs/libcfs/libcfs
+       # Prevent local MODOPTS_LIBCFS being passed as part of environment
+       # variable to remote nodes
+       unset MODOPTS_LIBCFS
 
-    [ "$PTLDEBUG" ] && lctl set_param debug="$PTLDEBUG"
-    [ "$SUBSYSTEM" ] && lctl set_param subsystem_debug="${SUBSYSTEM# }"
-    load_module ../lnet/lnet/lnet
+       set_default_debug
+       load_module ../lnet/lnet/lnet
        case $NETTYPE in
        o2ib)
                LNETLND="o2iblnd/ko2iblnd"
@@ -2165,6 +2167,15 @@ sync_all_data() {
                grep -v 'Found no match'
 }
 
+wait_zfs_commit() {
+       # the occupied disk space will be released
+       # only after DMUs are committed
+       if [[ $(facet_fstype $1) == zfs ]]; then
+               echo "sleep $2 for ZFS OSD"
+               sleep $2
+       fi
+}
+
 wait_delete_completed_mds() {
        local MAX_WAIT=${1:-20}
        # for ZFS, waiting more time for DMUs to be committed
@@ -2185,6 +2196,7 @@ wait_delete_completed_mds() {
                mds2sync="$mds2sync $node"
        done
        if [ -z "$mds2sync" ]; then
+               wait_zfs_commit $SINGLEMDS $ZFS_WAIT
                return
        fi
        mds2sync=$(comma_list $mds2sync)
@@ -2202,16 +2214,7 @@ wait_delete_completed_mds() {
                        "$LCTL get_param -n osc.*MDT*.sync_*" | calc_sum)
                #echo "$node: $changes changes on all"
                if [[ $changes -eq 0 ]]; then
-                       etime=$(date +%s)
-                       #echo "delete took $((etime - stime)) seconds"
-
-                       # the occupied disk space will be released
-                       # only after DMUs are committed
-                       if [[ $(facet_fstype $SINGLEMDS) == zfs ]]; then
-                               echo "sleep $ZFS_WAIT for ZFS OSD"
-                               sleep $ZFS_WAIT
-                       fi
-
+                       wait_zfs_commit $SINGLEMDS $ZFS_WAIT
                        return
                fi
                sleep 1
@@ -2333,6 +2336,19 @@ wait_mds_ost_sync () {
        return 1
 }
 
+# Wait OSTs to be active on both client and MDT side.
+wait_osts_up() {
+       local cmd="$LCTL get_param -n lov.$FSNAME-clilov-*.target_obd |
+               awk 'BEGIN {c = 0} /ACTIVE/{c += 1} END {printf \\\"%d\\\", c}'"
+       wait_update $HOSTNAME "eval $cmd" $OSTCOUNT ||
+               error "wait_update OSTs up on client failed"
+
+       cmd="$LCTL get_param -n lod.$FSNAME-MDT*-*.target_obd | sort -u |
+            awk 'BEGIN {c = 0} /ACTIVE/{c += 1} END {printf \\\"%d\\\", c}'"
+       wait_update_facet $SINGLEMDS "eval $cmd" $OSTCOUNT ||
+               error "wait_update OSTs up on MDT failed"
+}
+
 wait_destroy_complete () {
        echo "Waiting for local destroys to complete"
        # MAX value shouldn't be big as this mean server responsiveness
@@ -6004,9 +6020,8 @@ _wait_osc_import_state() {
 
        if [[ $facet == client* ]]; then
                # During setup time, the osc might not be setup, it need wait
-               # until list_param can return valid value. And also if there
-               # are mulitple osc entries we should list all of them before
-               # go to wait.
+               # until list_param can return valid value.
+               param="osc.${ost%?}[^mM]*.ost_server_uuid"
                params=$($LCTL list_param $param 2>/dev/null || true)
                while [ -z "$params" ]; do
                        if [ $i -ge $maxtime ]; then
@@ -6201,14 +6216,24 @@ oos_full() {
        return $OSCFULL
 }
 
-pool_list () {
-   do_facet mgs lctl pool_list $1
+list_pool() {
+       echo -e "$(do_facet $SINGLEMDS $LCTL pool_list $1 | sed '1d')"
+}
+
+check_pool_not_exist() {
+       local fsname=${1%%.*}
+       local poolname=${1##$fsname.}
+       [[ $# -ne 1 ]] && return 0
+       [[ x$poolname = x ]] &&  return 0
+       list_pool $fsname | grep -w $1 && return 1
+       return 0
 }
 
 create_pool() {
        local fsname=${1%%.*}
        local poolname=${1##$fsname.}
 
+       trap "destroy_test_pools $fsname" EXIT
        do_facet mgs lctl pool_new $1
        local RC=$?
        # get param should return err unless pool is created
@@ -6248,13 +6273,12 @@ remove_pool_from_list () {
 }
 
 destroy_pool_int() {
-    local ost
-    local OSTS=$(do_facet $SINGLEMDS lctl pool_list $1 | \
-        awk '$1 !~ /^Pool:/ {print $1}')
-    for ost in $OSTS; do
-        do_facet mgs lctl pool_remove $1 $ost
-    done
-    do_facet mgs lctl pool_destroy $1
+       local ost
+       local OSTS=$(list_pool $1)
+       for ost in $OSTS; do
+               do_facet mgs lctl pool_remove $1 $ost
+       done
+       do_facet mgs lctl pool_destroy $1
 }
 
 # <fsname>.<poolname> or <poolname>
@@ -6266,7 +6290,8 @@ destroy_pool() {
 
        local RC
 
-       pool_list $fsname.$poolname || return $?
+       check_pool_not_exist $fsname.$poolname
+       [[ $? -eq 0 ]] && return 0
 
        destroy_pool_int $fsname.$poolname
        RC=$?
@@ -6283,28 +6308,27 @@ destroy_pool() {
                2>/dev/null || echo foo" "foo" || error "destroy pool failed $1"
 
        remove_pool_from_list $fsname.$poolname
+
        return $RC
 }
 
 destroy_pools () {
-    local fsname=${1:-$FSNAME}
-    local poolname
-    local listvar=${fsname}_CREATED_POOLS
-
-    pool_list $fsname
+       local fsname=${1:-$FSNAME}
+       local poolname
+       local listvar=${fsname}_CREATED_POOLS
 
-    [ x${!listvar} = x ] && return 0
+       [ x${!listvar} = x ] && return 0
 
-    echo destroy the created pools: ${!listvar}
-    for poolname in ${!listvar//,/ }; do
-        destroy_pool $fsname.$poolname
-    done
+       echo "Destroy the created pools: ${!listvar}"
+       for poolname in ${!listvar//,/ }; do
+               destroy_pool $fsname.$poolname
+       done
 }
 
-cleanup_pools () {
-    local fsname=${1:-$FSNAME}
-    trap 0
-    destroy_pools $fsname
+destroy_test_pools () {
+       trap 0
+       local fsname=${1:-$FSNAME}
+       destroy_pools $fsname || true
 }
 
 gather_logs () {