LU-3665 tests: Cleanup echo client after obdfilter-survey

Some failures of obdfilter-survey do not cause an error in
obdfilter-survey.sh In some cases obdfilter-survey did not
cleanup echo clients it had created and that could hang
umount of osts.
Change test-framework.sh::cleanupall to remove echo clients before
trying to umount to prevent the echo clients from holding the OST or
MDS/MGS open forever.

Test-Parameters: trivial testlist=obdfilter-survey osscount=1 ostcount=2 mdscount=1 mdtcount=1
Test-Parameters: trivial testlist=obdfilter-survey mdtfilesystemtype=zfs ostfilesystemtype=zfs osscount=1 ostcount=2 mdscount=1 mdtcount=1
Signed-off-by: Nathaniel Clark <nathaniel.l.clark@intel.com>
Change-Id: I63ae59da84101c782aa9d5e7216cce3b3b1ff2fe
Reviewed-on: https://review.whamcloud.com/9350
Tested-by: Jenkins
Tested-by: Maloo <hpdd-maloo@intel.com>
Reviewed-by: Mike Pershin <mike.pershin@intel.com>
Reviewed-by: Andriy Skulysh <c17819@cray.com>
Reviewed-by: Oleg Drokin <oleg.drokin@intel.com>

diff --git a/lustre-iokit/obdfilter-survey/obdfilter-survey b/lustre-iokit/obdfilter-survey/obdfilter-survey
index c512ecf..382db0c 100755 (executable)
--- a/lustre-iokit/obdfilter-survey/obdfilter-survey
+++ b/lustre-iokit/obdfilter-survey/obdfilter-survey
@@ -1,5 +1,4 @@
 #!/bin/bash
-set -e
 ######################################################################
 # customize per survey
 
@@ -471,7 +470,7 @@ for ((i=0; i < $ndevs; i++)); do
        host=${host_names[$i]}
        devno=($(get_ec_devno $host "${client_names[$i]}" "${ost_names[$i]}"))
        if ((${#devno[@]} != 3)); then
-               exit 1
+               cleanup 2
        fi
        devnos[$i]=${devno[0]}
        client_names[$i]=${devno[1]}
@@ -529,7 +528,7 @@ for ((rsz = $rszlo; rsz <= $rszhi; rsz*=2)); do
                                rm $tmpf
                                if [ $first_obj = "ERROR" ]; then
                                        print_summary "created object #s on $client_name not contiguous"
-                                       exit 1
+                                       cleanup 3
                                fi
                                first_objs[$idx]=$first_obj
                        done # $ndevs
@@ -539,7 +538,7 @@ for ((rsz = $rszlo; rsz <= $rszhi; rsz*=2)); do
                                declare -a pidarray
                                for host in ${unique_hosts[@]}; do
                                        remote_shell $host \
-                                           "lctl set_param -n osd*.*OST*.force_sync 1 &>/dev/null || true"
+                                           "lctl set_param -n osd*.*.force_sync 1 &>/dev/null || true"
                                        echo "starting run for test: $test rsz: $rsz " \
                                        "threads: $thr objects: $nobj" >> ${vmstatf}_${host}
                                done

diff --git a/lustre/tests/obdfilter-survey.sh b/lustre/tests/obdfilter-survey.sh
index a2f3ccc..9a96e56 100644 (file)
--- a/lustre/tests/obdfilter-survey.sh
+++ b/lustre/tests/obdfilter-survey.sh
@@ -80,8 +80,10 @@ obdflter_survey_run () {
        local cmd="NETTYPE=$NETTYPE thrlo=$thrlo nobjhi=$nobjhi thrhi=$thrhi size=$size case=$case rslt_loc=${TMP} targets=\"$targets\" $OBDSURVEY"
        echo + $cmd
        eval $cmd
+       local rc=$?
 
        cat ${TMP}/obdfilter_survey*
+       [ $rc = 0 ] || error "$OBDSURVEY failed: $rc"
 }
 test_1a () {
        obdflter_survey_run disk

diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh
index 2c2d545..c88646d 100755 (executable)
--- a/lustre/tests/test-framework.sh
+++ b/lustre/tests/test-framework.sh
@@ -4150,8 +4150,8 @@ cleanupall() {
        nfs_client_mode && return
        cifs_client_mode && return
 
-       CLEANUP_DM_DEV=true stopall $*
        cleanup_echo_devs
+       CLEANUP_DM_DEV=true stopall $*
 
        unload_modules
        cleanup_sk