LU-3000 tests: several fixes of tests for DNE

1. sanity 27u, set PRECREATE_FAIL for every MDS.

2. sanity 133d, use mkdir, instead of test_mkdir to
avoid checking rename stats on remote MDTs.

3. conf-sanity 32, use do_rpc_nodes for reload module,
in case it needs to reload module on remote nodes, and
also umount mdt2 in 32c.

4. conf-sanity 42, add load_modules to avoid missing
module errors in some cases.

5. skip test_53 and test_225 when there are more than
one MDT.

6. redirect error message to /dev/null, to avoid
too much error message in the log, which might
explore the size of the log.

Test-parameters: mdscount=2 mdtcount=2 testlist=sanity,conf-sanity

Signed-off-by: wang di <di.wang@intel.com>
Change-Id: Ic09d97fb68b923b3a5862371a59c5ba655c278a7
Reviewed-on: http://review.whamcloud.com/5877
Reviewed-by: Li Wei <wei.g.li@intel.com>
Tested-by: Hudson
Tested-by: Maloo <whamcloud.maloo@gmail.com>
Reviewed-by: Jian Yu <jian.yu@intel.com>
Reviewed-by: Andreas Dilger <andreas.dilger@intel.com>

diff --git a/lustre/tests/conf-sanity.sh b/lustre/tests/conf-sanity.sh
index 32ddff1..1117718 100644 (file)
--- a/lustre/tests/conf-sanity.sh
+++ b/lustre/tests/conf-sanity.sh
@@ -1346,8 +1346,6 @@ t32_check() {
 }
 
 t32_test_cleanup() {
-       local node=$(facet_active_host $SINGLEMDS)
-       local r="do_node $node"
        local tmp=$TMP/t32
        local rc=$?
 
@@ -1358,13 +1356,14 @@ t32_test_cleanup() {
                $r umount -d $tmp/mnt/mdt || rc=$?
        fi
        if $shall_cleanup_mdt1; then
-               $r umount -d $tmp/mnt/mdt1 || rc=$?
+               $r2 umount -d $tmp/mnt/mdt1 || rc=$?
        fi
        if $shall_cleanup_ost; then
                $r umount -d $tmp/mnt/ost || rc=$?
        fi
-       $r rm -rf $tmp || rc=$?
-       rm -rf $tmp || rc=$?
+
+       $r rm -rf $tmp
+       rm -rf $tmp
        return $rc
 }
 
@@ -1391,7 +1390,7 @@ t32_reload_modules() {
                        all_removed=true
                do_rpc_nodes $node check_mem_leak || return 1
                if $all_removed; then
-                       load_modules
+                       do_rpc_nodes $node load_modules
                        return 0
                fi
                sleep 5
@@ -1409,8 +1408,7 @@ t32_wait_til_devices_gone() {
        echo wait for devices to go
        while ((i < 20)); do
                devices=$(do_rpc_nodes $node $LCTL device_list | wc -l)
-               echo $device
-               ((devices == 0)) && return 1
+               ((devices == 0)) && return 0
                sleep 5
                i=$((i + 1))
        done
@@ -1805,6 +1803,14 @@ t32_test() {
                }
                shall_cleanup_lustre=false
        else
+               if [ "$dne_upgrade" != "no" ]; then
+                       $r2 umount -d $tmp/mnt/mdt1 || {
+                               error_noexit "Unmounting the MDT2"
+                               return 1
+                       }
+                       shall_cleanup_mdt1=false
+               fi
+
                $r umount -d $tmp/mnt/mdt || {
                        error_noexit "Unmounting the MDT"
                        return 1
@@ -2386,6 +2392,7 @@ test_42() { #bug 14693
 
        do_facet mgs $LCTL conf_param $FSNAME.sys.some_wrong_param=20
        cleanup || error "stopping $FSNAME failed with invalid sys param"
+       load_modules
        setup
        check_mount || "client was not mounted with invalid sys param"
        cleanup || error "stopping $FSNAME failed with invalid sys param"

diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh
index f1f8bff..e9f2966 100644 (file)
--- a/lustre/tests/sanity.sh
+++ b/lustre/tests/sanity.sh
@@ -1499,22 +1499,24 @@ test_27t() { # bug 10864
 run_test 27t "check that utils parse path correctly"
 
 test_27u() { # bug 4900
-        [ "$OSTCOUNT" -lt "2" ] && skip_env "too few OSTs" && return
-        remote_mds_nodsh && skip "remote MDS with nodsh" && return
+       [ "$OSTCOUNT" -lt "2" ] && skip_env "too few OSTs" && return
+       remote_mds_nodsh && skip "remote MDS with nodsh" && return
+       local index
+       local list=$(comma_list $(mdts_nodes))
 
 #define OBD_FAIL_MDS_OSC_PRECREATE      0x139
-        do_facet $SINGLEMDS lctl set_param fail_loc=0x139
-        test_mkdir -p $DIR/$tdir
+       do_nodes $list $LCTL set_param fail_loc=0x139
+       test_mkdir -p $DIR/$tdir
        rm -rf $DIR/$tdir/*
-        createmany -o $DIR/$tdir/t- 1000
-        do_facet $SINGLEMDS lctl set_param fail_loc=0
+       createmany -o $DIR/$tdir/t- 1000
+       do_nodes $list $LCTL set_param fail_loc=0
 
-        TLOG=$DIR/$tfile.getstripe
-        $GETSTRIPE $DIR/$tdir > $TLOG
-        OBJS=`awk -vobj=0 '($1 == 0) { obj += 1 } END { print obj;}' $TLOG`
-        unlinkmany $DIR/$tdir/t- 1000
-        [ $OBJS -gt 0 ] && \
-                error "$OBJS objects created on OST-0.  See $TLOG" || pass
+       TLOG=$DIR/$tfile.getstripe
+       $GETSTRIPE $DIR/$tdir > $TLOG
+       OBJS=`awk -vobj=0 '($1 == 0) { obj += 1 } END { print obj;}' $TLOG`
+       unlinkmany $DIR/$tdir/t- 1000
+       [ $OBJS -gt 0 ] && \
+               error "$OBJS objects created on OST-0.  See $TLOG" || pass
 }
 run_test 27u "skip object creation on OSC w/o objects =========="
 
@@ -8287,20 +8289,20 @@ get_rename_size() {
 
 test_133d() {
        [ $PARALLEL == "yes" ] && skip "skip parallel run" && return
-    remote_ost_nodsh && skip "remote OST with nodsh" && return
-    remote_mds_nodsh && skip "remote MDS with nodsh" && return
-    do_facet $SINGLEMDS $LCTL list_param mdt.*.rename_stats ||
-        { skip "MDS doesn't support rename stats"; return; }
+       remote_ost_nodsh && skip "remote OST with nodsh" && return
+       remote_mds_nodsh && skip "remote MDS with nodsh" && return
+       do_facet $SINGLEMDS $LCTL list_param mdt.*.rename_stats ||
+       { skip "MDS doesn't support rename stats"; return; }
 
-    local testdir1=$DIR/${tdir}/stats_testdir1
-    local testdir2=$DIR/${tdir}/stats_testdir2
+       local testdir1=$DIR/${tdir}/stats_testdir1
+       local testdir2=$DIR/${tdir}/stats_testdir2
 
-    do_facet $SINGLEMDS $LCTL set_param mdt.*.rename_stats=clear
+       do_facet $SINGLEMDS $LCTL set_param mdt.*.rename_stats=clear
 
-    test_mkdir -p ${testdir1} || error "mkdir failed"
-    test_mkdir -p ${testdir2} || error "mkdir failed"
+       mkdir -p ${testdir1} || error "mkdir failed"
+       mkdir -p ${testdir2} || error "mkdir failed"
 
-    createmany -o $testdir1/test 512 || error "createmany failed"
+       createmany -o $testdir1/test 512 || error "createmany failed"
 
        # check samedir rename size
        mv ${testdir1}/test0 ${testdir1}/test_0
@@ -8319,18 +8321,18 @@ test_133d() {
        echo "source rename dir size: ${testdir1_size}"
        echo "target rename dir size: ${testdir2_size}"
 
-    local cmd="do_facet $SINGLEMDS $LCTL get_param mdt.*.rename_stats"
-    eval $cmd || error "$cmd failed"
-    local samedir=$($cmd | grep 'same_dir')
-    local same_sample=$(get_rename_size $testdir1_size)
-    [ -z "$samedir" ] && error "samedir_rename_size count error"
-    [ "$same_sample" -eq 1 ] || error "samedir_rename_size error $same_sample"
-    echo "Check same dir rename stats success"
+       local cmd="do_facet $SINGLEMDS $LCTL get_param mdt.*.rename_stats"
+       eval $cmd || error "$cmd failed"
+       local samedir=$($cmd | grep 'same_dir')
+       local same_sample=$(get_rename_size $testdir1_size)
+       [ -z "$samedir" ] && error "samedir_rename_size count error"
+       [ "$same_sample" -eq 1 ] || error "samedir_rename_size error $same_sample"
+       echo "Check same dir rename stats success"
 
-    do_facet $SINGLEMDS $LCTL set_param mdt.*.rename_stats=clear
+       do_facet $SINGLEMDS $LCTL set_param mdt.*.rename_stats=clear
 
-    # check crossdir rename size
-    mv ${testdir1}/test_0 ${testdir2}/test_0
+       # check crossdir rename size
+       mv ${testdir1}/test_0 ${testdir2}/test_0
 
        testdir1_size=$(ls -l $DIR/${tdir} |
                awk '/stats_testdir1/ {print $5}')
@@ -8346,15 +8348,15 @@ test_133d() {
        echo "source rename dir size: ${testdir1_size}"
        echo "target rename dir size: ${testdir2_size}"
 
-    eval $cmd || error "$cmd failed"
-    local crossdir=$($cmd | grep 'crossdir')
-    local src_sample=$(get_rename_size $testdir1_size crossdir_src)
-    local tgt_sample=$(get_rename_size $testdir2_size crossdir_tgt)
-    [ -z "$crossdir" ] && error "crossdir_rename_size count error"
-    [ "$src_sample" -eq 1 ] || error "crossdir_rename_size error $src_sample"
-    [ "$tgt_sample" -eq 1 ] || error "crossdir_rename_size error $tgt_sample"
-    echo "Check cross dir rename stats success"
-    rm -rf $DIR/${tdir}
+       eval $cmd || error "$cmd failed"
+       local crossdir=$($cmd | grep 'crossdir')
+       local src_sample=$(get_rename_size $testdir1_size crossdir_src)
+       local tgt_sample=$(get_rename_size $testdir2_size crossdir_tgt)
+       [ -z "$crossdir" ] && error "crossdir_rename_size count error"
+       [ "$src_sample" -eq 1 ] || error "crossdir_rename_size error $src_sample"
+       [ "$tgt_sample" -eq 1 ] || error "crossdir_rename_size error $tgt_sample"
+       echo "Check cross dir rename stats success"
+       rm -rf $DIR/${tdir}
 }
 run_test 133d "Verifying rename_stats ========================================"
 
@@ -10661,9 +10663,13 @@ run_test 224b "Don't panic on bulk IO failure"
 MDSSURVEY=${MDSSURVEY:-$(which mds-survey 2>/dev/null || true)}
 test_225a () {
        [ $PARALLEL == "yes" ] && skip "skip parallel run" && return
-       if [ -z ${MDSSURVEY} ]; then
-              skip_env "mds-survey not found" && return
-       fi
+       if [ -z ${MDSSURVEY} ]; then
+             skip_env "mds-survey not found" && return
+       fi
+
+       [ $MDSCOUNT -ge 2 ] &&
+               skip "skipping now for more than one MDT" && return
+
        [ $(lustre_version_code $SINGLEMDS) -ge $(version_code 2.2.51) ] ||
             { skip "Need MDS version at least 2.2.51"; return; }
 
@@ -10686,16 +10692,19 @@ run_test 225a "Metadata survey sanity with zero-stripe"
 
 test_225b () {
        [ $PARALLEL == "yes" ] && skip "skip parallel run" && return
-       if [ -z ${MDSSURVEY} ]; then
-              skip_env "mds-survey not found" && return
-       fi
-       [ $(lustre_version_code $SINGLEMDS) -ge $(version_code 2.2.51) ] ||
-            { skip "Need MDS version at least 2.2.51"; return; }
 
-       if [ $($LCTL dl | grep -c osc) -eq 0 ]; then
-              skip_env "Need to mount OST to test" && return
-       fi
+       if [ -z ${MDSSURVEY} ]; then
+             skip_env "mds-survey not found" && return
+       fi
+       [ $(lustre_version_code $SINGLEMDS) -ge $(version_code 2.2.51) ] ||
+           { skip "Need MDS version at least 2.2.51"; return; }
+
+       if [ $($LCTL dl | grep -c osc) -eq 0 ]; then
+             skip_env "Need to mount OST to test" && return
+       fi
 
+       [ $MDSCOUNT -ge 2 ] &&
+               skip "skipping now for more than one MDT" && return
        local mds=$(facet_host $SINGLEMDS)
        local target=$(do_nodes $mds 'lctl dl' | \
                       awk "{if (\$2 == \"UP\" && \$3 == \"mdt\") {print \$4}}")

diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh
index 09d6298..48281c5 100644 (file)
--- a/lustre/tests/test-framework.sh
+++ b/lustre/tests/test-framework.sh
@@ -5138,7 +5138,7 @@ wait_osc_import_state() {
        #until list_param can return valid value. And also if there
        #are mulitple osc entries we should list all of them before
        #go to wait.
-       local params=$($LCTL list_param $param || true)
+       local params=$($LCTL list_param $param 2>/dev/null || true)
        while [ -z "$params" ]; do
                if [ $i -ge $maxtime ]; then
                        echo "can't get $param by list_param in $maxtime secs"
@@ -5152,7 +5152,7 @@ wait_osc_import_state() {
                fi
                sleep 1
                i=$((i + 1))
-               params=$($LCTL list_param $param || true)
+               params=$($LCTL list_param $param 2>/dev/null || true)
        done
 
        if ! do_rpc_nodes "$(facet_host $facet)" \