LU-1189 tests: run save_lustre_params() on facets

This patch fixes save_lustre_params() and restore_lustre_params()
to get and set Lustre parameters through do_facet() instead of
do_node(). This will fix the issue that restore_lustre_params() still
tries to restore the saved params on the original node which will
likely become inactive under failover test environment.

Test-Parameters: envdefinitions=SLOW=yes,ENABLE_QUOTA=yes \
clientcount=4 osscount=2 mdscount=2 austeroptions=-R \
failover=true useiscsi=true \
testlist=replay-vbr,replay-dual

Signed-off-by: Jian Yu <jian.yu@intel.com>
Change-Id: I5bfc4797727aaaa15fac1e34b6a7182ae8f26da8
Reviewed-on: http://review.whamcloud.com/5628
Tested-by: Hudson
Tested-by: Maloo <whamcloud.maloo@gmail.com>
Reviewed-by: Li Wei <wei.g.li@intel.com>
Reviewed-by: Andreas Dilger <andreas.dilger@intel.com>
Reviewed-by: Wei Liu <wei3.liu@intel.com>

diff --git a/lustre/tests/obdfilter-survey.sh b/lustre/tests/obdfilter-survey.sh
index d78f7b4..5058fa7 100644 (file)
--- a/lustre/tests/obdfilter-survey.sh
+++ b/lustre/tests/obdfilter-survey.sh
@@ -140,10 +140,12 @@ check_jbd_values_facets () {
 test_1b () {
        local param_file=$TMP/$tfile-params
 
-       do_nodesv $(comma_list $(osts_nodes)) lctl get_param obdfilter.${FSNAME}-*.sync_journal
-
-       save_lustre_params $(comma_list $(osts_nodes)) "obdfilter.${FSNAME}-*.sync_journal" >$param_file
-       do_nodesv $(comma_list $(osts_nodes)) lctl set_param obdfilter.${FSNAME}-*.sync_journal=0
+       do_nodesv $(comma_list $(osts_nodes)) \
+               $LCTL get_param obdfilter.${FSNAME}-*.sync_journal
+       save_lustre_params $(get_facets OST) \
+               "obdfilter.${FSNAME}-*.sync_journal" > $param_file
+       do_nodesv $(comma_list $(osts_nodes)) \
+               $LCTL set_param obdfilter.${FSNAME}-*.sync_journal=0
 
        local stime=$(date +%s)
        thrlo=4 nobjhi=1 thrhi=4 obdflter_survey_run disk
@@ -175,10 +177,12 @@ run_test 2a "Stripe F/S over the Network"
 test_2b () {
        local param_file=$TMP/$tfile-params
 
-       do_nodesv $(comma_list $(osts_nodes)) lctl get_param obdfilter.${FSNAME}-*.sync_journal
-
-       save_lustre_params $(comma_list $(osts_nodes)) "obdfilter.${FSNAME}-*.sync_journal" >$param_file
-       do_nodesv $(comma_list $(osts_nodes)) lctl set_param obdfilter.${FSNAME}-*.sync_journal=0
+       do_nodesv $(comma_list $(osts_nodes)) \
+               $LCTL get_param obdfilter.${FSNAME}-*.sync_journal
+       save_lustre_params $(get_facets OST) \
+               "obdfilter.${FSNAME}-*.sync_journal" > $param_file
+       do_nodesv $(comma_list $(osts_nodes)) \
+               $LCTL set_param obdfilter.${FSNAME}-*.sync_journal=0
 
        local stime=$(date +%s)
        thrlo=4 nobjhi=1 thrhi=4 obdflter_survey_run netdisk

diff --git a/lustre/tests/replay-dual.sh b/lustre/tests/replay-dual.sh
index 148766a..44a4c2f 100755 (executable)
--- a/lustre/tests/replay-dual.sh
+++ b/lustre/tests/replay-dual.sh
@@ -491,7 +491,7 @@ run_test 20 "recovery time is not increasing"
 test_21a() {
     local param_file=$TMP/$tfile-params
 
-    save_lustre_params $(facet_active_host $SINGLEMDS) "mdt.*.commit_on_sharing" > $param_file
+       save_lustre_params $SINGLEMDS "mdt.*.commit_on_sharing" > $param_file
     do_facet $SINGLEMDS lctl set_param mdt.*.commit_on_sharing=1
     touch  $MOUNT1/$tfile-1
     mv  $MOUNT2/$tfile-1 $MOUNT2/$tfile-2
@@ -558,7 +558,7 @@ test_21b() {
 
     local num=$(get_mds_dir $MOUNT1)
 
-    save_lustre_params $(facet_active_host mds$num) "mdt.*.commit_on_sharing" > $param_file
+       save_lustre_params mds$num "mdt.*.commit_on_sharing" > $param_file
 
     # COS enabled
     local COS=1

diff --git a/lustre/tests/replay-vbr.sh b/lustre/tests/replay-vbr.sh
index af1a196..2b1f031 100644 (file)
--- a/lustre/tests/replay-vbr.sh
+++ b/lustre/tests/replay-vbr.sh
@@ -94,7 +94,7 @@ get_version() {
 
 #save COS setting
 cos_param_file=$TMP/rvbr-cos-params
-save_lustre_params $(comma_list $(mdts_nodes)) "mdt.*.commit_on_sharing" > $cos_param_file
+save_lustre_params $(get_facets MDS) "mdt.*.commit_on_sharing" > $cos_param_file
 
 test_0a() {
         get_version $CLIENT1 $DIR/$tdir/1a || true

diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh
index f5d07f6..4728c75 100644 (file)
--- a/lustre/tests/sanity.sh
+++ b/lustre/tests/sanity.sh
@@ -10518,16 +10518,20 @@ run_test 215 "/proc/sys/lnet exists and has proper content - bugs 18102, 21079,
 
 test_216() { # bug 20317
        [ $PARALLEL == "yes" ] && skip "skip parallel run" && return
-        remote_ost_nodsh && skip "remote OST with nodsh" && return
-        local node
-        local p="$TMP/sanityN-$TESTNAME.parameters"
-        save_lustre_params $HOSTNAME "osc.*.contention_seconds" > $p
-        for node in $(osts_nodes); do
-                save_lustre_params $node "ldlm.namespaces.filter-*.max_nolock_bytes" >> $p
-                save_lustre_params $node "ldlm.namespaces.filter-*.contended_locks" >> $p
-                save_lustre_params $node "ldlm.namespaces.filter-*.contention_seconds" >> $p
-        done
-        clear_osc_stats
+       remote_ost_nodsh && skip "remote OST with nodsh" && return
+
+       local node
+       local facets=$(get_facets OST)
+       local p="$TMP/$TESTSUITE-$TESTNAME.parameters"
+
+       save_lustre_params client "osc.*.contention_seconds" > $p
+       save_lustre_params $facets \
+               "ldlm.namespaces.filter-*.max_nolock_bytes" >> $p
+       save_lustre_params $facets \
+               "ldlm.namespaces.filter-*.contended_locks" >> $p
+       save_lustre_params $facets \
+               "ldlm.namespaces.filter-*.contention_seconds" >> $p
+       clear_osc_stats
 
         # agressive lockless i/o settings
         for node in $(osts_nodes); do

diff --git a/lustre/tests/sanityn.sh b/lustre/tests/sanityn.sh
index 07a4fb8..5e87402 100644 (file)
--- a/lustre/tests/sanityn.sh
+++ b/lustre/tests/sanityn.sh
@@ -742,8 +742,8 @@ enable_lockless_truncate() {
 }
 
 test_32a() { # bug 11270
-        local p="$TMP/sanityN-$TESTNAME.parameters"
-        save_lustre_params $HOSTNAME osc.*.lockless_truncate > $p
+       local p="$TMP/$TESTSUITE-$TESTNAME.parameters"
+       save_lustre_params client "osc.*.lockless_truncate" > $p
         cancel_lru_locks osc
         enable_lockless_truncate 1
         rm -f $DIR1/$tfile
@@ -781,15 +781,19 @@ run_test 32a "lockless truncate"
 test_32b() { # bug 11270
         remote_ost_nodsh && skip "remote OST with nodsh" && return
 
-        local node
-        local p="$TMP/sanityN-$TESTNAME.parameters"
-        save_lustre_params $HOSTNAME "osc.*.contention_seconds" > $p
-        for node in $(osts_nodes); do
-                save_lustre_params $node "ldlm.namespaces.filter-*.max_nolock_bytes" >> $p
-                save_lustre_params $node "ldlm.namespaces.filter-*.contended_locks" >> $p
-                save_lustre_params $node "ldlm.namespaces.filter-*.contention_seconds" >> $p
-        done
-        clear_osc_stats
+       local node
+       local facets=$(get_facets OST)
+       local p="$TMP/$TESTSUITE-$TESTNAME.parameters"
+
+       save_lustre_params client "osc.*.contention_seconds" > $p
+       save_lustre_params $facets \
+               "ldlm.namespaces.filter-*.max_nolock_bytes" >> $p
+       save_lustre_params $facets \
+               "ldlm.namespaces.filter-*.contended_locks" >> $p
+       save_lustre_params $facets \
+               "ldlm.namespaces.filter-*.contention_seconds" >> $p
+       clear_osc_stats
+
         # agressive lockless i/o settings
         for node in $(osts_nodes); do
                 do_node $node 'lctl set_param -n ldlm.namespaces.filter-*.max_nolock_bytes 2000000; lctl set_param -n ldlm.namespaces.filter-*.contended_locks 0; lctl set_param -n ldlm.namespaces.filter-*.contention_seconds 60'
@@ -851,7 +855,8 @@ test_33a() {
     local param_file=$TMP/$tfile-params
     local fstype=$(facet_fstype $SINGLEMDS)
 
-    save_lustre_params $(comma_list $(mdts_nodes)) "mdt.*.commit_on_sharing" > $param_file
+       save_lustre_params $(get_facets MDS) \
+               "mdt.*.commit_on_sharing" > $param_file
 
     local COS
     local jbdold="N/A"
@@ -902,8 +907,9 @@ test_33b() {
        local nfiles=${TEST33_NFILES:-10000}
        local param_file=$TMP/$tfile-params
 
-       save_lustre_params $(comma_list $(mdts_nodes)) \
-                               "mdt.*.commit_on_sharing" > $param_file
+       save_lustre_params $(get_facets MDS) \
+               "mdt.*.commit_on_sharing" > $param_file
+
        local COS
        local jbdold
        local jbdnew

diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh
index 6a02df4..3912ed7 100644 (file)
--- a/lustre/tests/test-framework.sh
+++ b/lustre/tests/test-framework.sh
@@ -4880,21 +4880,33 @@ calc_osc_kbytes () {
         $LCTL get_param -n osc.*[oO][sS][cC][-_][0-9a-f]*.$1 | calc_sum
 }
 
-# save_lustre_params(node, parameter_mask)
-# generate a stream of formatted strings (<node> <param name>=<param value>)
+# save_lustre_params(comma separated facet list, parameter_mask)
+# generate a stream of formatted strings (<facet> <param name>=<param value>)
 save_lustre_params() {
-        local s
-        do_nodesv $1 "lctl get_param $2 | while read s; do echo \\\$s; done"
+       local facets=$1
+       local facet
+       local nodes
+       local node
+
+       for facet in ${facets//,/ }; do
+               node=$(facet_active_host $facet)
+               [[ *\ $node\ * = " $nodes " ]] && continue
+               nodes="$nodes $node"
+
+               do_node $node "$LCTL get_param $2 |
+                       while read s; do echo $facet \\\$s; done"
+       done
 }
 
 # restore lustre parameters from input stream, produces by save_lustre_params
 restore_lustre_params() {
-        local node
-        local name
-        local val
-        while IFS=" =" read node name val; do
-                do_node ${node//:/} "lctl set_param -n $name $val"
-        done
+       local facet
+       local name
+       local val
+
+       while IFS=" =" read facet name val; do
+               do_facet $facet "$LCTL set_param -n $name $val"
+       done
 }
 
 check_catastrophe() {