LU-17056 tests: force osc import reconnect in sanity-sec 30b

In sanity-sec test_30b, force reconnect of idle osc imports
so that security flavor is correctly updated.
In case of failure, dump more information about state of the imports
and the srpc connections.

Test-Parameters: trivial
Test-Parameters: testlist=sanity-sec env=ONLY=30b,ONLY_REPEAT=50,SHARED_KEY=true mdscount=2 mdtcount=4 osscount=1 ostcount=8 clientcount=2
Test-Parameters: testlist=sanity-sec env=ONLY=30b,ONLY_REPEAT=50,SHARED_KEY=true mdscount=2 mdtcount=4 osscount=1 ostcount=8 clientcount=2
Test-Parameters: testlist=sanity-sec env=ONLY=30b,ONLY_REPEAT=50,SHARED_KEY=true mdscount=2 mdtcount=4 osscount=1 ostcount=8 clientcount=2
Test-Parameters: testlist=sanity-sec env=ONLY=30b,ONLY_REPEAT=50,SHARED_KEY=true mdscount=2 mdtcount=4 osscount=1 ostcount=8 clientcount=2
Test-Parameters: testlist=sanity-sec env=ONLY=30b,ONLY_REPEAT=50,SHARED_KEY=true mdscount=2 mdtcount=4 osscount=1 ostcount=8 clientcount=2
Test-Parameters: testlist=sanity-sec env=ONLY=30b,ONLY_REPEAT=50,SHARED_KEY=true mdscount=2 mdtcount=4 osscount=1 ostcount=8 clientcount=2
Signed-off-by: Sebastien Buisson <sbuisson@ddn.com>
Change-Id: Iaecc7321b12e61a266e97d3640a3288f0e7ec9dd
Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/54349
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>
Reviewed-by: Jian Yu <yujian@whamcloud.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>

diff --git a/lustre/tests/sanity-sec.sh b/lustre/tests/sanity-sec.sh
index 652e669..5384a8a 100755 (executable)
--- a/lustre/tests/sanity-sec.sh
+++ b/lustre/tests/sanity-sec.sh
@@ -2423,18 +2423,26 @@ run_test 30 "check for invalid shared key"
 basic_ios() {
        local flvr=$1
 
-       mkdir -p $DIR/$tdir || error "mkdir $flvr"
-       touch $DIR/$tdir/f0 || error "touch $flvr"
-       ls $DIR/$tdir || error "ls $flvr"
-       dd if=/dev/zero of=$DIR/$tdir/f0 conv=fsync bs=1M count=10 \
+       mkdir -p $DIR/$tdir/dir0 || error "mkdir $flvr"
+       touch $DIR/$tdir/dir0/f0 || error "touch $flvr"
+       ls $DIR/$tdir/dir0 || error "ls $flvr"
+       dd if=/dev/zero of=$DIR/$tdir/dir0/f0 conv=fsync bs=1M count=10 \
                >& /dev/null || error "dd $flvr"
-       rm -f $DIR/$tdir/f0 || error "rm $flvr"
-       rmdir $DIR/$tdir || error "rmdir $flvr"
+       rm -f $DIR/$tdir/dir0/f0 || error "rm $flvr"
+       rmdir $DIR/$tdir/dir0 || error "rmdir $flvr"
 
        sync ; sync
        echo 3 > /proc/sys/vm/drop_caches
 }
 
+cleanup_30b() {
+       # restore clients' idle_timeout
+       for c in ${clients//,/ }; do
+               param=IDLETIME_$(echo $c | cut -d'.' -f1 | sed s+-+_+g)
+               do_node $c "lctl set_param osc.*.idle_timeout=${!param}"
+       done
+}
+
 test_30b() {
        local save_flvr=$SK_FLAVOR
 
@@ -2442,8 +2450,25 @@ test_30b() {
                skip "need shared key feature for this test"
        fi
 
+       # save clients' idle_timeout, and set all to 0 for this test,
+       # as we do not want connections to go idle
+       for c in ${clients//,/ }; do
+               param=IDLETIME_$(echo $c | cut -d'.' -f1 | sed s+-+_+g)
+               idle=$(do_node $c lctl get_param -n osc.*.idle_timeout |
+                       head -n1)
+               eval export $param=\$idle
+               do_node $c lctl set_param osc.*.idle_timeout=0
+       done
+
+       stack_trap cleanup_30b EXIT
        stack_trap restore_to_default_flavor EXIT
 
+       lfs mkdir -i 0 -c 1 $DIR/$tdir || error "mkdir $DIR/$tdir failed"
+       lfs setstripe -c -1 $DIR/$tdir/fileA ||
+               error "setstripe $DIR/$tdir/fileA failed"
+       echo 30b > $DIR/$tdir/fileA ||
+               error "wrtie to $DIR/$tdir/fileA failed"
+
        for flvr in skn ska ski skpi; do
                # set flavor
                SK_FLAVOR=$flvr

diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh
index 660d20c..2f79852 100755 (executable)
--- a/lustre/tests/test-framework.sh
+++ b/lustre/tests/test-framework.sh
@@ -9156,6 +9156,21 @@ flvr_cnt_cli2mdt()
        echo $cnt
 }
 
+flvr_dump_cli2mdt()
+{
+       local clients=${CLIENTS:-$HOSTNAME}
+
+       for c in ${clients//,/ }; do
+               do_node $c lctl get_param \
+                        mdc.*-*-mdc-*.$PROC_CLI 2>/dev/null
+
+               if $GSS_SK; then
+                       do_node $c lctl get_param \
+                                mdc.*-MDT*-mdc-*.$PROC_CON 2>/dev/null
+               fi
+       done
+}
+
 flvr_cnt_cli2ost()
 {
        local flavor=$1
@@ -9185,6 +9200,21 @@ flvr_cnt_cli2ost()
        echo $cnt
 }
 
+flvr_dump_cli2ost()
+{
+       local clients=${CLIENTS:-$HOSTNAME}
+
+       for c in ${clients//,/ }; do
+               do_node $c lctl get_param \
+                       osc.*OST*-osc-[^M][^D][^T]*.$PROC_CLI 2>/dev/null
+
+               if $GSS_SK; then
+                       do_node $c lctl get_param \
+                              osc.*OST*-osc-[^M][^D][^T]*.$PROC_CON 2>/dev/null
+               fi
+       done
+}
+
 flvr_cnt_mdt2mdt()
 {
        local flavor=$1
@@ -9215,6 +9245,19 @@ flvr_cnt_mdt2mdt()
        echo $cnt;
 }
 
+flvr_dump_mdt2mdt()
+{
+       for num in `seq $MDSCOUNT`; do
+               do_facet mds$num lctl get_param \
+                       osp.*-MDT*osp-MDT*.$PROC_CLI 2>/dev/null
+
+               if $GSS_SK; then
+                       do_facet mds$num lctl get_param \
+                               osp.*-MDT*osp-MDT*.$PROC_CON 2>/dev/null
+               fi
+       done
+}
+
 flvr_cnt_mdt2ost()
 {
        local flavor=$1
@@ -9243,6 +9286,21 @@ flvr_cnt_mdt2ost()
        echo $cnt;
 }
 
+flvr_dump_mdt2ost()
+{
+       for num in `seq $MDSCOUNT`; do
+               mdtosc=$(get_mdtosc_proc_path mds$num)
+               mdtosc=${mdtosc/-MDT*/-MDT\*}
+               do_facet mds$num lctl get_param \
+                               os[cp].$mdtosc.$PROC_CLI 2>/dev/null
+
+               if $GSS_SK; then
+                       do_facet mds$num lctl get_param \
+                               os[cp].$mdtosc.$PROC_CON 2>/dev/null
+               fi
+       done
+}
+
 flvr_cnt_mgc2mgs()
 {
        local flavor=$1
@@ -9285,6 +9343,51 @@ do_check_flavor()
        echo $res
 }
 
+do_dump_imp_state()
+{
+       local clients=${CLIENTS:-$HOSTNAME}
+       local type=$1
+
+       for c in ${clients//,/ }; do
+               [ "$type" == "osc" ] &&
+                       do_node $c lctl get_param osc.*.idle_timeout
+               do_node $c lctl get_param $type.*.import |
+                       grep -E "name:|state:"
+       done
+}
+
+do_dump_flavor()
+{
+       local dir=$1        # from to
+
+       if [ $dir == "cli2mdt" ]; then
+               do_dump_imp_state mdc
+               flvr_dump_cli2mdt
+       elif [ $dir == "cli2ost" ]; then
+               do_dump_imp_state osc
+               flvr_dump_cli2ost
+       elif [ $dir == "mdt2mdt" ]; then
+               flvr_dump_mdt2mdt
+       elif [ $dir == "mdt2ost" ]; then
+               flvr_dump_mdt2ost
+       elif [ $dir == "all2ost" ]; then
+               flvr_dump_mdt2ost
+               do_dump_imp_state osc
+               flvr_dump_cli2ost
+       elif [ $dir == "all2mdt" ]; then
+               flvr_dump_mdt2mdt
+               do_dump_imp_state mdc
+               flvr_dump_cli2mdt
+       elif [ $dir == "all2all" ]; then
+               flvr_dump_mdt2ost
+               do_dump_imp_state osc
+               flvr_dump_cli2ost
+               flvr_dump_mdt2mdt
+               do_dump_imp_state mdc
+               flvr_dump_cli2mdt
+       fi
+}
+
 wait_flavor()
 {
        local dir=$1        # from to
@@ -9302,8 +9405,8 @@ wait_flavor()
        done
 
        echo "Error checking $flavor of $dir: expect $expect, actual $res"
-#      echo "Dumping additional logs for SK debug.."
        do_nodes $(comma_list $(all_server_nodes)) "keyctl show"
+       do_dump_flavor $dir
        if $dump; then
                gather_logs $(comma_list $(nodes_list))
        fi