From fa2cfb49decf3d897f63023c998a23fd98c5c3ea Mon Sep 17 00:00:00 2001 From: Sebastien Buisson Date: Mon, 28 Aug 2023 10:09:53 +0200 Subject: [PATCH] LU-17056 tests: force osc import reconnect in sanity-sec 30b In sanity-sec test_30b, force reconnect of idle osc imports so that security flavor is correctly updated. In case of failure, dump more information about state of the imports and the srpc connections. Test-Parameters: trivial Test-Parameters: testlist=sanity-sec env=ONLY=30b,ONLY_REPEAT=50,SHARED_KEY=true mdscount=2 mdtcount=4 osscount=1 ostcount=8 clientcount=2 Test-Parameters: testlist=sanity-sec env=ONLY=30b,ONLY_REPEAT=50,SHARED_KEY=true mdscount=2 mdtcount=4 osscount=1 ostcount=8 clientcount=2 Test-Parameters: testlist=sanity-sec env=ONLY=30b,ONLY_REPEAT=50,SHARED_KEY=true mdscount=2 mdtcount=4 osscount=1 ostcount=8 clientcount=2 Test-Parameters: testlist=sanity-sec env=ONLY=30b,ONLY_REPEAT=50,SHARED_KEY=true mdscount=2 mdtcount=4 osscount=1 ostcount=8 clientcount=2 Test-Parameters: testlist=sanity-sec env=ONLY=30b,ONLY_REPEAT=50,SHARED_KEY=true mdscount=2 mdtcount=4 osscount=1 ostcount=8 clientcount=2 Test-Parameters: testlist=sanity-sec env=ONLY=30b,ONLY_REPEAT=50,SHARED_KEY=true mdscount=2 mdtcount=4 osscount=1 ostcount=8 clientcount=2 Signed-off-by: Sebastien Buisson Change-Id: Iaecc7321b12e61a266e97d3640a3288f0e7ec9dd Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/54349 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Andreas Dilger Reviewed-by: Jian Yu Reviewed-by: Oleg Drokin --- lustre/tests/sanity-sec.sh | 37 ++++++++++++--- lustre/tests/test-framework.sh | 105 ++++++++++++++++++++++++++++++++++++++++- 2 files changed, 135 insertions(+), 7 deletions(-) diff --git a/lustre/tests/sanity-sec.sh b/lustre/tests/sanity-sec.sh index 652e669..5384a8a 100755 --- a/lustre/tests/sanity-sec.sh +++ b/lustre/tests/sanity-sec.sh @@ -2423,18 +2423,26 @@ run_test 30 "check for invalid shared key" basic_ios() { local flvr=$1 - mkdir -p $DIR/$tdir || error "mkdir $flvr" - touch $DIR/$tdir/f0 || error "touch $flvr" - ls $DIR/$tdir || error "ls $flvr" - dd if=/dev/zero of=$DIR/$tdir/f0 conv=fsync bs=1M count=10 \ + mkdir -p $DIR/$tdir/dir0 || error "mkdir $flvr" + touch $DIR/$tdir/dir0/f0 || error "touch $flvr" + ls $DIR/$tdir/dir0 || error "ls $flvr" + dd if=/dev/zero of=$DIR/$tdir/dir0/f0 conv=fsync bs=1M count=10 \ >& /dev/null || error "dd $flvr" - rm -f $DIR/$tdir/f0 || error "rm $flvr" - rmdir $DIR/$tdir || error "rmdir $flvr" + rm -f $DIR/$tdir/dir0/f0 || error "rm $flvr" + rmdir $DIR/$tdir/dir0 || error "rmdir $flvr" sync ; sync echo 3 > /proc/sys/vm/drop_caches } +cleanup_30b() { + # restore clients' idle_timeout + for c in ${clients//,/ }; do + param=IDLETIME_$(echo $c | cut -d'.' -f1 | sed s+-+_+g) + do_node $c "lctl set_param osc.*.idle_timeout=${!param}" + done +} + test_30b() { local save_flvr=$SK_FLAVOR @@ -2442,8 +2450,25 @@ test_30b() { skip "need shared key feature for this test" fi + # save clients' idle_timeout, and set all to 0 for this test, + # as we do not want connections to go idle + for c in ${clients//,/ }; do + param=IDLETIME_$(echo $c | cut -d'.' -f1 | sed s+-+_+g) + idle=$(do_node $c lctl get_param -n osc.*.idle_timeout | + head -n1) + eval export $param=\$idle + do_node $c lctl set_param osc.*.idle_timeout=0 + done + + stack_trap cleanup_30b EXIT stack_trap restore_to_default_flavor EXIT + lfs mkdir -i 0 -c 1 $DIR/$tdir || error "mkdir $DIR/$tdir failed" + lfs setstripe -c -1 $DIR/$tdir/fileA || + error "setstripe $DIR/$tdir/fileA failed" + echo 30b > $DIR/$tdir/fileA || + error "wrtie to $DIR/$tdir/fileA failed" + for flvr in skn ska ski skpi; do # set flavor SK_FLAVOR=$flvr diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh index 660d20c..2f79852 100755 --- a/lustre/tests/test-framework.sh +++ b/lustre/tests/test-framework.sh @@ -9156,6 +9156,21 @@ flvr_cnt_cli2mdt() echo $cnt } +flvr_dump_cli2mdt() +{ + local clients=${CLIENTS:-$HOSTNAME} + + for c in ${clients//,/ }; do + do_node $c lctl get_param \ + mdc.*-*-mdc-*.$PROC_CLI 2>/dev/null + + if $GSS_SK; then + do_node $c lctl get_param \ + mdc.*-MDT*-mdc-*.$PROC_CON 2>/dev/null + fi + done +} + flvr_cnt_cli2ost() { local flavor=$1 @@ -9185,6 +9200,21 @@ flvr_cnt_cli2ost() echo $cnt } +flvr_dump_cli2ost() +{ + local clients=${CLIENTS:-$HOSTNAME} + + for c in ${clients//,/ }; do + do_node $c lctl get_param \ + osc.*OST*-osc-[^M][^D][^T]*.$PROC_CLI 2>/dev/null + + if $GSS_SK; then + do_node $c lctl get_param \ + osc.*OST*-osc-[^M][^D][^T]*.$PROC_CON 2>/dev/null + fi + done +} + flvr_cnt_mdt2mdt() { local flavor=$1 @@ -9215,6 +9245,19 @@ flvr_cnt_mdt2mdt() echo $cnt; } +flvr_dump_mdt2mdt() +{ + for num in `seq $MDSCOUNT`; do + do_facet mds$num lctl get_param \ + osp.*-MDT*osp-MDT*.$PROC_CLI 2>/dev/null + + if $GSS_SK; then + do_facet mds$num lctl get_param \ + osp.*-MDT*osp-MDT*.$PROC_CON 2>/dev/null + fi + done +} + flvr_cnt_mdt2ost() { local flavor=$1 @@ -9243,6 +9286,21 @@ flvr_cnt_mdt2ost() echo $cnt; } +flvr_dump_mdt2ost() +{ + for num in `seq $MDSCOUNT`; do + mdtosc=$(get_mdtosc_proc_path mds$num) + mdtosc=${mdtosc/-MDT*/-MDT\*} + do_facet mds$num lctl get_param \ + os[cp].$mdtosc.$PROC_CLI 2>/dev/null + + if $GSS_SK; then + do_facet mds$num lctl get_param \ + os[cp].$mdtosc.$PROC_CON 2>/dev/null + fi + done +} + flvr_cnt_mgc2mgs() { local flavor=$1 @@ -9285,6 +9343,51 @@ do_check_flavor() echo $res } +do_dump_imp_state() +{ + local clients=${CLIENTS:-$HOSTNAME} + local type=$1 + + for c in ${clients//,/ }; do + [ "$type" == "osc" ] && + do_node $c lctl get_param osc.*.idle_timeout + do_node $c lctl get_param $type.*.import | + grep -E "name:|state:" + done +} + +do_dump_flavor() +{ + local dir=$1 # from to + + if [ $dir == "cli2mdt" ]; then + do_dump_imp_state mdc + flvr_dump_cli2mdt + elif [ $dir == "cli2ost" ]; then + do_dump_imp_state osc + flvr_dump_cli2ost + elif [ $dir == "mdt2mdt" ]; then + flvr_dump_mdt2mdt + elif [ $dir == "mdt2ost" ]; then + flvr_dump_mdt2ost + elif [ $dir == "all2ost" ]; then + flvr_dump_mdt2ost + do_dump_imp_state osc + flvr_dump_cli2ost + elif [ $dir == "all2mdt" ]; then + flvr_dump_mdt2mdt + do_dump_imp_state mdc + flvr_dump_cli2mdt + elif [ $dir == "all2all" ]; then + flvr_dump_mdt2ost + do_dump_imp_state osc + flvr_dump_cli2ost + flvr_dump_mdt2mdt + do_dump_imp_state mdc + flvr_dump_cli2mdt + fi +} + wait_flavor() { local dir=$1 # from to @@ -9302,8 +9405,8 @@ wait_flavor() done echo "Error checking $flavor of $dir: expect $expect, actual $res" -# echo "Dumping additional logs for SK debug.." do_nodes $(comma_list $(all_server_nodes)) "keyctl show" + do_dump_flavor $dir if $dump; then gather_logs $(comma_list $(nodes_list)) fi -- 1.8.3.1