X-Git-Url: https://git.whamcloud.com/?a=blobdiff_plain;f=lustre%2Ftests%2Fsanity-gss.sh;h=8b695b6eae38c3b9b6af55179fd11d4ccc794a41;hb=28806162c234a178365853a393aa401e180256e2;hp=56370ce968e0e2779ba5ee313a1b80521b337a47;hpb=60ea5e49e11678edaa11223a3d6aa3d502964e4c;p=fs%2Flustre-release.git diff --git a/lustre/tests/sanity-gss.sh b/lustre/tests/sanity-gss.sh index 56370ce..8b695b6 100644 --- a/lustre/tests/sanity-gss.sh +++ b/lustre/tests/sanity-gss.sh @@ -1,5 +1,4 @@ #!/bin/bash -# vim:expandtab:shiftwidth=4:softtabstop=4:tabstop=4: # # Run select tests by setting ONLY, or as arguments to the script. # Skip specific tests by setting EXCEPT. @@ -12,22 +11,13 @@ ONLY=${ONLY:-"$*"} ALWAYS_EXCEPT=${ALWAYS_EXCEPT:-"$SANITY_GSS_EXCEPT"} # UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT! -[ "$SLOW" = "no" ] && EXCEPT_SLOW="100 101" - -# Tests that fail on uml -CPU=`awk '/model/ {print $4}' /proc/cpuinfo` -[ "$CPU" = "UML" ] && EXCEPT="$EXCEPT" - -case `uname -r` in -2.6*) FSTYPE=${FSTYPE:-ldiskfs}; ALWAYS_EXCEPT="$ALWAYS_EXCEPT " ;; -*) error "unsupported kernel (gss only works with 2.6.x)" ;; -esac - SRCDIR=`dirname $0` export PATH=$PWD/$SRCDIR:$SRCDIR:$SRCDIR/../utils:$SRCDIR/../utils/gss:$PATH:/sbin export NAME=${NAME:-local} SAVE_PWD=$PWD +export MULTIOP=${MULTIOP:-multiop} + CLEANUP=${CLEANUP:-""} SETUP=${SETUP:-""} @@ -35,18 +25,14 @@ LUSTRE=${LUSTRE:-`dirname $0`/..} . $LUSTRE/tests/test-framework.sh init_test_env $@ . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh} +init_logging -if [ $UID -ne 0 ]; then - echo "Warning: running as non-root uid $UID" - RUNAS_ID="$UID" - RUNAS="" -else - RUNAS_ID=${RUNAS_ID:-500} - RUNAS=${RUNAS:-"runas -u $RUNAS_ID"} +require_dsh_mds || exit 0 - # $RUNAS_ID may get set incorrectly somewhere else - [ $RUNAS_ID -eq 0 ] && error "\$RUNAS_ID set to 0, but \$UID is also 0!" -fi +[ "$SLOW" = "no" ] && EXCEPT_SLOW="100 101" + +# $RUNAS_ID may get set incorrectly somewhere else +[ $UID -eq 0 -a $RUNAS_ID -eq 0 ] && error "\$RUNAS_ID set to 0, but \$UID is also 0!" # remove $SEC, we'd like to control everything by ourselves unset SEC @@ -57,259 +43,38 @@ unset SEC KRB5_CCACHE_DIR=/tmp KRB5_CRED=$KRB5_CCACHE_DIR/krb5cc_$RUNAS_ID KRB5_CRED_SAVE=$KRB5_CCACHE_DIR/krb5cc.sanity.save -CLICOUNT=2 -cnt_mdt2ost=0 -cnt_mdt2mdt=0 -cnt_cli2ost=0 -cnt_cli2mdt=0 -cnt_all2ost=0 -cnt_all2mdt=0 -cnt_all2all=0 DBENCH_PID=0 -PROC_CLI="srpc.info" # set manually GSS=true GSS_KRB5=true -# we want double mount -MOUNT_2=${MOUNT_2:-"yes"} -cleanup_and_setup_lustre - -rm -rf $DIR/${TESTSUITE}/[df][0-9]* -rm -rf $DIR/[df][0-9]* - -check_runas_id $RUNAS_ID $RUNAS - -build_test_filter - prepare_krb5_creds() { + echo prepare krb5 cred rm -f $KRB5_CRED_SAVE + echo RUNAS=$RUNAS $RUNAS krb5_login.sh || exit 1 [ -f $KRB5_CRED ] || exit 2 + echo CRED=$KRB5_CRED cp $KRB5_CRED $KRB5_CRED_SAVE } -combination() -{ - local M=$1 - local N=$2 - local R=1 - - if [ $M -lt $N ]; then - R=0 - else - N=$((N + 1)) - while [ $N -le $M ]; do - R=$((R * N)) - N=$((N + 1)) - done - fi - - echo $R - return 0 -} - -calc_connection_cnt() { - # MDT->MDT = 2 * C(M, 2) - # MDT->OST = M * O - # CLI->OST = C * O - # CLI->MDT = C * M - comb_m2=$(combination $MDSCOUNT 2) - - cnt_mdt2mdt=$((comb_m2 * 2)) - cnt_mdt2ost=$((MDSCOUNT * OSTCOUNT)) - cnt_cli2ost=$((CLICOUNT * OSTCOUNT)) - cnt_cli2mdt=$((CLICOUNT * MDSCOUNT)) - cnt_all2ost=$((cnt_mdt2ost + cnt_cli2ost)) - cnt_all2mdt=$((cnt_mdt2mdt + cnt_cli2mdt)) - cnt_all2all=$((cnt_mdt2ost + cnt_mdt2mdt + cnt_cli2ost + cnt_cli2mdt)) -} - -set_rule() -{ - local tgt=$1 - local net=$2 - local dir=$3 - local flavor=$4 - local cmd="$tgt.srpc.flavor" - - if [ $net == "any" ]; then - net="default" - fi - cmd="$cmd.$net" - - if [ $dir != "any" ]; then - cmd="$cmd.$dir" - fi - - cmd="$cmd=$flavor" - log "Setting sptlrpc rule: $cmd" - do_facet mgs "$LCTL conf_param $cmd" -} - -count_flvr() -{ - output=$1 - flavor=$2 - - echo "$output" | grep rpc | grep $flavor | wc -l -} - -flvr_cnt_cli2mdt() -{ - local flavor=$1 - - output=`do_facet client cat $LPROC/mdc/*-MDT*-mdc-*/$PROC_CLI 2>/dev/null` - count_flvr "$output" $flavor -} - -flvr_cnt_cli2ost() -{ - local flavor=$1 - - output=`do_facet client cat $LPROC/osc/*OST*-osc-[^M][^D][^T]*/$PROC_CLI 2>/dev/null` - count_flvr "$output" $flavor -} - -flvr_cnt_mdt2mdt() -{ - local flavor=$1 - local cnt=0 - - if [ $MDSCOUNT -le 1 ]; then - echo 0 - return - fi - - for num in `seq $MDSCOUNT`; do - output=`do_facet mds$num cat $LPROC/mdc/*-MDT*-mdc[0-9]*/$PROC_CLI 2>/dev/null` - tmpcnt=`count_flvr "$output" $flavor` - cnt=$((cnt + tmpcnt)) - done - echo $cnt; -} - -flvr_cnt_mdt2ost() -{ - local flavor=$1 - local cnt=0 - - for num in `seq $MDSCOUNT`; do - output=`do_facet mds$num cat $LPROC/osc/*OST*-osc-MDT*/$PROC_CLI 2>/dev/null` - tmpcnt=`count_flvr "$output" $flavor` - cnt=$((cnt + tmpcnt)) - done - echo $cnt; -} - -do_check_flavor() -{ - local dir=$1 # from to - local flavor=$2 # flavor expected - local res=0 - - if [ $dir == "cli2mdt" ]; then - res=`flvr_cnt_cli2mdt $flavor` - elif [ $dir == "cli2ost" ]; then - res=`flvr_cnt_cli2ost $flavor` - elif [ $dir == "mdt2mdt" ]; then - res=`flvr_cnt_mdt2mdt $flavor` - elif [ $dir == "mdt2ost" ]; then - res=`flvr_cnt_mdt2ost $flavor` - elif [ $dir == "all2ost" ]; then - res1=`flvr_cnt_mdt2ost $flavor` - res2=`flvr_cnt_cli2ost $flavor` - res=$((res1 + res2)) - elif [ $dir == "all2mdt" ]; then - res1=`flvr_cnt_mdt2mdt $flavor` - res2=`flvr_cnt_cli2mdt $flavor` - res=$((res1 + res2)) - elif [ $dir == "all2all" ]; then - res1=`flvr_cnt_mdt2ost $flavor` - res2=`flvr_cnt_cli2ost $flavor` - res3=`flvr_cnt_mdt2mdt $flavor` - res4=`flvr_cnt_cli2mdt $flavor` - res=$((res1 + res2 + res3 + res4)) - fi - - echo $res -} - -wait_flavor() -{ - local dir=$1 # from to - local flavor=$2 # flavor expected - local expect=$3 # number expected - local res=0 - - for ((i=0;i<20;i++)); do - echo -n "checking..." - res=$(do_check_flavor $dir $flavor) - if [ $res -eq $expect ]; then - echo "found $res $flavor connections of $dir, OK" - return 0 - else - echo "found $res $flavor connections of $dir, not ready ($expect)" - sleep 4 - fi - done - - echo "Error checking $flavor of $dir: expect $expect, actual $res" - return 1 -} - -restore_to_default_flavor() -{ - local proc=$LPROC/mgs/MGS/live/$FSNAME - - echo "restoring to default flavor..." - - nrule=`do_facet mgs cat $proc 2>/dev/null | grep ".srpc.flavor." | wc -l` - - # remove all existing rules if any - if [ $nrule -ne 0 ]; then - echo "$nrule existing rules" - for rule in `do_facet mgs cat $proc 2>/dev/null | grep ".srpc.flavor."`; do - echo "remove rule: $rule" - spec=`echo $rule | awk -F = '{print $1}'` - do_facet mgs "$LCTL conf_param $spec=" - done - fi - - # verify no rules left - nrule=`do_facet mgs cat $proc 2>/dev/null | grep ".srpc.flavor." | wc -l` - [ $nrule -ne 0 ] && error "still $nrule rules left" - - # wait for default flavor to be applied - # currently default flavor for all connections are 'null' - wait_flavor all2all null $cnt_all2all - echo "now at default flavor settings" -} - -set_flavor_all() -{ - local flavor=$1 +prepare_krb5_creds - echo "setting all flavor to $flavor" +# we want double mount +MOUNT_2=${MOUNT_2:-"yes"} +check_and_setup_lustre - res=$(do_check_flavor all2all $flavor) - if [ $res -eq $cnt_all2all ]; then - echo "already have total $res $flavor connections" - return - fi +rm -rf $DIR/[df][0-9]* - echo "found $res $flavor out of total $cnt_all2all connections" - restore_to_default_flavor +check_runas_id $RUNAS_ID $RUNAS_ID $RUNAS - set_rule $FSNAME any any $flavor - wait_flavor all2all $flavor $cnt_all2all -} +build_test_filter start_dbench() { NPROC=`cat /proc/cpuinfo 2>/dev/null | grep ^processor | wc -l` - [ $NPROC -lt 2 ] && NPROC=2 + [ $NPROC -gt 2 ] && NPROC=2 sh rundbench $NPROC 1>/dev/null & DBENCH_PID=$! sleep 2 @@ -382,7 +147,6 @@ check_multiple_gss_daemons() { fi } -prepare_krb5_creds calc_connection_cnt umask 077 @@ -424,8 +188,10 @@ test_1() { local file=$DIR/$tfile chmod 0777 $DIR || error "chmod $DIR failed" + $RUNAS touch $DIR # access w/o cred $RUNAS kdestroy + $RUNAS $LFS flushctx $MOUNT || error "can't flush context on $MOUNT" $RUNAS touch $file && error "unexpected success" # access w/ cred @@ -446,7 +212,7 @@ test_2() { # cleanup all cred/ctx and touch $RUNAS kdestroy - $RUNAS $LFS flushctx || error "can't flush ctx" + $RUNAS $LFS flushctx $MOUNT || error "can't flush context on $MOUNT" $RUNAS touch $file2 && error "unexpected success" # restore and touch @@ -467,7 +233,7 @@ test_3() { $RUNAS cat $file > /dev/null || error "$RUNAS_ID cat error" # start multiop - $RUNAS multiop $file o_r & + $RUNAS $MULTIOP $file o_r & OPPID=$! # wait multiop finish its open() sleep 1 @@ -476,7 +242,7 @@ test_3() { # metadata check should fail, but file data check should success # because we always use root credential to OSTs $RUNAS kdestroy - $RUNAS $LFS flushctx + $RUNAS $LFS flushctx $MOUNT || error "can't flush context on $MOUNT" echo "destroied credentials/contexs for $RUNAS_ID" $RUNAS $CHECKSTAT -p 0666 $file && error "checkstat succeed" kill -s 10 $OPPID @@ -512,7 +278,7 @@ test_4() { check_gss_daemon_facet client lgssd && error "lgssd still running" # flush context, and touch - $RUNAS $LFS flushctx + $RUNAS $LFS flushctx $MOUNT || error "can't flush context on $MOUNT" $RUNAS touch $file2 & TOUCHPID=$! echo "waiting touch pid $TOUCHPID" @@ -540,12 +306,12 @@ test_5() { [ -f $file1 ] || error "$file1 not found" # stop lsvcgssd - send_sigint mds lsvcgssd + send_sigint $(comma_list $(mdts_nodes)) lsvcgssd sleep 5 - check_gss_daemon_facet mds lsvcgssd && error "lsvcgssd still running" + check_gss_daemon_nodes $(comma_list $(mdts_nodes)) lsvcgssd && error "lsvcgssd still running" # flush context, and touch - $RUNAS $LFS flushctx + $RUNAS $LFS flushctx $MOUNT || error "can't flush context on $MOUNT" $RUNAS touch $file2 & TOUCHPID=$! @@ -558,54 +324,108 @@ test_5() { # restart lsvcgssd, expect touch suceed echo "restart lsvcgssd and recovering" - do_facet mds "$LSVCGSSD -v" + start_gss_daemons $(comma_list $(mdts_nodes)) "$LSVCGSSD -v" sleep 5 - check_gss_daemon_facet mds lsvcgssd + check_gss_daemon_nodes $(comma_list $(mdts_nodes)) lsvcgssd wait $TOUCHPID || error "touch fail" [ -f $file2 ] || error "$file2 not found" } run_test 5 "lsvcgssd dead, operations lead to recovery" test_6() { + local nfile=10 + mkdir $DIR/d6 || error "mkdir $DIR/d6 failed" - cp -a /etc/* $DIR/d6/ || error "cp failed" + for ((i=0; i<$nfile; i++)); do + dd if=/dev/zero of=$DIR/d6/file$i bs=8k count=1 || error "dd file$i failed" + done ls -l $DIR/d6/* > /dev/null || error "ls failed" rm -rf $DIR2/d6/* || error "rm failed" + rmdir $DIR2/d6/ || error "rmdir failed" } run_test 6 "test basic DLM callback works" test_7() { - local tdir=$DIR/d7 - local num_osts + local tdir=$DIR/d7 + local num_osts + + # for open(), client only reserve space for default stripe count lovea, + # and server may return larger lovea in reply (because of larger stripe + # count), client need call enlarge_reqbuf() and save the replied lovea + # in request for future possible replay. + # + # Note: current script does NOT guarantee enlarge_reqbuf() will be in + # the path, however it does work in local test which has 2 OSTs and + # default stripe count is 1. + num_osts=$($LFS getstripe $MOUNT | egrep "^[0-9]*:.*ACTIVE" | wc -l) + echo "found $num_osts active OSTs" + [ $num_osts -lt 2 ] && + echo "skipping $TESTNAME (must have >= 2 OSTs)" && return + + mkdir $tdir || error "mkdir $tdir failed" + $LFS setstripe -c $num_osts $tdir || error "setstripe -c $num_osts" + + echo "creating..." + for ((i = 0; i < 20; i++)); do + dd if=/dev/zero of=$tdir/f$i bs=4k count=16 2>/dev/null + done + echo "reading..." + for ((i = 0; i < 20; i++)); do + dd if=$tdir/f$i of=/dev/null bs=4k count=16 2>/dev/null + done + rm -rf $tdir +} +run_test 7 "exercise enlarge_reqbuf()" - # - # for open(), client only reserve space for default stripe count lovea, - # and server may return larger lovea in reply (because of larger stripe - # count), client need call enlarge_reqbuf() and save the replied lovea - # in request for future possible replay. - # - # Note: current script does NOT guarantee enlarge_reqbuf() will be in - # the path, however it does work in local test which has 2 OSTs and - # default stripe count is 1. - # - num_osts=`$LFS getstripe $MOUNT | egrep "^[0-9]*:.*ACTIVE" | wc -l` - echo "found $num_osts active OSTs" - [ $num_osts -lt 2 ] && echo "skipping $TESTNAME (must have >= 2 OSTs)" && return +test_8() +{ + local ATHISTORY=$(do_facet $SINGLEMDS "find /sys/ -name at_history") + local ATOLDBASE=$(do_facet $SINGLEMDS "cat $ATHISTORY") + local REQ_DELAY + do_facet $SINGLEMDS "echo 8 >> $ATHISTORY" + + mkdir -p $DIR/d8 + chmod a+w $DIR/d8 + + $LCTL dk > /dev/null + debugsave + sysctl -w lnet.debug="+other" + + # wait for the at estimation come down, this is faster + while [ true ]; do + REQ_DELAY=`lctl get_param -n mdc.${FSNAME}-MDT0000-mdc-*.timeouts | + awk '/portal 12/ {print $5}' | tail -1` + [ $REQ_DELAY -le 5 ] && break + echo "current AT estimation is $REQ_DELAY, wait a little bit" + sleep 8 + done + REQ_DELAY=$((${REQ_DELAY} + ${REQ_DELAY} / 4 + 5)) - mkdir $tdir || error - $LFS setstripe $tdir 0 -1 -1 || error + # sleep sometime in ctx handle + do_facet $SINGLEMDS lctl set_param fail_val=$REQ_DELAY +#define OBD_FAIL_SEC_CTX_HDL_PAUSE 0x1204 + do_facet $SINGLEMDS lctl set_param fail_loc=0x1204 - echo "creating..." - for ((i=0;i<20;i++)); do - dd if=/dev/zero of=$tdir/f$i bs=4k count=16 2>/dev/null - done - echo "reading..." - for ((i=0;i<20;i++)); do - dd if=$tdir/f$i of=/dev/null bs=4k count=16 2>/dev/null - done - rm -rf $tdir + $RUNAS $LFS flushctx $MOUNT || error "can't flush context on $MOUNT" + + $RUNAS touch $DIR/d8/f & + TOUCHPID=$! + echo "waiting for touch (pid $TOUCHPID) to finish..." + sleep 2 # give it a chance to really trigger context init rpc + do_facet $SINGLEMDS $LCTL set_param fail_loc=0 + wait $TOUCHPID || error "touch should have succeeded" + + $LCTL dk | grep "Early reply #" || error "No early reply" + + debugrestore + do_facet $SINGLEMDS "echo $ATOLDBASE >> $ATHISTORY" || true } -run_test 7 "exercise enlarge_reqbuf()" +run_test 8 "Early reply sent for slow gss context negotiation" + +# +# following tests will manipulate flavors and may end with any flavor set, +# so each test should not assume any start flavor. +# test_90() { if [ "$SLOW" = "no" ]; then @@ -614,15 +434,21 @@ test_90() { total=60 fi + restore_to_default_flavor + set_rule $FSNAME any any krb5p + wait_flavor all2all krb5p + start_dbench for ((n=0;n<$total;n++)); do sleep 2 check_dbench echo "flush ctx ($n/$total) ..." - $LFS flushctx + $LFS flushctx $MOUNT || error "can't flush context on $MOUNT" done check_dbench + #sleep to let ctxs be re-established + sleep 10 stop_dbench } run_test 90 "recoverable from losing contexts under load" @@ -630,24 +456,23 @@ run_test 90 "recoverable from losing contexts under load" test_99() { local nrule_old=0 local nrule_new=0 - local max=32 + local max=64 # # general rules # - nrule_old=`do_facet mgs cat $LPROC/mgs/MGS/live/$FSNAME 2>/dev/null \ + nrule_old=`do_facet mgs lctl get_param -n mgs.MGS.live.$FSNAME 2>/dev/null \ | grep "$FSNAME.srpc.flavor." | wc -l` echo "original general rules: $nrule_old" for ((i = $nrule_old; i < $max; i++)); do set_rule $FSNAME elan$i any krb5n || error "set rule $i" done - set_rule $FSNAME elan100 any krb5n && error "set $max rule should fail" for ((i = $nrule_old; i < $max; i++)); do set_rule $FSNAME elan$i any || error "remove rule $i" done - nrule_new=`do_facet mgs cat $LPROC/mgs/MGS/live/$FSNAME 2>/dev/null \ + nrule_new=`do_facet mgs lctl get_param -n mgs.MGS.live.$FSNAME 2>/dev/null \ | grep "$FSNAME.srpc.flavor." | wc -l` if [ $nrule_new != $nrule_old ]; then error "general rule: $nrule_new != $nrule_old" @@ -656,25 +481,24 @@ test_99() { # # target-specific rules # - nrule_old=`do_facet mgs cat $LPROC/mgs/MGS/live/$FSNAME 2>/dev/null \ + nrule_old=`do_facet mgs lctl get_param -n mgs.MGS.live.$FSNAME 2>/dev/null \ | grep "$FSNAME-MDT0000.srpc.flavor." | wc -l` echo "original target rules: $nrule_old" for ((i = $nrule_old; i < $max; i++)); do set_rule $FSNAME-MDT0000 elan$i any krb5i || error "set rule $i" done - set_rule $FSNAME-MDT0000 elan100 any krb5i && error "set $max rule should fail" for ((i = $nrule_old; i < $max; i++)); do set_rule $FSNAME-MDT0000 elan$i any || error "remove rule $i" done - nrule_new=`do_facet mgs cat $LPROC/mgs/MGS/live/$FSNAME 2>/dev/null \ + nrule_new=`do_facet mgs lctl get_param -n mgs.MGS.live.$FSNAME 2>/dev/null \ | grep "$FSNAME-MDT0000.srpc.flavor." | wc -l` if [ $nrule_new != $nrule_old ]; then error "general rule: $nrule_new != $nrule_old" fi } -run_test 99 "maximum sptlrpc rules limitation" +run_test 99 "set large number of sptlrpc rules" error_dbench() { @@ -697,23 +521,23 @@ test_100() { # all: null -> krb5n -> krb5a -> krb5i -> krb5p -> plain # set_rule $FSNAME any any krb5n - wait_flavor all2all krb5n $cnt_all2all || error_dbench "1" + wait_flavor all2all krb5n || error_dbench "1" check_dbench set_rule $FSNAME any any krb5a - wait_flavor all2all krb5a $cnt_all2all || error_dbench "2" + wait_flavor all2all krb5a || error_dbench "2" check_dbench set_rule $FSNAME any any krb5i - wait_flavor all2all krb5i $cnt_all2all || error_dbench "3" + wait_flavor all2all krb5i || error_dbench "3" check_dbench set_rule $FSNAME any any krb5p - wait_flavor all2all krb5p $cnt_all2all || error_dbench "4" + wait_flavor all2all krb5p || error_dbench "4" check_dbench set_rule $FSNAME any any plain - wait_flavor all2all plain $cnt_all2all || error_dbench "5" + wait_flavor all2all plain || error_dbench "5" check_dbench # @@ -723,19 +547,19 @@ test_100() { # C - O: krb5n # set_rule $FSNAME any mdt2mdt krb5a - wait_flavor mdt2mdt krb5a $cnt_mdt2mdt || error_dbench "6" + wait_flavor mdt2mdt krb5a || error_dbench "6" check_dbench set_rule $FSNAME any cli2mdt krb5i - wait_flavor cli2mdt krb5i $cnt_cli2mdt || error_dbench "7" + wait_flavor cli2mdt krb5i || error_dbench "7" check_dbench set_rule $FSNAME any mdt2ost krb5p - wait_flavor mdt2ost krb5p $cnt_mdt2ost || error_dbench "8" + wait_flavor mdt2ost krb5p || error_dbench "8" check_dbench set_rule $FSNAME any cli2ost krb5n - wait_flavor cli2ost krb5n $cnt_cli2ost || error_dbench "9" + wait_flavor cli2ost krb5n || error_dbench "9" check_dbench # @@ -746,11 +570,11 @@ test_100() { # set_rule $FSNAME-MDT0000 any any krb5p set_rule $FSNAME-OST0000 any any krb5i - wait_flavor mdt2mdt krb5a $cnt_mdt2mdt || error_dbench "10" - wait_flavor cli2mdt krb5i $cnt_cli2mdt || error_dbench "11" + wait_flavor mdt2mdt krb5a || error_dbench "10" + wait_flavor cli2mdt krb5i || error_dbench "11" check_dbench - wait_flavor mdt2ost krb5p $cnt_mdt2ost || error_dbench "12" - wait_flavor cli2ost krb5n $cnt_cli2ost || error_dbench "13" + wait_flavor mdt2ost krb5p || error_dbench "12" + wait_flavor cli2ost krb5n || error_dbench "13" # # delete all dir-specific rules @@ -760,10 +584,10 @@ test_100() { set_rule $FSNAME any mdt2ost set_rule $FSNAME any cli2ost wait_flavor mdt2mdt krb5p $((MDSCOUNT - 1)) || error_dbench "14" - wait_flavor cli2mdt krb5p $CLICOUNT || error_dbench "15" + wait_flavor cli2mdt krb5p $(get_clients_mount_count) || error_dbench "15" check_dbench wait_flavor mdt2ost krb5i $MDSCOUNT || error_dbench "16" - wait_flavor cli2ost krb5i $CLICOUNT || error_dbench "17" + wait_flavor cli2ost krb5i $(get_clients_mount_count) || error_dbench "17" check_dbench # @@ -773,7 +597,7 @@ test_100() { # set_rule $FSNAME-MDT0000 any any set_rule $FSNAME-OST0000 any any || error_dbench "18" - wait_flavor all2all plain $cnt_all2all || error_dbench "19" + wait_flavor all2all plain || error_dbench "19" check_dbench stop_dbench @@ -782,51 +606,44 @@ run_test 100 "change security flavor on the fly under load" switch_sec_test() { - local count=$1 - local flavor0=$2 - local flavor1=$3 - local flavor2=$4 - local df_pid=0 - local wait_time=$((TIMEOUT + TIMEOUT / 4)) + local flavor0=$1 + local flavor1=$2 + local filename=$DIR/$tfile + local multiop_pid local num # - # stop gss daemon, then switch to flavor1 (which should be a gss flavor), - # and run a 'df' which should hanging, wait the request timeout and - # resend, then switch the flavor to another one. To exercise the code of - # switching ctx/sec for a resend request. + # after set to flavor0, start multop which use flavor0 rpc, and let + # server drop the reply; then switch to flavor1, the resend should be + # completed using flavor1. To exercise the code of switching ctx/sec + # for a resend request. # - echo ">>>>>>>>>>>>>>> Testing $flavor0 -> $flavor1 -> $flavor2..." + log ">>>>>>>>>>>>>>> Testing $flavor0 -> $flavor1 <<<<<<<<<<<<<<<<<<<" - echo "(0) set base flavor $flavor0" set_rule $FSNAME any cli2mdt $flavor0 - wait_flavor cli2mdt $flavor0 $count - df $MOUNT - if [ $? -ne 0 ]; then - error "initial df failed" - fi - - stop_gss_daemons + wait_flavor cli2mdt $flavor0 + rm -f $filename || error "remove old $filename failed" + +#MDS_REINT = 36 +#define OBD_FAIL_PTLRPC_DROP_REQ_OPC 0x513 + do_facet $SINGLEMDS lctl set_param fail_val=36 + do_facet $SINGLEMDS lctl set_param fail_loc=0x513 + log "starting multiop" + $MULTIOP $filename m & + multiop_pid=$! + echo "multiop pid=$multiop_pid" sleep 1 - echo "(1) $flavor0 -> $flavor1" set_rule $FSNAME any cli2mdt $flavor1 - wait_flavor cli2mdt $flavor1 $count - df $MOUNT & - df_pid=$! - sleep 1 + wait_flavor cli2mdt $flavor1 - echo "waiting $wait_time seconds for df ($df_pid)" - sleep $wait_time - num=`ps --no-headers -p $df_pid 2>/dev/null | wc -l` - [ $num -eq 1 ] || error "df already ended ($num)" - echo "process $df_pid is still hanging there... OK" + num=`ps --no-headers -p $multiop_pid 2>/dev/null | wc -l` + [ $num -eq 1 ] || error "multiop($multiop_pid) already ended ($num)" + echo "process $multiop_pid is still hanging there... OK" - echo "(2) set end flavor $flavor2" - set_rule $FSNAME any cli2mdt $flavor2 - wait_flavor cli2mdt $flavor2 $count - start_gss_daemons - wait $df_pid || error "df returned error" + do_facet $SINGLEMDS lctl set_param fail_loc=0 + log "waiting for multiop ($multiop_pid) to finish" + wait $multiop_pid || error "multiop returned error" } test_101() @@ -834,18 +651,18 @@ test_101() # started from default flavors restore_to_default_flavor - switch_sec_test $cnt_cli2mdt null krb5n null - switch_sec_test $cnt_cli2mdt null krb5a null - switch_sec_test $cnt_cli2mdt null krb5i null - switch_sec_test $cnt_cli2mdt null krb5p null - switch_sec_test $cnt_cli2mdt null krb5i plain - switch_sec_test $cnt_cli2mdt plain krb5p plain - switch_sec_test $cnt_cli2mdt plain krb5n krb5a - switch_sec_test $cnt_cli2mdt krb5a krb5i krb5p - switch_sec_test $cnt_cli2mdt krb5p krb5a krb5n - switch_sec_test $cnt_cli2mdt krb5n krb5p krb5i + switch_sec_test null plain + switch_sec_test plain krb5n + switch_sec_test krb5n krb5a + switch_sec_test krb5a krb5i + switch_sec_test krb5i krb5p + switch_sec_test krb5p null + switch_sec_test null krb5p + switch_sec_test krb5p krb5i + switch_sec_test krb5i plain + switch_sec_test plain krb5p } -run_test 101 "switch ctx as well as sec for resending request" +run_test 101 "switch ctx/sec for resending request" error_102() { @@ -873,7 +690,7 @@ test_102() { set_rule $FSNAME any any null check_dbench - wait_flavor all2all null $cnt_all2all || error_dbench "1" + wait_flavor all2all null || error_dbench "1" check_dbench echo "waiting for 15s and check again" @@ -888,7 +705,7 @@ test_102() { set_rule $FSNAME any any krb5i check_dbench - wait_flavor all2all krb5i $cnt_all2all || error_dbench "2" + wait_flavor all2all krb5i || error_dbench "2" check_dbench echo "waiting for 15s and check again" @@ -899,6 +716,74 @@ test_102() { } run_test 102 "survive from insanely fast flavor switch" -equals_msg `basename $0`: test complete, cleaning up +test_150() { + local save_opts + local count + local clients=$CLIENTS + + [ -z $clients ] && clients=$HOSTNAME + + # started from default flavors + restore_to_default_flavor + + # at this time no rules has been set on mgs; mgc use null + # flavor connect to mgs. + count=`flvr_cnt_mgc2mgs null` + [ $count -eq 1 ] || error "$count mgc connection use null flavor" + + zconf_umount_clients $clients $MOUNT || return 1 + + # mount client with conflict flavor - should fail + save_opts=$MOUNTOPT + MOUNTOPT="$MOUNTOPT,mgssec=krb5p" + zconf_mount_clients $clients $MOUNT && \ + error "mount with conflict flavor should have failed" + MOUNTOPT=$save_opts + + # mount client with same flavor - should succeed + save_opts=$MOUNTOPT + MOUNTOPT="$MOUNTOPT,mgssec=null" + zconf_mount_clients $clients $MOUNT || \ + error "mount with same flavor should have succeeded" + MOUNTOPT=$save_opts + zconf_umount_clients $clients $MOUNT || return 2 + + # mount client with default flavor - should succeed + zconf_mount_clients $clients $MOUNT || \ + error "mount with default flavor should have succeeded" +} +run_test 150 "secure mgs connection: client flavor setting" + +test_151() { + local save_opts + + # set mgs only accept krb5p + set_rule _mgs any any krb5p + + # umount everything, modules still loaded + stopall + + # mount mgs with default flavor, in current framework it means mgs+mdt1. + # the connection of mgc of mdt1 to mgs is expected fail. + DEVNAME=$(mdsdevname 1) + start mds1 $DEVNAME $MDS_MOUNT_OPTS && error "mount with default flavor should have failed" + + # mount with unauthorized flavor should fail + save_opts=$MDS_MOUNT_OPTS + MDS_MOUNT_OPTS="$MDS_MOUNT_OPTS,mgssec=null" + start mds1 $DEVNAME $MDS_MOUNT_OPTS && error "mount with unauthorized flavor should have failed" + MDS_MOUNT_OPTS=$save_opts + + # mount with designated flavor should succeed + save_opts=$MDS_MOUNT_OPTS + MDS_MOUNT_OPTS="$MDS_MOUNT_OPTS,mgssec=krb5p" + start mds1 $DEVNAME $MDS_MOUNT_OPTS || error "mount with designated flavor should have succeeded" + MDS_MOUNT_OPTS=$save_opts + + stop mds1 -f +} +run_test 151 "secure mgs connection: server flavor control" + +complete $SECONDS check_and_cleanup_lustre -[ -f "$TESTSUITELOG" ] && cat $TESTSUITELOG || true +exit_status