2 # vim:shiftwidth=4:softtabstop=4:tabstop=4:
4 # Run select tests by setting ONLY, or as arguments to the script.
5 # Skip specific tests by setting EXCEPT.
7 # e.g. ONLY="22 23" or ONLY="`seq 32 39`" or EXCEPT="31"
11 # bug number for skipped test:
12 ALWAYS_EXCEPT=${ALWAYS_EXCEPT:-"$SANITY_GSS_EXCEPT"}
13 # UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT!
15 # Tests that fail on uml
16 CPU=`awk '/model/ {print $4}' /proc/cpuinfo`
17 [ "$CPU" = "UML" ] && EXCEPT="$EXCEPT"
20 2.6*) ALWAYS_EXCEPT="$ALWAYS_EXCEPT " ;;
21 *) error "unsupported kernel (gss only works with 2.6.x)" ;;
25 export PATH=$PWD/$SRCDIR:$SRCDIR:$SRCDIR/../utils:$SRCDIR/../utils/gss:$PATH:/sbin
26 export NAME=${NAME:-local}
29 export MULTIOP=${MULTIOP:-multiop}
31 CLEANUP=${CLEANUP:-""}
34 LUSTRE=${LUSTRE:-`dirname $0`/..}
35 . $LUSTRE/tests/test-framework.sh
37 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
40 require_dsh_mds || exit 0
42 [ "$SLOW" = "no" ] && EXCEPT_SLOW="100 101"
44 # $RUNAS_ID may get set incorrectly somewhere else
45 [ $UID -eq 0 -a $RUNAS_ID -eq 0 ] && error "\$RUNAS_ID set to 0, but \$UID is also 0!"
47 # remove $SEC, we'd like to control everything by ourselves
51 # global variables of this sanity
54 KRB5_CRED=$KRB5_CCACHE_DIR/krb5cc_$RUNAS_ID
55 KRB5_CRED_SAVE=$KRB5_CCACHE_DIR/krb5cc.sanity.save
62 prepare_krb5_creds() {
63 echo prepare krb5 cred
66 $RUNAS krb5_login.sh || exit 1
67 [ -f $KRB5_CRED ] || exit 2
69 cp $KRB5_CRED $KRB5_CRED_SAVE
74 # we want double mount
75 MOUNT_2=${MOUNT_2:-"yes"}
76 check_and_setup_lustre
78 rm -rf $DIR/[df][0-9]*
80 check_runas_id $RUNAS_ID $RUNAS_ID $RUNAS
86 NPROC=`cat /proc/cpuinfo 2>/dev/null | grep ^processor | wc -l`
87 [ $NPROC -gt 2 ] && NPROC=2
88 sh rundbench $NPROC 1>/dev/null &
92 num=`ps --no-headers -p $DBENCH_PID 2>/dev/null | wc -l`
93 if [ $num -ne 1 ]; then
94 error "failed to start dbench $NPROC"
96 echo "started dbench with $NPROC processes at background"
104 num=`ps --no-headers -p $DBENCH_PID 2>/dev/null | wc -l`
105 if [ $num -eq 0 ]; then
106 echo "dbench $DBENCH_PID already finished"
107 wait $DBENCH_PID || error "dbench $PID exit with error"
109 elif [ $num -ne 1 ]; then
111 error "found $num instance of pid $DBENCH_PID ???"
120 killall dbench 2>/dev/null
121 num=`ps --no-headers -p $DBENCH_PID | wc -l`
122 if [ $num -eq 0 ]; then
123 echo "dbench finished"
126 echo "dbench $DBENCH_PID is still running, waiting 2s..."
130 wait $DBENCH_PID || true
134 restore_krb5_cred() {
135 cp $KRB5_CRED_SAVE $KRB5_CRED
136 chown $RUNAS_ID:$RUNAS_ID $KRB5_CRED
137 chmod 0600 $KRB5_CRED
140 check_multiple_gss_daemons() {
143 local gssd_name=`basename $gssd`
145 for ((i=0;i<10;i++)); do
146 do_facet $facet "$gssd -v &"
149 # wait daemons entering "stable" status
152 num=`do_facet $facet ps -o cmd -C $gssd_name | grep $gssd_name | wc -l`
153 echo "$num instance(s) of $gssd_name are running"
155 if [ $num -ne 1 ]; then
156 error "$gssd_name not unique"
166 echo "bring up gss daemons..."
169 echo "check with someone already running..."
170 check_multiple_gss_daemons $my_facet $LSVCGSSD
172 check_multiple_gss_daemons $my_facet $LGSSD
175 echo "check with someone run & finished..."
176 do_facet $my_facet killall -q -2 lgssd lsvcgssd || true
177 sleep 5 # wait fully exit
178 check_multiple_gss_daemons $my_facet $LSVCGSSD
180 check_multiple_gss_daemons $my_facet $LGSSD
183 echo "check refresh..."
184 do_facet $my_facet killall -q -2 lgssd lsvcgssd || true
185 sleep 5 # wait fully exit
186 do_facet $my_facet ipcrm -S 0x3b92d473
187 check_multiple_gss_daemons $my_facet $LSVCGSSD
189 do_facet $my_facet ipcrm -S 0x3a92d473
190 check_multiple_gss_daemons $my_facet $LGSSD
193 run_test 0 "start multiple gss daemons"
198 local file=$DIR/$tfile
200 chmod 0777 $DIR || error "chmod $DIR failed"
204 $RUNAS $LFS flushctx $MOUNT || error "can't flush context on $MOUNT"
205 $RUNAS touch $file && error "unexpected success"
209 $RUNAS touch $file || error "should not fail"
210 [ -f $file ] || error "$file not found"
212 run_test 1 "access with or without krb5 credential"
215 local file1=$DIR/$tfile-1
216 local file2=$DIR/$tfile-2
218 chmod 0777 $DIR || error "chmod $DIR failed"
219 # current access should be ok
220 $RUNAS touch $file1 || error "can't touch $file1"
221 [ -f $file1 ] || error "$file1 not found"
223 # cleanup all cred/ctx and touch
225 $RUNAS $LFS flushctx $MOUNT || error "can't flush context on $MOUNT"
226 $RUNAS touch $file2 && error "unexpected success"
230 $RUNAS touch $file2 || error "should not fail"
231 [ -f $file2 ] || error "$file2 not found"
233 run_test 2 "lfs flushctx"
236 local file=$DIR/$tfile
239 echo "aaaaaaaaaaaaaaaaa" > $file
241 $CHECKSTAT -p 0666 $file || error "$UID checkstat error"
242 $RUNAS $CHECKSTAT -p 0666 $file || error "$RUNAS_ID checkstat error"
243 $RUNAS cat $file > /dev/null || error "$RUNAS_ID cat error"
246 $RUNAS $MULTIOP $file o_r &
248 # wait multiop finish its open()
251 # cleanup all cred/ctx and check
252 # metadata check should fail, but file data check should success
253 # because we always use root credential to OSTs
255 $RUNAS $LFS flushctx $MOUNT || error "can't flush context on $MOUNT"
256 echo "destroied credentials/contexs for $RUNAS_ID"
257 $RUNAS $CHECKSTAT -p 0666 $file && error "checkstat succeed"
259 wait $OPPID || error "read file data failed"
260 echo "read file data OK"
262 # restore and check again
264 echo "restored credentials for $RUNAS_ID"
265 $RUNAS $CHECKSTAT -p 0666 $file || error "$RUNAS_ID checkstat (2) error"
266 echo "$RUNAS_ID checkstat OK"
267 $CHECKSTAT -p 0666 $file || error "$UID checkstat (2) error"
268 echo "$UID checkstat OK"
269 $RUNAS cat $file > /dev/null || error "$RUNAS_ID cat (2) error"
270 echo "$RUNAS_ID read file data OK"
272 run_test 3 "local cache under DLM lock"
275 local file1=$DIR/$tfile-1
276 local file2=$DIR/$tfile-2
278 ! $GSS_PIPEFS && skip "pipefs not used" && return
280 chmod 0777 $DIR || error "chmod $DIR failed"
281 # current access should be ok
282 $RUNAS touch $file1 || error "can't touch $file1"
283 [ -f $file1 ] || error "$file1 not found"
286 send_sigint client lgssd
288 check_gss_daemon_facet client lgssd && error "lgssd still running"
290 # flush context, and touch
291 $RUNAS $LFS flushctx $MOUNT || error "can't flush context on $MOUNT"
292 $RUNAS touch $file2 &
294 echo "waiting touch pid $TOUCHPID"
295 wait $TOUCHPID && error "touch should fail"
298 do_facet client "$LGSSD -v"
300 check_gss_daemon_facet client lgssd
302 # touch new should succeed
303 $RUNAS touch $file2 || error "can't touch $file2"
304 [ -f $file2 ] || error "$file2 not found"
306 run_test 4 "lgssd dead, operations should wait timeout and fail"
309 local file1=$DIR/$tfile-1
310 local file2=$DIR/$tfile-2
311 local wait_time=$((TIMEOUT + TIMEOUT / 2))
313 chmod 0777 $DIR || error "chmod $DIR failed"
314 # current access should be ok
315 $RUNAS touch $file1 || error "can't touch $file1"
316 [ -f $file1 ] || error "$file1 not found"
319 send_sigint $(comma_list $(mdts_nodes)) lsvcgssd
321 check_gss_daemon_nodes $(comma_list $(mdts_nodes)) lsvcgssd && error "lsvcgssd still running"
323 # flush context, and touch
324 $RUNAS $LFS flushctx $MOUNT || error "can't flush context on $MOUNT"
325 $RUNAS touch $file2 &
329 echo "waiting $wait_time seconds for touch pid $TOUCHPID"
331 num=`ps --no-headers -p $TOUCHPID | wc -l`
332 [ $num -eq 1 ] || error "touch already ended ($num)"
333 echo "process $TOUCHPID still hanging there... OK"
335 # restart lsvcgssd, expect touch suceed
336 echo "restart lsvcgssd and recovering"
337 start_gss_daemons $(comma_list $(mdts_nodes)) "$LSVCGSSD -v"
339 check_gss_daemon_nodes $(comma_list $(mdts_nodes)) lsvcgssd
340 wait $TOUCHPID || error "touch fail"
341 [ -f $file2 ] || error "$file2 not found"
343 run_test 5 "lsvcgssd dead, operations lead to recovery"
348 mkdir $DIR/d6 || error "mkdir $DIR/d6 failed"
349 for ((i=0; i<$nfile; i++)); do
350 dd if=/dev/zero of=$DIR/d6/file$i bs=8k count=1 || error "dd file$i failed"
352 ls -l $DIR/d6/* > /dev/null || error "ls failed"
353 rm -rf $DIR2/d6/* || error "rm failed"
354 rmdir $DIR2/d6/ || error "rmdir failed"
356 run_test 6 "test basic DLM callback works"
363 # for open(), client only reserve space for default stripe count lovea,
364 # and server may return larger lovea in reply (because of larger stripe
365 # count), client need call enlarge_reqbuf() and save the replied lovea
366 # in request for future possible replay.
368 # Note: current script does NOT guarantee enlarge_reqbuf() will be in
369 # the path, however it does work in local test which has 2 OSTs and
370 # default stripe count is 1.
372 num_osts=`$LFS getstripe $MOUNT | egrep "^[0-9]*:.*ACTIVE" | wc -l`
373 echo "found $num_osts active OSTs"
374 [ $num_osts -lt 2 ] && echo "skipping $TESTNAME (must have >= 2 OSTs)" && return
377 $LFS setstripe -c $num_osts $tdir || error
380 for ((i=0;i<20;i++)); do
381 dd if=/dev/zero of=$tdir/f$i bs=4k count=16 2>/dev/null
384 for ((i=0;i<20;i++)); do
385 dd if=$tdir/f$i of=/dev/null bs=4k count=16 2>/dev/null
389 run_test 7 "exercise enlarge_reqbuf()"
393 local ATHISTORY=$(do_facet $SINGLEMDS "find /sys/ -name at_history")
394 local ATOLDBASE=$(do_facet $SINGLEMDS "cat $ATHISTORY")
396 do_facet $SINGLEMDS "echo 8 >> $ATHISTORY"
403 sysctl -w lnet.debug="+other"
405 # wait for the at estimation come down, this is faster
407 REQ_DELAY=`lctl get_param -n mdc.${FSNAME}-MDT0000-mdc-*.timeouts |
408 awk '/portal 12/ {print $5}' | tail -1`
409 [ $REQ_DELAY -le 5 ] && break
410 echo "current AT estimation is $REQ_DELAY, wait a little bit"
413 REQ_DELAY=$((${REQ_DELAY} + ${REQ_DELAY} / 4 + 5))
415 # sleep sometime in ctx handle
416 do_facet $SINGLEMDS lctl set_param fail_val=$REQ_DELAY
417 #define OBD_FAIL_SEC_CTX_HDL_PAUSE 0x1204
418 do_facet $SINGLEMDS lctl set_param fail_loc=0x1204
420 $RUNAS $LFS flushctx $MOUNT || error "can't flush context on $MOUNT"
422 $RUNAS touch $DIR/d8/f &
424 echo "waiting for touch (pid $TOUCHPID) to finish..."
425 sleep 2 # give it a chance to really trigger context init rpc
426 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
427 wait $TOUCHPID || error "touch should have succeeded"
429 $LCTL dk | grep "Early reply #" || error "No early reply"
432 do_facet $SINGLEMDS "echo $ATOLDBASE >> $ATHISTORY" || true
434 run_test 8 "Early reply sent for slow gss context negotiation"
437 # following tests will manipulate flavors and may end with any flavor set,
438 # so each test should not assume any start flavor.
442 if [ "$SLOW" = "no" ]; then
448 restore_to_default_flavor
449 set_rule $FSNAME any any krb5p
450 wait_flavor all2all krb5p
454 for ((n=0;n<$total;n++)); do
457 echo "flush ctx ($n/$total) ..."
458 $LFS flushctx $MOUNT || error "can't flush context on $MOUNT"
461 #sleep to let ctxs be re-established
465 run_test 90 "recoverable from losing contexts under load"
475 nrule_old=`do_facet mgs lctl get_param -n mgs.MGS.live.$FSNAME 2>/dev/null \
476 | grep "$FSNAME.srpc.flavor." | wc -l`
477 echo "original general rules: $nrule_old"
479 for ((i = $nrule_old; i < $max; i++)); do
480 set_rule $FSNAME elan$i any krb5n || error "set rule $i"
482 for ((i = $nrule_old; i < $max; i++)); do
483 set_rule $FSNAME elan$i any || error "remove rule $i"
486 nrule_new=`do_facet mgs lctl get_param -n mgs.MGS.live.$FSNAME 2>/dev/null \
487 | grep "$FSNAME.srpc.flavor." | wc -l`
488 if [ $nrule_new != $nrule_old ]; then
489 error "general rule: $nrule_new != $nrule_old"
493 # target-specific rules
495 nrule_old=`do_facet mgs lctl get_param -n mgs.MGS.live.$FSNAME 2>/dev/null \
496 | grep "$FSNAME-MDT0000.srpc.flavor." | wc -l`
497 echo "original target rules: $nrule_old"
499 for ((i = $nrule_old; i < $max; i++)); do
500 set_rule $FSNAME-MDT0000 elan$i any krb5i || error "set rule $i"
502 for ((i = $nrule_old; i < $max; i++)); do
503 set_rule $FSNAME-MDT0000 elan$i any || error "remove rule $i"
506 nrule_new=`do_facet mgs lctl get_param -n mgs.MGS.live.$FSNAME 2>/dev/null \
507 | grep "$FSNAME-MDT0000.srpc.flavor." | wc -l`
508 if [ $nrule_new != $nrule_old ]; then
509 error "general rule: $nrule_new != $nrule_old"
512 run_test 99 "set large number of sptlrpc rules"
525 # started from default flavors
526 restore_to_default_flavor
528 # running dbench background
532 # all: null -> krb5n -> krb5a -> krb5i -> krb5p -> plain
534 set_rule $FSNAME any any krb5n
535 wait_flavor all2all krb5n || error_dbench "1"
538 set_rule $FSNAME any any krb5a
539 wait_flavor all2all krb5a || error_dbench "2"
542 set_rule $FSNAME any any krb5i
543 wait_flavor all2all krb5i || error_dbench "3"
546 set_rule $FSNAME any any krb5p
547 wait_flavor all2all krb5p || error_dbench "4"
550 set_rule $FSNAME any any plain
551 wait_flavor all2all plain || error_dbench "5"
560 set_rule $FSNAME any mdt2mdt krb5a
561 wait_flavor mdt2mdt krb5a || error_dbench "6"
564 set_rule $FSNAME any cli2mdt krb5i
565 wait_flavor cli2mdt krb5i || error_dbench "7"
568 set_rule $FSNAME any mdt2ost krb5p
569 wait_flavor mdt2ost krb5p || error_dbench "8"
572 set_rule $FSNAME any cli2ost krb5n
573 wait_flavor cli2ost krb5n || error_dbench "9"
580 # nothing should be changed because they are override by above dir rules
582 set_rule $FSNAME-MDT0000 any any krb5p
583 set_rule $FSNAME-OST0000 any any krb5i
584 wait_flavor mdt2mdt krb5a || error_dbench "10"
585 wait_flavor cli2mdt krb5i || error_dbench "11"
587 wait_flavor mdt2ost krb5p || error_dbench "12"
588 wait_flavor cli2ost krb5n || error_dbench "13"
591 # delete all dir-specific rules
593 set_rule $FSNAME any mdt2mdt
594 set_rule $FSNAME any cli2mdt
595 set_rule $FSNAME any mdt2ost
596 set_rule $FSNAME any cli2ost
597 wait_flavor mdt2mdt krb5p $((MDSCOUNT - 1)) || error_dbench "14"
598 wait_flavor cli2mdt krb5p $(get_clients_mount_count) || error_dbench "15"
600 wait_flavor mdt2ost krb5i $MDSCOUNT || error_dbench "16"
601 wait_flavor cli2ost krb5i $(get_clients_mount_count) || error_dbench "17"
609 set_rule $FSNAME-MDT0000 any any
610 set_rule $FSNAME-OST0000 any any || error_dbench "18"
611 wait_flavor all2all plain || error_dbench "19"
616 run_test 100 "change security flavor on the fly under load"
622 local filename=$DIR/$tfile
627 # after set to flavor0, start multop which use flavor0 rpc, and let
628 # server drop the reply; then switch to flavor1, the resend should be
629 # completed using flavor1. To exercise the code of switching ctx/sec
630 # for a resend request.
632 log ">>>>>>>>>>>>>>> Testing $flavor0 -> $flavor1 <<<<<<<<<<<<<<<<<<<"
634 set_rule $FSNAME any cli2mdt $flavor0
635 wait_flavor cli2mdt $flavor0
636 rm -f $filename || error "remove old $filename failed"
639 #define OBD_FAIL_PTLRPC_DROP_REQ_OPC 0x513
640 do_facet $SINGLEMDS lctl set_param fail_val=36
641 do_facet $SINGLEMDS lctl set_param fail_loc=0x513
642 log "starting multiop"
643 $MULTIOP $filename m &
645 echo "multiop pid=$multiop_pid"
648 set_rule $FSNAME any cli2mdt $flavor1
649 wait_flavor cli2mdt $flavor1
651 num=`ps --no-headers -p $multiop_pid 2>/dev/null | wc -l`
652 [ $num -eq 1 ] || error "multiop($multiop_pid) already ended ($num)"
653 echo "process $multiop_pid is still hanging there... OK"
655 do_facet $SINGLEMDS lctl set_param fail_loc=0
656 log "waiting for multiop ($multiop_pid) to finish"
657 wait $multiop_pid || error "multiop returned error"
662 # started from default flavors
663 restore_to_default_flavor
665 switch_sec_test null plain
666 switch_sec_test plain krb5n
667 switch_sec_test krb5n krb5a
668 switch_sec_test krb5a krb5i
669 switch_sec_test krb5i krb5p
670 switch_sec_test krb5p null
671 switch_sec_test null krb5p
672 switch_sec_test krb5p krb5i
673 switch_sec_test krb5i plain
674 switch_sec_test plain krb5p
676 run_test 101 "switch ctx/sec for resending request"
689 # started from default flavors
690 restore_to_default_flavor
692 # run dbench background
695 echo "Testing null->krb5n->krb5a->krb5i->krb5p->plain->null"
696 set_rule $FSNAME any any krb5n
697 set_rule $FSNAME any any krb5a
698 set_rule $FSNAME any any krb5i
699 set_rule $FSNAME any any krb5p
700 set_rule $FSNAME any any plain
701 set_rule $FSNAME any any null
704 wait_flavor all2all null || error_dbench "1"
707 echo "waiting for 15s and check again"
711 echo "Testing null->krb5i->null->krb5i->null..."
712 for ((i=0; i<10; i++)); do
713 set_rule $FSNAME any any krb5i
714 set_rule $FSNAME any any null
716 set_rule $FSNAME any any krb5i
719 wait_flavor all2all krb5i || error_dbench "2"
722 echo "waiting for 15s and check again"
728 run_test 102 "survive from insanely fast flavor switch"
733 local clients=$CLIENTS
735 [ -z $clients ] && clients=$HOSTNAME
737 # started from default flavors
738 restore_to_default_flavor
740 # at this time no rules has been set on mgs; mgc use null
741 # flavor connect to mgs.
742 count=`flvr_cnt_mgc2mgs null`
743 [ $count -eq 1 ] || error "$count mgc connection use null flavor"
745 zconf_umount_clients $clients $MOUNT || return 1
747 # mount client with conflict flavor - should fail
749 MOUNTOPT="$MOUNTOPT,mgssec=krb5p"
750 zconf_mount_clients $clients $MOUNT && \
751 error "mount with conflict flavor should have failed"
754 # mount client with same flavor - should succeed
756 MOUNTOPT="$MOUNTOPT,mgssec=null"
757 zconf_mount_clients $clients $MOUNT || \
758 error "mount with same flavor should have succeeded"
760 zconf_umount_clients $clients $MOUNT || return 2
762 # mount client with default flavor - should succeed
763 zconf_mount_clients $clients $MOUNT || \
764 error "mount with default flavor should have succeeded"
766 run_test 150 "secure mgs connection: client flavor setting"
771 # set mgs only accept krb5p
772 set_rule _mgs any any krb5p
774 # umount everything, modules still loaded
777 # mount mgs with default flavor, in current framework it means mgs+mdt1.
778 # the connection of mgc of mdt1 to mgs is expected fail.
779 DEVNAME=$(mdsdevname 1)
780 start mds1 $DEVNAME $MDS_MOUNT_OPTS && error "mount with default flavor should have failed"
782 # mount with unauthorized flavor should fail
783 save_opts=$MDS_MOUNT_OPTS
784 MDS_MOUNT_OPTS="$MDS_MOUNT_OPTS,mgssec=null"
785 start mds1 $DEVNAME $MDS_MOUNT_OPTS && error "mount with unauthorized flavor should have failed"
786 MDS_MOUNT_OPTS=$save_opts
788 # mount with designated flavor should succeed
789 save_opts=$MDS_MOUNT_OPTS
790 MDS_MOUNT_OPTS="$MDS_MOUNT_OPTS,mgssec=krb5p"
791 start mds1 $DEVNAME $MDS_MOUNT_OPTS || error "mount with designated flavor should have succeeded"
792 MDS_MOUNT_OPTS=$save_opts
796 run_test 151 "secure mgs connection: server flavor control"
799 check_and_cleanup_lustre