3 # add uml1 uml2 uml3 in your /etc/hosts
5 # FIXME - there is no reason to use all of these different
6 # return codes, espcially when most of them are mapped to something
7 # else anyway. The combination of test number and return code
8 # figure out what failed.
14 # These tests don't apply to mountconf
15 # xml xml xml xml xml xml dumb FIXME
16 MOUNTCONFSKIP="10 11 12 13 13b 14 15 18"
18 # bug number for skipped test:
19 ALWAYS_EXCEPT=" $CONF_SANITY_EXCEPT $MOUNTCONFSKIP"
20 # UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT!
23 PATH=$PWD/$SRCDIR:$SRCDIR:$SRCDIR/../utils:$PATH
25 LUSTRE=${LUSTRE:-`dirname $0`/..}
26 RLUSTRE=${RLUSTRE:-$LUSTRE}
29 . $LUSTRE/tests/test-framework.sh
31 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
42 # who knows if/where $TUNEFS is installed? Better reformat if it fails...
43 do_facet ${facet} "$TUNEFS --writeconf $MDSDEV" || echo "tunefs failed, reformatting instead" && reformat
48 # The MGS must be started before the OSTs for a new fs, so start
49 # and stop to generate the startup logs.
58 echo "start mds service on `facet_active_host mds`"
59 start mds $MDSDEV $MDS_MOUNT_OPTS || return 94
63 echo "stop mds service on `facet_active_host mds`"
64 # These tests all use non-failover stop
65 stop mds -f || return 97
69 echo "start ost1 service on `facet_active_host ost1`"
70 start ost1 `ostdevname 1` $OST_MOUNT_OPTS || return 95
74 echo "stop ost1 service on `facet_active_host ost1`"
75 # These tests all use non-failover stop
76 stop ost1 -f || return 98
80 echo "start ost2 service on `facet_active_host ost2`"
81 start ost2 `ostdevname 2` $OST_MOUNT_OPTS || return 92
85 echo "stop ost2 service on `facet_active_host ost2`"
86 # These tests all use non-failover stop
87 stop ost2 -f || return 93
92 echo "mount $FSNAME on ${MOUNTPATH}....."
93 zconf_mount `hostname` $MOUNTPATH || return 96
97 local SAVEMOUNTOPT=$MOUNTOPT
100 echo "remount '$1' lustre on ${MOUNTPATH}....."
101 zconf_mount `hostname` $MOUNTPATH || return 96
102 MOUNTOPT=$SAVEMOUNTOPT
107 echo "umount lustre on ${MOUNTPATH}....."
108 zconf_umount `hostname` $MOUNTPATH || return 97
111 manual_umount_client(){
112 echo "manual umount lustre on ${MOUNT}...."
113 do_facet client "umount -d $MOUNT"
123 stop_mds || return 201
124 stop_ost || return 202
125 unload_modules || return 203
129 umount_client $MOUNT || return 200
130 cleanup_nocli || return $?
134 do_facet client "cp /etc/passwd $DIR/a" || return 71
135 do_facet client "rm $DIR/a" || return 72
136 # make sure lustre is actually mounted (touch will block,
137 # but grep won't, so do it after)
138 do_facet client "grep $MOUNT' ' /proc/mounts > /dev/null" || return 73
139 echo "setup single mount lustre success"
143 do_facet client "touch $DIR/a" || return 71
144 do_facet client "rm $DIR/a" || return 72
145 do_facet client "touch $DIR2/a" || return 73
146 do_facet client "rm $DIR2/a" || return 74
147 echo "setup double mount lustre success"
152 if [ "$ONLY" == "setup" ]; then
157 if [ "$ONLY" == "cleanup" ]; then
162 #create single point mountpoint
169 check_mount || return 41
172 run_test 0 "single mount setup"
176 echo "start ost second time..."
178 check_mount || return 42
181 run_test 1 "start up ost twice (should return errors)"
186 echo "start mds second time.."
189 check_mount || return 43
192 run_test 2 "start up mds twice (should return err)"
196 #mount.lustre returns an error if already in mtab
197 mount_client $MOUNT && return $?
198 check_mount || return 44
201 run_test 3 "mount client twice (should return err)"
205 touch $DIR/$tfile || return 85
209 # ok for ost to fail shutdown
210 if [ 202 -ne $eno ]; then
215 run_test 4 "force cleanup ost, then cleanup"
219 touch $DIR/$tfile || return 1
220 stop_mds -f || return 2
222 # cleanup may return an error from the failed
223 # disconnects; for now I'll consider this successful
224 # if all the modules have unloaded.
228 echo "killing umount"
229 kill -TERM $UMOUNT_PID
230 echo "waiting for umount to finish"
232 if grep " $MOUNT " /etc/mtab; then
233 echo "test 5: mtab after failed umount"
237 echo "killing umount"
238 kill -TERM $UMOUNT_PID
239 echo "waiting for umount to finish"
241 grep " $MOUNT " /etc/mtab && echo "test 5: mtab after second umount" && return 11
245 # stop_mds is a no-op here, and should not fail
246 cleanup_nocli || return $?
247 # df may have lingering entry
249 # mtab may have lingering entry
250 grep -v $MOUNT" " /etc/mtab > $TMP/mtabtemp
251 mv $TMP/mtabtemp /etc/mtab
253 run_test 5 "force cleanup mds, then cleanup"
257 [ -d $MOUNT ] || mkdir -p $MOUNT
258 grep " $MOUNT " /etc/mtab && echo "test 5b: mtab before mount" && return 10
259 mount_client $MOUNT && return 1
260 grep " $MOUNT " /etc/mtab && echo "test 5b: mtab after failed mount" && return 11
262 # stop_mds is a no-op here, and should not fail
263 cleanup_nocli || return $?
266 run_test 5b "mds down, cleanup after failed mount (bug 2712) (should return errs)"
271 [ -d $MOUNT ] || mkdir -p $MOUNT
272 grep " $MOUNT " /etc/mtab && echo "test 5c: mtab before mount" && return 10
273 mount -t lustre $MGSNID:/wrong.$FSNAME $MOUNT || :
274 grep " $MOUNT " /etc/mtab && echo "test 5c: mtab after failed mount" && return 11
276 cleanup_nocli || return $?
278 run_test 5c "cleanup after failed mount (bug 2712) (should return errs)"
284 grep " $MOUNT " /etc/mtab && echo "test 5d: mtab before mount" && return 10
285 mount_client $MOUNT || return 1
287 grep " $MOUNT " /etc/mtab && echo "test 5d: mtab after unmount" && return 11
290 run_test 5d "mount with ost down"
295 # give MDS a chance to connect to OSTs (bz 10476)
298 #define OBD_FAIL_PTLRPC_DELAY_SEND 0x506
299 do_facet client "sysctl -w lustre.fail_loc=0x80000506"
300 grep " $MOUNT " /etc/mtab && echo "test 5e: mtab before mount" && return 10
301 mount_client $MOUNT || echo "mount failed (not fatal)"
303 grep " $MOUNT " /etc/mtab && echo "test 5e: mtab after unmount" && return 11
306 run_test 5e "delayed connect, don't crash (bug 10268)"
311 mount_client ${MOUNT} || return 87
312 touch $DIR/a || return 86
315 run_test 6 "manual umount, then mount again"
320 cleanup_nocli || return $?
322 run_test 7 "manual umount, then cleanup"
327 check_mount2 || return 45
328 umount_client $MOUNT2
331 run_test 8 "double mount setup"
336 do_facet ost1 sysctl lnet.debug=\'inode trace\' || return 1
337 do_facet ost1 sysctl lnet.subsystem_debug=\'mds ost\' || return 1
339 CHECK_PTLDEBUG="`do_facet ost1 sysctl -n lnet.debug`"
340 if [ "$CHECK_PTLDEBUG" ] && [ "$CHECK_PTLDEBUG" = "trace inode" ];then
341 echo "lnet.debug success"
343 echo "lnet.debug: want 'trace inode', have '$CHECK_PTLDEBUG'"
346 CHECK_SUBSYS="`do_facet ost1 sysctl -n lnet.subsystem_debug`"
347 if [ "$CHECK_SUBSYS" ] && [ "$CHECK_SUBSYS" = "mds ost" ]; then
348 echo "lnet.subsystem_debug success"
350 echo "lnet.subsystem_debug: want 'mds ost', have '$CHECK_SUBSYS'"
353 stop_ost || return $?
356 run_test 9 "test ptldebug and subsystem for mkfs"
359 echo "generate configuration with the same name for node and mds"
360 OLDXMLCONFIG=$XMLCONFIG
361 XMLCONFIG="broken.xml"
362 [ -f "$XMLCONFIG" ] && rm -f $XMLCONFIG
366 echo "the name for node and mds is the same"
367 do_lmc --add mds --node ${facet}_facet --mds ${facet}_facet \
368 --dev $MDSDEV --size $MDSSIZE || return $?
369 do_lmc --add lov --mds ${facet}_facet --lov lov1 --stripe_sz \
370 $STRIPE_BYTES --stripe_cnt $STRIPES_PER_OBJ \
371 --stripe_pattern 0 || return $?
372 add_ost ost --lov lov1 --dev $OSTDEV --size $OSTSIZE
374 add_facet $facet --lustre_upcall $UPCALL
375 do_lmc --add mtpt --node ${facet}_facet --mds mds_facet \
376 --lov lov1 --path $MOUNT
382 check_mount || return 41
386 XMLCONFIG=$OLDXMLCONFIG
388 run_test 10 "mount lustre with the same name for node and mds"
391 OLDXMLCONFIG=$XMLCONFIG
392 XMLCONFIG="conf11.xml"
394 [ -f "$XMLCONFIG" ] && rm -f $XMLCONFIG
395 add_mds mds --dev $MDSDEV --size $MDSSIZE
396 add_ost ost --dev $OSTDEV --size $OSTSIZE
397 add_client client mds --path $MOUNT --ost ost_svc || return $?
398 echo "Default lov config success!"
400 [ -f "$XMLCONFIG" ] && rm -f $XMLCONFIG
401 add_mds mds --dev $MDSDEV --size $MDSSIZE
402 add_ost ost --dev $OSTDEV --size $OSTSIZE
403 add_client client mds --path $MOUNT && return $?
404 echo "--add mtpt with neither --lov nor --ost will return error"
408 XMLCONFIG=$OLDXMLCONFIG
410 run_test 11 "use default lov configuration (should return error)"
413 OLDXMLCONFIG=$XMLCONFIG
414 XMLCONFIG="batch.xml"
415 BATCHFILE="batchfile"
418 [ -f "$XMLCONFIG" ] && rm -f $XMLCONFIG
419 [ -f "$BATCHFILE" ] && rm -f $BATCHFILE
420 echo "--add net --node $HOSTNAME --nid $HOSTNAME --nettype tcp" > $BATCHFILE
421 echo "--add mds --node $HOSTNAME --mds mds1 --mkfsoptions \"-I 128\"" >> $BATCHFILE
422 # --mkfsoptions "-I 128"
423 do_lmc -m $XMLCONFIG --batch $BATCHFILE || return $?
424 if [ `sed -n '/>-I 128</p' $XMLCONFIG | wc -l` -eq 1 ]; then
425 echo "matched double quote success"
427 echo "matched double quote fail"
432 echo "--add net --node $HOSTNAME --nid $HOSTNAME --nettype tcp" > $BATCHFILE
433 echo "--add mds --node $HOSTNAME --mds mds1 --mkfsoptions \"-I 128" >> $BATCHFILE
434 # --mkfsoptions "-I 128
435 do_lmc -m $XMLCONFIG --batch $BATCHFILE && return $?
436 echo "unmatched double quote should return error"
440 echo "--add net --node $HOSTNAME --nid $HOSTNAME --nettype tcp" > $BATCHFILE
441 echo "--add mds --node $HOSTNAME --mds mds1 --mkfsoptions '-I 128'" >> $BATCHFILE
442 # --mkfsoptions '-I 128'
443 do_lmc -m $XMLCONFIG --batch $BATCHFILE || return $?
444 if [ `sed -n '/>-I 128</p' $XMLCONFIG | wc -l` -eq 1 ]; then
445 echo "matched single quote success"
447 echo "matched single quote fail"
452 echo "--add net --node $HOSTNAME --nid $HOSTNAME --nettype tcp" > $BATCHFILE
453 echo "--add mds --node $HOSTNAME --mds mds1 --mkfsoptions '-I 128" >> $BATCHFILE
454 # --mkfsoptions '-I 128
455 do_lmc -m $XMLCONFIG --batch $BATCHFILE && return $?
456 echo "unmatched single quote should return error"
460 echo "--add net --node $HOSTNAME --nid $HOSTNAME --nettype tcp" > $BATCHFILE
461 echo "--add mds --node $HOSTNAME --mds mds1 --mkfsoptions \-\I\ \128" >> $BATCHFILE
462 # --mkfsoptions \-\I\ \128
463 do_lmc -m $XMLCONFIG --batch $BATCHFILE || return $?
464 if [ `sed -n '/>-I 128</p' $XMLCONFIG | wc -l` -eq 1 ]; then
465 echo "backslash followed by a whitespace/letter success"
467 echo "backslash followed by a whitespace/letter fail"
472 echo "--add net --node $HOSTNAME --nid $HOSTNAME --nettype tcp" > $BATCHFILE
473 echo "--add mds --node $HOSTNAME --mds mds1 --mkfsoptions -I\ 128\\" >> $BATCHFILE
474 # --mkfsoptions -I\ 128\
475 do_lmc -m $XMLCONFIG --batch $BATCHFILE && return $?
476 echo "backslash followed by nothing should return error"
479 XMLCONFIG=$OLDXMLCONFIG
481 run_test 12 "lmc --batch, with single/double quote, backslash in batchfile"
484 OLDXMLCONFIG=$XMLCONFIG
485 XMLCONFIG="conf13-1.xml"
487 # check long uuid will be truncated properly and uniquely
488 echo "To generate XML configuration file(with long ost name): $XMLCONFIG"
489 [ -f "$XMLCONFIG" ] && rm -f $XMLCONFIG
490 do_lmc --add net --node $HOSTNAME --nid $HOSTNAME --nettype tcp
491 do_lmc --add mds --node $HOSTNAME --mds mds1_name_longer_than_31characters
492 do_lmc --add mds --node $HOSTNAME --mds mds2_name_longer_than_31characters
493 if [ ! -f "$XMLCONFIG" ]; then
494 echo "Error:no file $XMLCONFIG created!"
497 EXPECTEDMDS1UUID="e_longer_than_31characters_UUID"
498 EXPECTEDMDS2UUID="longer_than_31characters_UUID_2"
499 FOUNDMDS1UUID=`awk -F"'" '/<mds .*uuid=/' $XMLCONFIG | sed -n '1p' \
500 | sed "s/ /\n\r/g" | awk -F"'" '/uuid=/{print $2}'`
501 FOUNDMDS2UUID=`awk -F"'" '/<mds .*uuid=/' $XMLCONFIG | sed -n '2p' \
502 | sed "s/ /\n\r/g" | awk -F"'" '/uuid=/{print $2}'`
503 [ -z "$FOUNDMDS1UUID" ] && echo "MDS1 UUID empty" && return 1
504 [ -z "$FOUNDMDS2UUID" ] && echo "MDS2 UUID empty" && return 1
505 if ([ $EXPECTEDMDS1UUID = $FOUNDMDS1UUID ] && [ $EXPECTEDMDS2UUID = $FOUNDMDS2UUID ]) || \
506 ([ $EXPECTEDMDS1UUID = $FOUNDMDS2UUID ] && [ $EXPECTEDMDS2UUID = $FOUNDMDS1UUID ]); then
507 echo "Success:long uuid truncated successfully and being unique."
509 echo "Error:expected uuid for mds1 and mds2: $EXPECTEDMDS1UUID; $EXPECTEDMDS2UUID"
510 echo "but: found uuid for mds1 and mds2: $FOUNDMDS1UUID; $FOUNDMDS2UUID"
514 XMLCONFIG=$OLDXMLCONFIG
516 run_test 13 "check new_uuid of lmc operating correctly"
519 OLDXMLCONFIG=$XMLCONFIG
520 XMLCONFIG="conf13-1.xml"
521 SECONDXMLCONFIG="conf13-2.xml"
522 # check multiple invocations for lmc generate same XML configuration file
524 echo "Generate the first XML configuration file"
526 echo "mv $XMLCONFIG to $SECONDXMLCONFIG"
527 sed -e "s/mtime[^ ]*//" $XMLCONFIG > $SECONDXMLCONFIG || return $?
528 echo "Generate the second XML configuration file"
530 # don't compare .xml mtime, it will always be different
531 if [ `sed -e "s/mtime[^ ]*//" $XMLCONFIG | diff - $SECONDXMLCONFIG | wc -l` -eq 0 ]; then
532 echo "Success:multiple invocations for lmc generate same XML file"
534 echo "Error: multiple invocations for lmc generate different XML file"
538 rm -f $XMLCONFIG $SECONDXMLCONFIG
539 XMLCONFIG=$OLDXMLCONFIG
541 run_test 13b "check lmc generates consistent .xml file"
546 # create xml file with --mkfsoptions for ost
547 echo "create xml file with --mkfsoptions for ost"
548 add_mds mds --dev $MDSDEV --size $MDSSIZE
549 add_lov lov1 mds --stripe_sz $STRIPE_BYTES\
550 --stripe_cnt $STRIPES_PER_OBJ --stripe_pattern 0
551 add_ost ost --lov lov1 --dev $OSTDEV --size $OSTSIZE \
552 --mkfsoptions "-Llabel_conf_14"
553 add_client client mds --lov lov1 --path $MOUNT
555 FOUNDSTRING=`awk -F"<" '/<mkfsoptions>/{print $2}' $XMLCONFIG`
556 EXPECTEDSTRING="mkfsoptions>-Llabel_conf_14"
557 if [ "$EXPECTEDSTRING" != "$FOUNDSTRING" ]; then
558 echo "Error: expected: $EXPECTEDSTRING; found: $FOUNDSTRING"
561 echo "Success:mkfsoptions for ost written to xml file correctly."
563 # mount lustre to test lconf mkfsoptions-parsing
567 mount_client $MOUNT || return $?
568 if [ -z "`do_facet ost1 dumpe2fs -h $OSTDEV | grep label_conf_14`" ]; then
569 echo "Error: the mkoptions not applied to mke2fs of ost."
573 echo "lconf mkfsoptions for ost success"
577 run_test 14 "test mkfsoptions of ost for lmc and lconf"
581 [ -f $MOUNTLUSTRE ] && echo "remove $MOUNTLUSTRE" && rm -f $MOUNTLUSTRE
582 if [ -f $MOUNTLUSTRE.sav ]; then
583 echo "return original $MOUNTLUSTRE.sav to $MOUNTLUSTRE"
584 mv $MOUNTLUSTRE.sav $MOUNTLUSTRE
588 # this only tests the kernel mount command, not anything about lustre.
590 MOUNTLUSTRE=${MOUNTLUSTRE:-/sbin/mount.lustre}
594 echo "mount lustre on ${MOUNT} without $MOUNTLUSTRE....."
595 if [ -f "$MOUNTLUSTRE" ]; then
596 echo "save $MOUNTLUSTRE to $MOUNTLUSTRE.sav"
597 mv $MOUNTLUSTRE $MOUNTLUSTRE.sav && trap cleanup_15 EXIT INT
598 if [ -f $MOUNTLUSTRE ]; then
599 echo "$MOUNTLUSTRE cannot be moved, skipping test"
604 mount_client $MOUNT && error "mount succeeded" && return 1
605 echo "mount lustre on $MOUNT without $MOUNTLUSTRE failed as expected"
609 run_test 15 "zconf-mount without /sbin/mount.lustre (should return error)"
612 TMPMTPT="${MOUNT%/*}/conf16"
614 if [ ! -f "$MDSDEV" ]; then
615 echo "no $MDSDEV existing, so mount Lustre to create one"
617 check_mount || return 41
621 echo "change the mode of $MDSDEV/OBJECTS,LOGS,PENDING to 555"
622 do_facet mds "mkdir -p $TMPMTPT &&
623 mount -o loop -t ext3 $MDSDEV $TMPMTPT &&
624 chmod 555 $TMPMTPT/{OBJECTS,LOGS,PENDING} &&
625 umount $TMPMTPT" || return $?
627 echo "mount Lustre to change the mode of OBJECTS/LOGS/PENDING, then umount Lustre"
629 check_mount || return 41
632 echo "read the mode of OBJECTS/LOGS/PENDING and check if they has been changed properly"
633 EXPECTEDOBJECTSMODE=`do_facet mds "debugfs -R 'stat OBJECTS' $MDSDEV 2> /dev/null" | grep 'Mode: ' | sed -e "s/.*Mode: *//" -e "s/ *Flags:.*//"`
634 EXPECTEDLOGSMODE=`do_facet mds "debugfs -R 'stat LOGS' $MDSDEV 2> /dev/null" | grep 'Mode: ' | sed -e "s/.*Mode: *//" -e "s/ *Flags:.*//"`
635 EXPECTEDPENDINGMODE=`do_facet mds "debugfs -R 'stat PENDING' $MDSDEV 2> /dev/null" | grep 'Mode: ' | sed -e "s/.*Mode: *//" -e "s/ *Flags:.*//"`
637 if [ "$EXPECTEDOBJECTSMODE" = "0777" ]; then
638 echo "Success:Lustre change the mode of OBJECTS correctly"
640 echo "Error: Lustre does not change mode of OBJECTS properly"
644 if [ "$EXPECTEDLOGSMODE" = "0777" ]; then
645 echo "Success:Lustre change the mode of LOGS correctly"
647 echo "Error: Lustre does not change mode of LOGS properly"
651 if [ "$EXPECTEDPENDINGMODE" = "0777" ]; then
652 echo "Success:Lustre change the mode of PENDING correctly"
654 echo "Error: Lustre does not change mode of PENDING properly"
658 run_test 16 "verify that lustre will correct the mode of OBJECTS/LOGS/PENDING"
661 if [ ! -f "$MDSDEV" ]; then
662 echo "no $MDSDEV existing, so mount Lustre to create one"
664 check_mount || return 41
668 echo "Remove mds config log"
669 do_facet mds "debugfs -w -R 'unlink CONFIGS/$FSNAME-MDT0000' $MDSDEV || return \$?" || return $?
672 start_mds && return 42
675 run_test 17 "Verify failed mds_postsetup won't fail assertion (2936) (should return errs)"
678 [ -f $MDSDEV ] && echo "remove $MDSDEV" && rm -f $MDSDEV
679 echo "mount mds with large journal..."
682 #FIXME have to change MDS_MKFS_OPTS
685 echo "mount lustre system..."
687 check_mount || return 41
689 echo "check journal size..."
690 FOUNDJOURNALSIZE=`do_facet mds "debugfs -R 'stat <8>' $MDSDEV" | awk '/Size: / { print $NF; exit;}'`
691 if [ "$FOUNDJOURNALSIZE" = "79691776" ]; then
692 echo "Success:lconf creates large journals"
694 echo "Error:lconf not create large journals correctly"
695 echo "expected journal size: 79691776(76M), found journal size: $FOUNDJOURNALSIZE"
704 run_test 18 "check lconf creates large journals"
707 start_mds || return 1
708 stop_mds -f || return 2
710 run_test 19a "start/stop MDS without OSTs"
713 start_ost || return 1
714 stop_ost -f || return 2
716 run_test 19b "start/stop OSTs without MDS"
719 # first format the ost/mdt
723 check_mount || return 43
725 remount_client ro $MOUNT || return 44
726 touch $DIR/$tfile && echo "$DIR/$tfile created incorrectly" && return 45
727 [ -e $DIR/$tfile ] && echo "$DIR/$tfile exists incorrectly" && return 46
728 remount_client rw $MOUNT || return 47
730 [ ! -f $DIR/$tfile ] && echo "$DIR/$tfile missing" && return 48
731 MCNT=`grep -c $MOUNT /etc/mtab`
732 [ "$MCNT" -ne 1 ] && echo "$MOUNT in /etc/mtab $MCNT times" && return 49
737 run_test 20 "remount ro,rw mounts work and doesn't break /etc/mtab"
745 run_test 21a "start mds before ost, stop ost first"
753 run_test 21b "start ost before mds, stop mds first"
763 run_test 21c "start mds between two osts, stop mds last"
766 #reformat to remove all logs
769 echo Client mount before any osts are in the logs
771 check_mount && return 41
774 echo Client mount with ost in logs, but none running
778 # check_mount will block trying to contact ost
782 echo Client mount with a running ost
786 check_mount || return 41
791 run_test 22 "start a client before osts (should return errs)"
797 # force down client so that recovering mds waits for reconnect
798 zconf_umount `hostname` $MOUNT -f
799 # enter recovery on mds
801 # try to start a new client
802 mount_client $MOUNT &
805 MOUNT_LUSTRE_PID=`ps -ef | grep mount.lustre | grep -v grep | awk '{print $2}'`
806 echo mount pid is ${MOUNT_PID}, mount.lustre pid is ${MOUNT_LUSTRE_PID}
808 ps --ppid $MOUNT_LUSTRE_PID
809 # FIXME why o why can't I kill these? Manual "ctrl-c" works...
810 kill -TERM $MOUNT_PID
811 echo "waiting for mount to finish"
818 #this test isn't working yet
819 #run_test 23 "interrupt client during recovery mount delay"
822 local fs2mds_HOST=$mds_HOST
823 # test 8-char fsname as well
824 local FSNAME2=test1234
825 add fs2mds $MDS_MKFS_OPTS --fsname=${FSNAME2} --nomgs --mgsnode=$MGSNID --reformat ${MDSDEV}_2 || exit 10
827 local fs2ost_HOST=$ost_HOST
828 local fs2ostdev=$(ostdevname 1)_2
829 add fs2ost $OST_MKFS_OPTS --fsname=${FSNAME2} --reformat $fs2ostdev || exit 10
832 start fs2mds ${MDSDEV}_2 $MDS_MOUNT_OPTS
833 start fs2ost $fs2ostdev $OST_MOUNT_OPTS
835 mount -t lustre $MGSNID:/${FSNAME2} $MOUNT2 || return 1
837 check_mount || return 2
838 # files written on 1 should not show up on 2
839 cp /etc/passwd $DIR/$tfile
841 [ -e $MOUNT2/$tfile ] && error "File bleed" && return 7
843 cp /etc/passwd $MOUNT2/b || return 3
844 rm $MOUNT2/b || return 4
845 # 2 is actually mounted
846 grep $MOUNT2' ' /proc/mounts > /dev/null || return 5
848 facet_failover fs2mds
849 facet_failover fs2ost
852 # the MDS must remain up until last MDT
854 MDS=$(awk '($3 ~ "mdt" && $4 ~ "MDS") { print $4 }' $LPROC/devices)
855 [ -z "$MDS" ] && error "No MDS" && return 8
859 cleanup_nocli || return 6
861 run_test 24a "Multiple MDTs on a single node"
864 local fs2mds_HOST=$mds_HOST
865 add fs2mds $MDS_MKFS_OPTS --fsname=${FSNAME}2 --mgs --reformat ${MDSDEV}_2 || exit 10
867 start fs2mds ${MDSDEV}_2 $MDS_MOUNT_OPTS && return 2
870 run_test 24b "Multiple MGSs on a single node (should return err)"
874 check_mount || return 2
875 local MODULES=$($LCTL modules | awk '{ print $2 }')
876 rmmod $MODULES 2>/dev/null || true
879 run_test 25 "Verify modules are referenced"
883 # we need modules before mount for sysctl, so make sure...
884 [ -z "$(lsmod | grep lustre)" ] && modprobe lustre
885 #define OBD_FAIL_MDS_FS_SETUP 0x135
886 sysctl -w lustre.fail_loc=0x80000135
887 start_mds && echo MDS started && return 1
889 DEVS=$(cat $LPROC/devices | wc -l)
890 [ $DEVS -gt 0 ] && return 2
891 unload_modules || return 203
893 run_test 26 "MDT startup failure cleans LOV (should return errs)"
899 if [ $# -gt 2 ]; then
905 echo "Setting $PARAM from $ORIG to $FINAL"
906 $LCTL conf_param $PARAM=$FINAL
913 if [ $RESULT -eq $FINAL ]; then
914 echo "Updated config after $WAIT sec (got $RESULT)"
918 if [ $WAIT -eq $MAX ]; then
919 echo "Config update not seen: wanted $FINAL got $RESULT"
922 echo "Waiting $(($MAX - $WAIT)) secs for config update"
927 start_ost || return 1
928 start_mds || return 2
929 echo "Requeue thread should have started: "
930 ps -e | grep ll_cfg_requeue
931 set_and_check "cat $LPROC/obdfilter/$FSNAME-OST0000/client_cache_seconds" "$FSNAME-OST0000.ost.client_cache_seconds" || return 3
934 run_test 27a "Reacquire MGS lock if OST started first"
939 set_and_check "cat $LPROC/mds/$FSNAME-MDT0000/group_acquire_expire" "$FSNAME-MDT0000.mdt.group_acquire_expire" || return 3
940 set_and_check "cat $LPROC/mdc/$FSNAME-MDT0000-mdc-*/max_rpcs_in_flight" "$FSNAME-MDT0000.mdc.max_rpcs_in_flight" || return 4
943 run_test 27b "Reacquire MGS lock after failover"
947 TEST="cat $LPROC/llite/$FSNAME-*/max_read_ahead_whole_mb"
950 FINAL=$(($ORIG + 10))
951 set_and_check "$TEST" "$FSNAME.llite.max_read_ahead_whole_mb" || return 3
952 set_and_check "$TEST" "$FSNAME.llite.max_read_ahead_whole_mb" || return 3
953 umount_client $MOUNT || return 200
956 if [ $RESULT -ne $FINAL ]; then
957 echo "New config not seen: wanted $FINAL got $RESULT"
960 echo "New config success: got $RESULT"
964 run_test 28 "permanent parameter setting"
967 [ "$OSTCOUNT" -lt "2" ] && echo "skipping deactivate test" && return
968 setup > /dev/null 2>&1
972 local PARAM="$FSNAME-OST0001.osc.active"
973 local PROC_ACT="$LPROC/osc/$FSNAME-OST0001-osc-*/active"
974 local PROC_UUID="$LPROC/osc/$FSNAME-OST0001-osc-*/ost_server_uuid"
975 if [ ! -r $PROC_ACT ]; then
976 echo "Can't read $PROC_ACT"
977 ls $LPROC/osc/$FSNAME-*
980 ACTV=$(cat $PROC_ACT)
982 set_and_check "cat $PROC_ACT" "$PARAM" $DEAC || return 2
983 # also check ost_server_uuid status
984 RESULT=$(grep DEACTIV $PROC_UUID)
985 if [ -z "$RESULT" ]; then
986 echo "Live client not deactivated: $(cat $PROC_UUID)"
989 echo "Live client success: got $RESULT"
993 local MPROC="$LPROC/osc/$FSNAME-OST0001-osc/active"
994 if [ -r $MPROC ]; then
996 if [ $RESULT -ne $DEAC ]; then
997 echo "MDT not deactivated: $(cat $MPROC)"
1000 echo "MDT deactivated also"
1003 # test new client starts deactivated
1004 umount_client $MOUNT || return 200
1006 RESULT=$(grep DEACTIV $PROC_UUID | grep NEW)
1007 if [ -z "$RESULT" ]; then
1008 echo "New client not deactivated from start: $(cat $PROC_UUID)"
1011 echo "New client success: got $RESULT"
1014 # make sure it reactivates
1015 set_and_check "cat $PROC_ACT" "$PARAM" $ACTV || return 6
1017 umount_client $MOUNT
1020 #writeconf to remove all ost2 traces for subsequent tests
1023 run_test 29 "permanently remove an OST"
1026 # start mds first after writeconf
1030 TEST="cat $LPROC/llite/$FSNAME-*/max_read_ahead_whole_mb"
1032 for i in $(seq 1 20); do
1033 set_and_check "$TEST" "$FSNAME.llite.max_read_ahead_whole_mb" $i || return 3
1035 # make sure client restart still works
1036 umount_client $MOUNT
1037 mount_client $MOUNT || return 4
1038 [ "$($TEST)" -ne "$i" ] && return 5
1039 set_and_check "$TEST" "$FSNAME.llite.max_read_ahead_whole_mb" $ORIG || return 6
1042 run_test 30 "Big config llog"
1044 test_31() { # bug 10734
1045 # ipaddr must not exist
1046 mount -t lustre 4.3.2.1@tcp:/lustre $MOUNT || true
1049 run_test 31 "Connect to non-existent node (shouldn't crash)"
1052 [ -z "$TUNEFS" ] && echo "No tunefs" && return
1053 [ ! -r disk1_4.zip ] && echo "Cant find disk1_4.zip, skipping" && return
1054 unzip -o -j -d $TMP/$tdir disk1_4.zip || { echo "Cant unzip disk1_4, skipping" && return ; }
1056 sysctl lnet.debug=$PTLDEBUG
1058 $TUNEFS $TMP/$tdir/mds || error "tunefs failed"
1059 # nids are wrong, so client wont work, but server should start
1060 start mds $TMP/$tdir/mds "-o loop" || return 3
1061 local UUID=$(cat $LPROC/mds/lustre-MDT0000/uuid)
1063 [ "$UUID" == "mdsA_UUID" ] || error "UUID is wrong: $UUID"
1065 $TUNEFS --mgsnode=`hostname` $TMP/$tdir/ost1 || error "tunefs failed"
1066 start ost1 $TMP/$tdir/ost1 "-o loop" || return 5
1067 UUID=$(cat $LPROC/obdfilter/lustre-OST0000/uuid)
1069 [ "$UUID" == "ost1_UUID" ] || error "UUID is wrong: $UUID"
1071 local NID=$($LCTL list_nids | head -1)
1073 echo "OSC changes should return err:"
1074 $LCTL conf_param lustre-OST0000.osc.max_dirty_mb=15 && return 7
1075 $LCTL conf_param lustre-OST0000.failover.node=$NID && return 8
1077 echo "MDC changes should succeed:"
1078 $LCTL conf_param lustre-MDT0000.mdc.max_rpcs_in_flight=9 || return 9
1079 $LCTL conf_param lustre-MDT0000.failover.node=$NID || return 10
1082 # With a new good MDT failover nid, we should be able to mount a client
1083 # (but it cant talk to OST)
1085 set_and_check "cat $LPROC/mdc/*/max_rpcs_in_flight" "lustre-MDT0000.mdc.max_rpcs_in_flight" || return 11
1087 zconf_umount `hostname` $MOUNT -f
1090 # mount a second time to make sure we didnt leave upgrade flag on
1091 $TUNEFS --dryrun $TMP/$tdir/mds || error "tunefs failed"
1092 start mds $TMP/$tdir/mds "-o loop" || return 12
1095 [ -d $TMP/$tdir ] && rm -rf $TMP/$tdir
1097 run_test 32a "Upgrade from 1.4 (not live)"
1100 [ -z "$TUNEFS" ] && echo "No tunefs" && return
1101 [ ! -r disk1_4.zip ] && echo "Cant find disk1_4.zip, skipping" && return
1102 unzip -o -j -d $TMP/$tdir disk1_4.zip || { echo "Cant unzip disk1_4, skipping" && return ; }
1104 sysctl lnet.debug=$PTLDEBUG
1106 # writeconf will cause servers to register with their current nids
1107 $TUNEFS --writeconf $TMP/$tdir/mds || error "tunefs failed"
1108 start mds $TMP/$tdir/mds "-o loop" || return 3
1109 local UUID=$(cat $LPROC/mds/lustre-MDT0000/uuid)
1111 [ "$UUID" == "mdsA_UUID" ] || error "UUID is wrong: $UUID"
1113 $TUNEFS --mgsnode=`hostname` $TMP/$tdir/ost1 || error "tunefs failed"
1114 start ost1 $TMP/$tdir/ost1 "-o loop" || return 5
1115 UUID=$(cat $LPROC/obdfilter/lustre-OST0000/uuid)
1117 [ "$UUID" == "ost1_UUID" ] || error "UUID is wrong: $UUID"
1119 echo "OSC changes should succeed:"
1120 $LCTL conf_param lustre-OST0000.osc.max_dirty_mb=15 || return 7
1121 $LCTL conf_param lustre-OST0000.failover.node=$NID || return 8
1123 echo "MDC changes should succeed:"
1124 $LCTL conf_param lustre-MDT0000.mdc.max_rpcs_in_flight=9 || return 9
1127 # MDT and OST should have registered with new nids, so we should have
1128 # a fully-functioning client
1129 echo "Check client and old fs contents"
1131 set_and_check "cat $LPROC/mdc/*/max_rpcs_in_flight" "lustre-MDT0000.mdc.max_rpcs_in_flight" || return 11
1132 [ "$(cksum $MOUNT/passwd | cut -d' ' -f 1,2)" == "2479747619 779" ] || return 12
1136 [ -d $TMP/$tdir ] && rm -rf $TMP/$tdir
1138 run_test 32b "Upgrade from 1.4 with writeconf"
1140 umount_client $MOUNT
1144 echo "$0: completed"