2 # -*- mode: Bash; tab-width: 4; indent-tabs-mode: t; -*-
3 # vim:autoindent:shiftwidth=4:tabstop=4:
5 # FIXME - there is no reason to use all of these different
6 # return codes, espcially when most of them are mapped to something
7 # else anyway. The combination of test number and return code
8 # figure out what failed.
14 # bug number for skipped test:
16 ALWAYS_EXCEPT="$CONF_SANITY_EXCEPT"
17 # UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT!
20 PATH=$PWD/$SRCDIR:$SRCDIR:$SRCDIR/../utils:$PATH
22 PTLDEBUG=${PTLDEBUG:--1}
24 LUSTRE=${LUSTRE:-`dirname $0`/..}
25 RLUSTRE=${RLUSTRE:-$LUSTRE}
27 . $LUSTRE/tests/test-framework.sh
30 # STORED_MDSSIZE is used in test_18
31 if [ -n "$MDSSIZE" ]; then
32 STORED_MDSSIZE=$MDSSIZE
34 # use small MDS + OST size to speed formatting time
35 # do not use too small MDSSIZE/OSTSIZE, which affect the default jouranl size
38 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
40 require_dsh_mds || exit 0
41 require_dsh_ost || exit 0
43 [ "$SLOW" = "no" ] && EXCEPT_SLOW="0 1 2 3 6 7 15 18 24b 25 30 31 32 33 34a 45"
52 local facet=$SINGLEMDS
53 local dev=${facet}_dev
57 # who knows if/where $TUNEFS is installed? Better reformat if it fails...
58 do_facet ${facet} "$TUNEFS --writeconf ${!dev}" ||
59 echo "tunefs failed, reformatting instead" && reformat_and_config
64 # The MGS must be started before the OSTs for a new fs, so start
65 # and stop to generate the startup logs.
68 wait_osc_import_state mds ost FULL
73 reformat_and_config() {
75 if ! combined_mgs_mds ; then
83 start mgs $MGSDEV $mgs_MOUNT_OPTS
87 local facet=$SINGLEMDS
88 # we can not use MDSDEV1 here because SINGLEMDS could be set not to mds1 only
89 local num=$(echo $facet | tr -d "mds")
90 local dev=$(mdsdevname $num)
91 echo "start mds service on `facet_active_host $facet`"
92 start $facet ${dev} $MDS_MOUNT_OPTS || return 94
96 echo "stop mds service on `facet_active_host $SINGLEMDS`"
97 # These tests all use non-failover stop
98 stop $SINGLEMDS -f || return 97
102 echo "start ost1 service on `facet_active_host ost1`"
103 start ost1 `ostdevname 1` $OST_MOUNT_OPTS || return 95
107 echo "stop ost1 service on `facet_active_host ost1`"
108 # These tests all use non-failover stop
109 stop ost1 -f || return 98
113 echo "start ost2 service on `facet_active_host ost2`"
114 start ost2 `ostdevname 2` $OST_MOUNT_OPTS || return 92
118 echo "stop ost2 service on `facet_active_host ost2`"
119 # These tests all use non-failover stop
120 stop ost2 -f || return 93
125 echo "mount $FSNAME on ${MOUNTPATH}....."
126 zconf_mount `hostname` $MOUNTPATH || return 96
130 local SAVEMOUNTOPT=$MOUNTOPT
131 MOUNTOPT="remount,$1"
133 echo "remount '$1' lustre on ${MOUNTPATH}....."
134 zconf_mount `hostname` $MOUNTPATH || return 96
135 MOUNTOPT=$SAVEMOUNTOPT
140 echo "umount lustre on ${MOUNTPATH}....."
141 zconf_umount `hostname` $MOUNTPATH || return 97
144 manual_umount_client(){
147 echo "manual umount lustre on ${MOUNT}...."
148 do_facet client "umount -d ${FORCE} $MOUNT"
160 if ! combined_mgs_mds ; then
170 stop_mds || return 201
171 stop_ost || return 202
172 unload_modules || return 203
176 umount_client $MOUNT || return 200
177 cleanup_nocli || return $?
181 do_facet client "cp /etc/passwd $DIR/a" || return 71
182 do_facet client "rm $DIR/a" || return 72
183 # make sure lustre is actually mounted (touch will block,
184 # but grep won't, so do it after)
185 do_facet client "grep $MOUNT' ' /proc/mounts > /dev/null" || return 73
186 echo "setup single mount lustre success"
190 do_facet client "touch $DIR/a" || return 71
191 do_facet client "rm $DIR/a" || return 72
192 do_facet client "touch $DIR2/a" || return 73
193 do_facet client "rm $DIR2/a" || return 74
194 echo "setup double mount lustre success"
199 if [ "$ONLY" == "setup" ]; then
204 if [ "$ONLY" == "cleanup" ]; then
211 #create single point mountpoint
217 check_mount || return 41
220 run_test 0 "single mount setup"
224 echo "start ost second time..."
226 check_mount || return 42
229 run_test 1 "start up ost twice (should return errors)"
234 echo "start mds second time.."
237 check_mount || return 43
240 run_test 2 "start up mds twice (should return err)"
244 #mount.lustre returns an error if already in mtab
245 mount_client $MOUNT && return $?
246 check_mount || return 44
249 run_test 3 "mount client twice (should return err)"
253 touch $DIR/$tfile || return 85
257 # ok for ost to fail shutdown
258 if [ 202 -ne $eno ]; then
263 run_test 4 "force cleanup ost, then cleanup"
265 test_5a() { # was test_5
267 touch $DIR/$tfile || return 1
268 fuser -m -v $MOUNT && echo "$MOUNT is in use by user space process."
270 stop_mds -f || return 2
272 # cleanup may return an error from the failed
273 # disconnects; for now I'll consider this successful
274 # if all the modules have unloaded.
278 echo "killing umount"
279 kill -TERM $UMOUNT_PID
280 echo "waiting for umount to finish"
282 if grep " $MOUNT " /proc/mounts; then
283 echo "test 5: /proc/mounts after failed umount"
287 echo "killing umount"
288 kill -TERM $UMOUNT_PID
289 echo "waiting for umount to finish"
291 grep " $MOUNT " /proc/mounts && echo "test 5: /proc/mounts after second umount" && return 11
295 # stop_mds is a no-op here, and should not fail
296 cleanup_nocli || return $?
297 # df may have lingering entry
299 # mtab may have lingering entry
303 while [ "$WAIT" -ne "$MAX_WAIT" ]; do
305 grep -q $MOUNT" " /etc/mtab || break
306 echo "Waiting /etc/mtab updated ... "
307 WAIT=$(( WAIT + sleep))
309 [ "$WAIT" -eq "$MAX_WAIT" ] && error "/etc/mtab is not updated in $WAIT secs"
310 echo "/etc/mtab updated in $WAIT secs"
312 run_test 5a "force cleanup mds, then cleanup"
316 [ -d $MOUNT ] || mkdir -p $MOUNT
317 grep " $MOUNT " /etc/mtab && echo "test 5b: mtab before mount" && return 10
318 mount_client $MOUNT && return 1
319 grep " $MOUNT " /etc/mtab && echo "test 5b: mtab after failed mount" && return 11
321 # stop_mds is a no-op here, and should not fail
322 cleanup_nocli || return $?
325 run_test 5b "mds down, cleanup after failed mount (bug 2712) (should return errs)"
330 [ -d $MOUNT ] || mkdir -p $MOUNT
331 grep " $MOUNT " /etc/mtab && echo "test 5c: mtab before mount" && return 10
332 local oldfs="${FSNAME}"
333 FSNAME="wrong.${FSNAME}"
334 mount_client $MOUNT || :
336 grep " $MOUNT " /etc/mtab && echo "test 5c: mtab after failed mount" && return 11
338 cleanup_nocli || return $?
340 run_test 5c "cleanup after failed mount (bug 2712) (should return errs)"
346 grep " $MOUNT " /etc/mtab && echo "test 5d: mtab before mount" && return 10
347 mount_client $MOUNT || return 1
349 grep " $MOUNT " /etc/mtab && echo "test 5d: mtab after unmount" && return 11
352 run_test 5d "mount with ost down"
358 #define OBD_FAIL_PTLRPC_DELAY_SEND 0x506
359 do_facet client "lctl set_param fail_loc=0x80000506"
360 grep " $MOUNT " /etc/mtab && echo "test 5e: mtab before mount" && return 10
361 mount_client $MOUNT || echo "mount failed (not fatal)"
363 grep " $MOUNT " /etc/mtab && echo "test 5e: mtab after unmount" && return 11
366 run_test 5e "delayed connect, don't crash (bug 10268)"
371 mount_client ${MOUNT} || return 87
372 touch $DIR/a || return 86
375 run_test 6 "manual umount, then mount again"
380 cleanup_nocli || return $?
382 run_test 7 "manual umount, then cleanup"
387 check_mount2 || return 45
388 umount_client $MOUNT2
391 run_test 8 "double mount setup"
396 do_facet ost1 lctl set_param debug=\'inode trace\' || return 1
397 do_facet ost1 lctl set_param subsystem_debug=\'mds ost\' || return 1
399 CHECK_PTLDEBUG="`do_facet ost1 lctl get_param -n debug`"
400 if [ "$CHECK_PTLDEBUG" ] && { \
401 [ "$CHECK_PTLDEBUG" = "trace inode warning error emerg console" ] ||
402 [ "$CHECK_PTLDEBUG" = "trace inode" ]; }; then
403 echo "lnet.debug success"
405 echo "lnet.debug: want 'trace inode', have '$CHECK_PTLDEBUG'"
408 CHECK_SUBSYS="`do_facet ost1 lctl get_param -n subsystem_debug`"
409 if [ "$CHECK_SUBSYS" ] && [ "$CHECK_SUBSYS" = "mds ost" ]; then
410 echo "lnet.subsystem_debug success"
412 echo "lnet.subsystem_debug: want 'mds ost', have '$CHECK_SUBSYS'"
415 stop_ost || return $?
418 run_test 9 "test ptldebug and subsystem for mkfs"
420 # LOGS/PENDING do not exist anymore since CMD3
422 local TMPMTPT="${TMP}/conf16"
423 local dev=${SINGLEMDS}_dev
425 if [ ! -e "$MDSDEV" ]; then
426 log "no $MDSDEV existing, so mount Lustre to create one"
428 check_mount || return 41
432 [ -f "$MDSDEV" ] && LOOPOPT="-o loop"
434 log "change the mode of $MDSDEV/OBJECTS to 555"
435 do_facet $SINGLEMDS "mkdir -p $TMPMTPT &&
436 mount $LOOPOPT -t $FSTYPE $MDSDEV $TMPMTPT &&
437 chmod 555 $TMPMTPT/OBJECTS &&
438 umount $TMPMTPT" || return $?
440 log "mount Lustre to change the mode of OBJECTS, then umount Lustre"
442 check_mount || return 41
445 log "read the mode of OBJECTS and check if they has been changed properly"
446 EXPECTEDOBJECTSMODE=`do_facet $SINGLEMDS "$DEBUGFS -R 'stat OBJECTS' $MDSDEV 2> /dev/null" | grep 'Mode: ' | sed -e "s/.*Mode: *//" -e "s/ *Flags:.*//"`
448 if [ "$EXPECTEDOBJECTSMODE" = "0777" ]; then
449 log "Success:Lustre change the mode of OBJECTS correctly"
451 error "Lustre does not change mode of OBJECTS properly"
454 run_test 16 "verify that lustre will correct the mode of OBJECTS"
457 local dev=${SINGLEMDS}_dev
460 if [ ! -e "$MDSDEV" ]; then
461 echo "no $MDSDEV existing, so mount Lustre to create one"
463 check_mount || return 41
467 echo "Remove mds config log"
468 do_facet $SINGLEMDS "$DEBUGFS -w -R 'unlink CONFIGS/$FSNAME-MDT0000' $MDSDEV || return \$?" || return $?
471 start_mds && return 42
474 run_test 17 "Verify failed mds_postsetup won't fail assertion (2936) (should return errs)"
477 [ "$FSTYPE" != "ldiskfs" ] && skip "not needed for FSTYPE=$FSTYPE" && return
479 local dev=${SINGLEMDS}_dev
485 # check if current MDSSIZE is large enough
486 [ $MDSSIZE -ge $MIN ] && OK=1 && myMDSSIZE=$MDSSIZE && \
487 log "use MDSSIZE=$MDSSIZE"
489 # check if the global config has a large enough MDSSIZE
490 [ -z "$OK" -a ! -z "$STORED_MDSSIZE" ] && [ $STORED_MDSSIZE -ge $MIN ] && \
491 OK=1 && myMDSSIZE=$STORED_MDSSIZE && \
492 log "use STORED_MDSSIZE=$STORED_MDSSIZE"
494 # check if the block device is large enough
495 [ -z "$OK" -a -b $MDSDEV ] && \
496 [ "$(dd if=$MDSDEV of=/dev/null bs=1k count=1 skip=$MIN 2>&1 |
497 awk '($3 == "in") { print $1 }')" = "1+0" ] && OK=1 && \
498 myMDSSIZE=$MIN && log "use device $MDSDEV with MIN=$MIN"
500 # check if a loopback device has enough space for fs metadata (5%)
501 [ -z "$OK" ] && [ -f $MDSDEV -o ! -e $MDSDEV ] &&
502 SPACE=$(df -P $(dirname $MDSDEV) |
503 awk '($1 != "Filesystem") {print $4}') &&
504 [ $SPACE -gt $((MIN / 20)) ] && OK=1 && myMDSSIZE=$MIN && \
505 log "use file $MDSDEV with MIN=$MIN"
507 [ -z "$OK" ] && skip_env "$MDSDEV too small for ${MIN}kB MDS" && return
510 echo "mount mds with large journal..."
511 local OLD_MDS_MKFS_OPTS=$MDS_MKFS_OPTS
513 MDS_MKFS_OPTS="--mgs --mdt --fsname=$FSNAME --device-size=$myMDSSIZE --param sys.timeout=$TIMEOUT $MDSOPT"
516 echo "mount lustre system..."
518 check_mount || return 41
520 echo "check journal size..."
521 local FOUNDSIZE=`do_facet mds "$DEBUGFS -c -R 'stat <8>' $MDSDEV" | awk '/Size: / { print $NF; exit;}'`
522 if [ $FOUNDSIZE -gt $((32 * 1024 * 1024)) ]; then
523 log "Success: mkfs creates large journals. Size: $((FOUNDSIZE >> 20))M"
525 error "expected journal size > 32M, found $((FOUNDSIZE >> 20))M"
530 MDS_MKFS_OPTS=$OLD_MDS_MKFS_OPTS
533 run_test 18 "check mkfs creates large journals"
536 start_mds || return 1
537 stop_mds -f || return 2
539 run_test 19a "start/stop MDS without OSTs"
542 start_ost || return 1
543 stop_ost -f || return 2
545 run_test 19b "start/stop OSTs without MDS"
548 # first format the ost/mdt
552 check_mount || return 43
554 remount_client ro $MOUNT || return 44
555 touch $DIR/$tfile && echo "$DIR/$tfile created incorrectly" && return 45
556 [ -e $DIR/$tfile ] && echo "$DIR/$tfile exists incorrectly" && return 46
557 remount_client rw $MOUNT || return 47
559 [ ! -f $DIR/$tfile ] && echo "$DIR/$tfile missing" && return 48
560 MCNT=`grep -c $MOUNT /etc/mtab`
561 [ "$MCNT" -ne 1 ] && echo "$MOUNT in /etc/mtab $MCNT times" && return 49
566 run_test 20 "remount ro,rw mounts work and doesn't break /etc/mtab"
571 wait_osc_import_state mds ost FULL
575 run_test 21a "start mds before ost, stop ost first"
580 wait_osc_import_state mds ost FULL
584 run_test 21b "start ost before mds, stop mds first"
590 wait_osc_import_state mds ost2 FULL
594 #writeconf to remove all ost2 traces for subsequent tests
597 run_test 21c "start mds between two osts, stop mds last"
602 echo Client mount with ost in logs, but none running
604 # wait until mds connected to ost and open client connection
605 wait_osc_import_state mds ost FULL
608 # check_mount will block trying to contact ost
609 mcreate $DIR/$tfile || return 40
610 rm -f $DIR/$tfile || return 42
614 echo Client mount with a running ost
617 # if gss enabled, wait full time to let connection from
618 # mds to ost be established, due to the mismatch between
619 # initial connect timeout and gss context negotiation timeout.
620 # This perhaps could be remove after AT landed.
621 echo "sleep $((TIMEOUT + TIMEOUT + TIMEOUT))s"
622 sleep $((TIMEOUT + TIMEOUT + TIMEOUT))
625 check_mount || return 41
630 run_test 22 "start a client before osts (should return errs)"
632 test_23a() { # was test_23
636 # force down client so that recovering mds waits for reconnect
637 local running=$(grep -c $MOUNT /proc/mounts) || true
638 if [ $running -ne 0 ]; then
639 echo "Stopping client $MOUNT (opts: -f)"
643 # enter recovery on mds
645 # try to start a new client
646 mount_client $MOUNT &
648 MOUNT_PID=$(ps -ef | grep "t lustre" | grep -v grep | awk '{print $2}')
649 MOUNT_LUSTRE_PID=`ps -ef | grep mount.lustre | grep -v grep | awk '{print $2}'`
650 echo mount pid is ${MOUNT_PID}, mount.lustre pid is ${MOUNT_LUSTRE_PID}
652 ps --ppid $MOUNT_LUSTRE_PID
653 # FIXME why o why can't I kill these? Manual "ctrl-c" works...
654 kill -TERM $MOUNT_LUSTRE_PID
655 echo "waiting for mount to finish"
657 # we can not wait $MOUNT_PID because it is not a child of this shell
663 while [ "$WAIT" -lt "$MAX_WAIT" ]; do
665 PID1=$(ps -ef | awk '{print $2}' | grep -w $MOUNT_PID)
666 PID2=$(ps -ef | awk '{print $2}' | grep -w $MOUNT_LUSTRE_PID)
669 [ -z "$PID1" -a -z "$PID2" ] && break
670 echo "waiting for mount to finish ... "
671 WAIT=$(( WAIT + sleep))
673 if [ "$WAIT" -eq "$MAX_WAIT" ]; then
674 error "MOUNT_PID $MOUNT_PID and "\
675 "MOUNT_LUSTRE_PID $MOUNT_LUSTRE_PID still not killed in $WAIT secs"
681 run_test 23a "interrupt client during recovery mount delay"
686 test_23b() { # was test_23
689 # Simulate -EINTR during mount OBD_FAIL_LDLM_CLOSE_THREAD
690 lctl set_param fail_loc=0x80000313
694 run_test 23b "Simulate -EINTR during mount"
696 fs2mds_HOST=$mds_HOST
697 fs2ost_HOST=$ost_HOST
701 echo "umount $MOUNT2 ..."
702 umount $MOUNT2 || true
703 echo "stopping fs2mds ..."
704 stop fs2mds -f || true
705 echo "stopping fs2ost ..."
706 stop fs2ost -f || true
713 [ -n "$ost1_HOST" ] && fs2ost_HOST=$ost1_HOST
714 if [ -z "$fs2ost_DEV" -o -z "$fs2mds_DEV" ]; then
715 do_facet $SINGLEMDS [ -b "$MDSDEV" ] && \
716 skip_env "mixed loopback and real device not working" && return
719 local fs2mdsdev=${fs2mds_DEV:-${MDSDEV}_2}
720 local fs2ostdev=${fs2ost_DEV:-$(ostdevname 1)_2}
722 # test 8-char fsname as well
723 local FSNAME2=test1234
724 add fs2mds $MDS_MKFS_OPTS --fsname=${FSNAME2} --nomgs --mgsnode=$MGSNID --reformat $fs2mdsdev || exit 10
726 add fs2ost $OST_MKFS_OPTS --fsname=${FSNAME2} --reformat $fs2ostdev || exit 10
729 start fs2mds $fs2mdsdev $MDS_MOUNT_OPTS && trap cleanup_24a EXIT INT
730 start fs2ost $fs2ostdev $OST_MOUNT_OPTS
732 mount -t lustre $MGSNID:/${FSNAME2} $MOUNT2 || return 1
734 check_mount || return 2
735 # files written on 1 should not show up on 2
736 cp /etc/passwd $DIR/$tfile
738 [ -e $MOUNT2/$tfile ] && error "File bleed" && return 7
741 cp /etc/passwd $MOUNT2/b || return 3
742 rm $MOUNT2/b || return 4
743 # 2 is actually mounted
744 grep $MOUNT2' ' /proc/mounts > /dev/null || return 5
746 facet_failover fs2mds
747 facet_failover fs2ost
750 # the MDS must remain up until last MDT
752 MDS=$(do_facet $SINGLEMDS "lctl get_param -n devices" | awk '($3 ~ "mdt" && $4 ~ "MDT") { print $4 }' | head -1)
753 [ -z "$MDS" ] && error "No MDT" && return 8
755 cleanup_nocli || return 6
757 run_test 24a "Multiple MDTs on a single node"
760 if [ -z "$fs2mds_DEV" ]; then
761 do_facet $SINGLEMDS [ -b "$MDSDEV" ] && \
762 skip_env "mixed loopback and real device not working" && return
765 local fs2mdsdev=${fs2mds_DEV:-${MDSDEV}_2}
767 add fs2mds $MDS_MKFS_OPTS --fsname=${FSNAME}2 --mgs --reformat $fs2mdsdev || exit 10
769 start fs2mds $fs2mdsdev $MDS_MOUNT_OPTS && return 2
772 run_test 24b "Multiple MGSs on a single node (should return err)"
776 check_mount || return 2
777 local MODULES=$($LCTL modules | awk '{ print $2 }')
778 rmmod $MODULES 2>/dev/null || true
781 run_test 25 "Verify modules are referenced"
785 # we need modules before mount for sysctl, so make sure...
786 do_facet $SINGLEMDS "lsmod | grep -q lustre || modprobe lustre"
787 #define OBD_FAIL_MDS_FS_SETUP 0x135
788 do_facet $SINGLEMDS "lctl set_param fail_loc=0x80000135"
789 start_mds && echo MDS started && return 1
790 lctl get_param -n devices
791 DEVS=$(lctl get_param -n devices | wc -l)
792 [ $DEVS -gt 0 ] && return 2
793 unload_modules || return 203
795 run_test 26 "MDT startup failure cleans LOV (should return errs)"
801 local ORIG=$(do_facet $myfacet "$TEST")
802 if [ $# -gt 3 ]; then
808 echo "Setting $PARAM from $ORIG to $FINAL"
809 do_facet $SINGLEMDS "$LCTL conf_param $PARAM='$FINAL'" || error conf_param failed
811 wait_update $(facet_host $myfacet) "$TEST" "$FINAL" || error check failed!
815 start_ost || return 1
816 start_mds || return 2
817 echo "Requeue thread should have started: "
818 ps -e | grep ll_cfg_requeue
819 set_and_check ost1 "lctl get_param -n obdfilter.$FSNAME-OST0000.client_cache_seconds" "$FSNAME-OST0000.ost.client_cache_seconds" || return 3
822 run_test 27a "Reacquire MGS lock if OST started first"
827 local device=$(do_facet $SINGLEMDS "lctl get_param -n devices" | awk '($3 ~ "mdt" && $4 ~ "MDT") { print $4 }')
829 facet_failover $SINGLEMDS
830 set_and_check $SINGLEMDS "lctl get_param -n mdt.$device.identity_acquire_expire" "$device.mdt.identity_acquire_expire" || return 3
831 set_and_check client "lctl get_param -n mdc.$device-mdc-*.max_rpcs_in_flight" "$device.mdc.max_rpcs_in_flight" || return 4
835 run_test 27b "Reacquire MGS lock after failover"
839 TEST="lctl get_param -n llite.$FSNAME-*.max_read_ahead_whole_mb"
840 PARAM="$FSNAME.llite.max_read_ahead_whole_mb"
843 set_and_check client "$TEST" "$PARAM" $FINAL || return 3
844 FINAL=$(($FINAL + 1))
845 set_and_check client "$TEST" "$PARAM" $FINAL || return 4
846 umount_client $MOUNT || return 200
849 if [ $RESULT -ne $FINAL ]; then
850 echo "New config not seen: wanted $FINAL got $RESULT"
853 echo "New config success: got $RESULT"
855 set_and_check client "$TEST" "$PARAM" $ORIG || return 5
858 run_test 28 "permanent parameter setting"
861 [ "$OSTCOUNT" -lt "2" ] && skip_env "$OSTCOUNT < 2, skipping" && return
862 setup > /dev/null 2>&1
866 local PARAM="$FSNAME-OST0001.osc.active"
867 local PROC_ACT="osc.$FSNAME-OST0001-osc-[^M]*.active"
868 local PROC_UUID="osc.$FSNAME-OST0001-osc-[^M]*.ost_server_uuid"
870 ACTV=$(lctl get_param -n $PROC_ACT)
872 set_and_check client "lctl get_param -n $PROC_ACT" "$PARAM" $DEAC || return 2
873 # also check ost_server_uuid status
874 RESULT=$(lctl get_param -n $PROC_UUID | grep DEACTIV)
875 if [ -z "$RESULT" ]; then
876 echo "Live client not deactivated: $(lctl get_param -n $PROC_UUID)"
879 echo "Live client success: got $RESULT"
883 local MPROC="osc.$FSNAME-OST0001-osc-[M]*.active"
888 RESULT=`do_facet $SINGLEMDS " lctl get_param -n $MPROC"`
889 [ ${PIPESTATUS[0]} = 0 ] || error "Can't read $MPROC"
890 if [ $RESULT -eq $DEAC ]; then
891 echo "MDT deactivated also after $WAIT sec (got $RESULT)"
895 if [ $WAIT -eq $MAX ]; then
896 echo "MDT not deactivated: wanted $DEAC got $RESULT"
899 echo "Waiting $(($MAX - $WAIT)) secs for MDT deactivated"
902 # test new client starts deactivated
903 umount_client $MOUNT || return 200
905 RESULT=$(lctl get_param -n $PROC_UUID | grep DEACTIV | grep NEW)
906 if [ -z "$RESULT" ]; then
907 echo "New client not deactivated from start: $(lctl get_param -n $PROC_UUID)"
910 echo "New client success: got $RESULT"
913 # make sure it reactivates
914 set_and_check client "lctl get_param -n $PROC_ACT" "$PARAM" $ACTV || return 6
919 #writeconf to remove all ost2 traces for subsequent tests
922 run_test 29 "permanently remove an OST"
927 TEST="lctl get_param -n llite.$FSNAME-*.max_read_ahead_whole_mb"
929 LIST=(1 2 3 4 5 4 3 2 1 2 3 4 5 4 3 2 1 2 3 4 5)
930 for i in ${LIST[@]}; do
931 set_and_check client "$TEST" "$FSNAME.llite.max_read_ahead_whole_mb" $i || return 3
933 # make sure client restart still works
935 mount_client $MOUNT || return 4
936 [ "$($TEST)" -ne "$i" ] && return 5
937 set_and_check client "$TEST" "$FSNAME.llite.max_read_ahead_whole_mb" $ORIG || return 6
940 run_test 30 "Big config llog"
942 test_31() { # bug 10734
943 # ipaddr must not exist
944 mount -t lustre 4.3.2.1@tcp:/lustre $MOUNT || true
947 run_test 31 "Connect to non-existent node (shouldn't crash)"
949 # Use these start32/stop32 fn instead of t-f start/stop fn,
950 # for local devices, to skip global facet vars init
954 echo "Stopping local ${MOUNT%/*}/${facet} (opts:$@)"
955 umount -d $@ ${MOUNT%/*}/${facet}
964 mkdir -p ${MOUNT%/*}/${facet}
966 echo "Starting local ${facet}: $@ $device ${MOUNT%/*}/${facet}"
967 mount -t lustre $@ ${device} ${MOUNT%/*}/${facet}
969 if [ $RC -ne 0 ]; then
970 echo "mount -t lustre $@ ${device} ${MOUNT%/*}/${facet}"
971 echo "Start of ${device} of local ${facet} failed ${RC}"
985 echo "Cleanup test_32 umount $MOUNT ..."
986 umount -f $MOUNT || true
987 echo "Cleanup local mds ost1 ..."
993 client_only && skip "client only testing" && return 0
994 [ "$NETTYPE" = "tcp" ] || { skip "NETTYPE != tcp" && return 0; }
995 [ -z "$TUNEFS" ] && skip_env "No tunefs" && return 0
997 local DISK1_8=$LUSTRE/tests/disk1_8.tar.bz2
998 [ ! -r $DISK1_8 ] && skip_env "Cannot find $DISK1_8" && return 0
999 local tmpdir=$TMP/conf32a
1001 tar xjvf $DISK1_8 -C $tmpdir || \
1002 { skip_env "Cannot untar $DISK1_8" && return 0; }
1005 $LCTL set_param debug=$PTLDEBUG
1007 $TUNEFS $tmpdir/mds || error "tunefs failed"
1009 # nids are wrong, so client wont work, but server should start
1010 start32 mds1 $tmpdir/mds "-o loop,exclude=lustre-OST0000" && \
1011 trap cleanup_32 EXIT INT || return 3
1013 local UUID=$($LCTL get_param -n mdt.lustre-MDT0000.uuid)
1015 [ "$UUID" == "lustre-MDT0000_UUID" ] || error "UUID is wrong: $UUID"
1017 $TUNEFS --mgsnode=$HOSTNAME $tmpdir/ost1 || error "tunefs failed"
1018 start32 ost1 $tmpdir/ost1 "-o loop" || return 5
1019 UUID=$($LCTL get_param -n obdfilter.lustre-OST0000.uuid)
1021 [ "$UUID" == "lustre-OST0000_UUID" ] || error "UUID is wrong: $UUID"
1023 local NID=$($LCTL list_nids | head -1)
1025 echo "OSC changes should succeed:"
1026 $LCTL conf_param lustre-OST0000.osc.max_dirty_mb=15 || return 7
1027 $LCTL conf_param lustre-OST0000.failover.node=$NID || return 8
1030 echo "MDC changes should succeed:"
1031 $LCTL conf_param lustre-MDT0000.mdc.max_rpcs_in_flight=9 || return 9
1032 $LCTL conf_param lustre-MDT0000.failover.node=$NID || return 10
1035 echo "LOV changes should succeed:"
1036 $LCTL pool_new lustre.interop || return 11
1037 $LCTL conf_param lustre-MDT0000.lov.stripesize=4M || return 12
1042 # mount a second time to make sure we didnt leave upgrade flag on
1044 $TUNEFS --dryrun $tmpdir/mds || error "tunefs failed"
1045 start32 mds1 $tmpdir/mds "-o loop,exclude=lustre-OST0000" && \
1046 trap cleanup_32 EXIT INT || return 12
1050 rm -rf $tmpdir || true # true is only for TMP on NFS
1052 run_test 32a "Upgrade from 1.8 (not live)"
1055 client_only && skip "client only testing" && return 0
1056 [ "$NETTYPE" = "tcp" ] || { skip "NETTYPE != tcp" && return 0; }
1057 [ -z "$TUNEFS" ] && skip_env "No tunefs" && return
1059 local DISK1_8=$LUSTRE/tests/disk1_8.tar.bz2
1060 [ ! -r $DISK1_8 ] && skip_env "Cannot find $DISK1_8" && return 0
1061 local tmpdir=$TMP/conf32b
1063 tar xjvf $DISK1_8 -C $tmpdir || \
1064 { skip_env "Cannot untar $DISK1_8" && return ; }
1067 $LCTL set_param debug="config"
1068 local NEWNAME=lustre
1070 # writeconf will cause servers to register with their current nids
1071 $TUNEFS --writeconf --fsname=$NEWNAME $tmpdir/mds || error "tunefs failed"
1072 start32 mds1 $tmpdir/mds "-o loop" && \
1073 trap cleanup_32 EXIT INT || return 3
1075 local UUID=$($LCTL get_param -n mdt.${NEWNAME}-MDT0000.uuid)
1077 [ "$UUID" == "${NEWNAME}-MDT0000_UUID" ] || error "UUID is wrong: $UUID"
1079 $TUNEFS --mgsnode=$HOSTNAME --writeconf --fsname=$NEWNAME $tmpdir/ost1 ||\
1080 error "tunefs failed"
1081 start32 ost1 $tmpdir/ost1 "-o loop" || return 5
1082 UUID=$($LCTL get_param -n obdfilter.${NEWNAME}-OST0000.uuid)
1084 [ "$UUID" == "${NEWNAME}-OST0000_UUID" ] || error "UUID is wrong: $UUID"
1086 local NID=$($LCTL list_nids | head -1)
1088 echo "OSC changes should succeed:"
1089 $LCTL conf_param ${NEWNAME}-OST0000.osc.max_dirty_mb=15 || return 7
1090 $LCTL conf_param ${NEWNAME}-OST0000.failover.node=$NID || return 8
1093 echo "MDC changes should succeed:"
1094 $LCTL conf_param ${NEWNAME}-MDT0000.mdc.max_rpcs_in_flight=9 || return 9
1095 $LCTL conf_param ${NEWNAME}-MDT0000.failover.node=$NID || return 10
1098 echo "LOV changes should succeed:"
1099 $LCTL pool_new ${NEWNAME}.interop || return 11
1100 $LCTL conf_param ${NEWNAME}-MDT0000.lov.stripesize=4M || return 12
1103 # MDT and OST should have registered with new nids, so we should have
1104 # a fully-functioning client
1105 echo "Check client and old fs contents"
1107 local device=`h2$NETTYPE $HOSTNAME`:/$NEWNAME
1108 echo "Starting local client: $HOSTNAME: $device $MOUNT"
1109 mount -t lustre $device $MOUNT || return 1
1111 local old=$($LCTL get_param -n mdc.*.max_rpcs_in_flight)
1112 local new=$((old + 5))
1113 $LCTL conf_param ${NEWNAME}-MDT0000.mdc.max_rpcs_in_flight=$new
1114 wait_update $HOSTNAME "$LCTL get_param -n mdc.*.max_rpcs_in_flight" $new || return 11
1116 [ "$(cksum $MOUNT/passwd | cut -d' ' -f 1,2)" == "94306271 1478" ] || return 12
1121 rm -rf $tmpdir || true # true is only for TMP on NFS
1123 run_test 32b "Upgrade from 1.8 with writeconf"
1125 test_33a() { # bug 12333, was test_33
1127 local FSNAME2=test-123
1128 [ -n "$ost1_HOST" ] && fs2ost_HOST=$ost1_HOST
1130 if [ -z "$fs2ost_DEV" -o -z "$fs2mds_DEV" ]; then
1131 do_facet $SINGLEMDS [ -b "$MDSDEV" ] && \
1132 skip_env "mixed loopback and real device not working" && return
1135 local fs2mdsdev=${fs2mds_DEV:-${MDSDEV}_2}
1136 local fs2ostdev=${fs2ost_DEV:-$(ostdevname 1)_2}
1137 add fs2mds $MDS_MKFS_OPTS --mkfsoptions='\"-J size=8\"' --fsname=${FSNAME2} --reformat $fs2mdsdev || exit 10
1138 add fs2ost $OST_MKFS_OPTS --fsname=${FSNAME2} --index=8191 --mgsnode=$MGSNID --reformat $fs2ostdev || exit 10
1140 start fs2mds $fs2mdsdev $MDS_MOUNT_OPTS && trap cleanup_24a EXIT INT
1141 start fs2ost $fs2ostdev $OST_MOUNT_OPTS
1142 do_facet $SINGLEMDS "$LCTL conf_param $FSNAME2.sys.timeout=200" || rc=1
1144 mount -t lustre $MGSNID:/${FSNAME2} $MOUNT2 || rc=2
1147 cp /etc/hosts $MOUNT2/ || rc=3
1148 $LFS getstripe $MOUNT2/hosts
1153 rm -rf $MOUNT2 $fs2mdsdev $fs2ostdev
1154 cleanup_nocli || rc=6
1157 run_test 33a "Mount ost with a large index number"
1159 test_33b() { # was test_34
1162 do_facet client dd if=/dev/zero of=$MOUNT/24 bs=1024k count=1
1163 # Drop lock cancelation reply during umount
1164 #define OBD_FAIL_LDLM_CANCEL 0x304
1165 do_facet client lctl set_param fail_loc=0x80000304
1166 #lctl set_param debug=-1
1167 umount_client $MOUNT
1170 run_test 33b "Drop cancel during umount"
1174 do_facet client "sh runmultiop_bg_pause $DIR/file O_c"
1175 manual_umount_client
1177 do_facet client killall -USR1 multiop
1178 if [ $rc -eq 0 ]; then
1179 error "umount not fail!"
1184 run_test 34a "umount with opened file should be fail"
1189 touch $DIR/$tfile || return 1
1190 stop_mds --force || return 2
1192 manual_umount_client --force
1194 if [ $rc -ne 0 ]; then
1195 error "mtab after failed umount - rc $rc"
1201 run_test 34b "force umount with failed mds should be normal"
1205 touch $DIR/$tfile || return 1
1206 stop_ost --force || return 2
1208 manual_umount_client --force
1210 if [ $rc -ne 0 ]; then
1211 error "mtab after failed umount - rc $rc"
1217 run_test 34c "force umount with failed ost should be normal"
1219 test_35a() { # bug 12459
1222 DBG_SAVE="`lctl get_param -n debug`"
1223 lctl set_param debug="ha"
1225 log "Set up a fake failnode for the MDS"
1227 local device=$(do_facet $SINGLEMDS "lctl get_param -n devices" | awk '($3 ~ "mdt" && $4 ~ "MDT") { print $4 }' | head -1)
1228 do_facet $SINGLEMDS $LCTL conf_param ${device}.failover.node=$FAKENID || return 4
1230 log "Wait for RECONNECT_INTERVAL seconds (10s)"
1233 MSG="conf-sanity.sh test_35a `date +%F%kh%Mm%Ss`"
1236 log "Stopping the MDT:"
1237 stop_mds || return 5
1239 df $MOUNT > /dev/null 2>&1 &
1241 log "Restarting the MDT:"
1242 start_mds || return 6
1243 log "Wait for df ($DFPID) ... "
1246 lctl set_param debug="$DBG_SAVE"
1248 # retrieve from the log the first server that the client tried to
1249 # contact after the connection loss
1250 $LCTL dk $TMP/lustre-log-$TESTNAME.log
1251 NEXTCONN=`awk "/${MSG}/ {start = 1;}
1252 /import_select_connection.*$device-mdc.* using connection/ {
1254 if (\\\$NF ~ /$FAKENID/)
1260 }" $TMP/lustre-log-$TESTNAME.log`
1261 [ "$NEXTCONN" != "0" ] && log "The client didn't try to reconnect to the last active server (tried ${NEXTCONN} instead)" && return 7
1264 run_test 35a "Reconnect to the last active server first"
1266 test_35b() { # bug 18674
1267 remote_mds || { skip "local MDS" && return 0; }
1271 $LCTL set_param debug="ha"
1273 MSG="conf-sanity.sh test_35b `date +%F%kh%Mm%Ss`"
1276 log "Set up a fake failnode for the MDS"
1278 local device=$(do_facet mds "$LCTL get_param -n devices" | \
1279 awk '($3 ~ "mdt" && $4 ~ "MDT") { print $4 }' | head -1)
1280 do_facet mds "$LCTL conf_param ${device}.failover.node=$FAKENID" || \
1283 local at_max_saved=0
1284 # adaptive timeouts may prevent seeing the issue
1285 if at_is_enabled; then
1286 at_max_saved=$(at_max_get mds)
1287 at_max_set 0 mds client
1290 mkdir -p $MOUNT/testdir
1291 touch $MOUNT/testdir/test
1293 log "Injecting EBUSY on MDS"
1294 # Setting OBD_FAIL_MDS_RESEND=0x136
1295 do_facet mds "$LCTL set_param fail_loc=0x80000136" || return 2
1297 log "Stat on a test file"
1298 stat $MOUNT/testdir/test
1300 log "Stop injecting EBUSY on MDS"
1301 do_facet mds "$LCTL set_param fail_loc=0" || return 3
1302 rm -f $MOUNT/testdir/test
1305 # restore adaptive timeout
1306 [ $at_max_saved -ne 0 ] && at_max_set $at_max_saved mds client
1308 $LCTL dk $TMP/lustre-log-$TESTNAME.log
1310 # retrieve from the log if the client has ever tried to
1311 # contact the fake server after the loss of connection
1312 FAILCONN=`awk "BEGIN {ret = 0;}
1313 /import_select_connection.*${FSNAME}-MDT0000-mdc.* using connection/ {
1315 if (\\\$NF ~ /$FAKENID/) {
1320 END {print ret}" $TMP/lustre-log-$TESTNAME.log`
1322 [ "$FAILCONN" == "0" ] && \
1323 log "ERROR: The client reconnection has not been triggered" && \
1325 [ "$FAILCONN" == "2" ] && \
1326 log "ERROR: The client tried to reconnect to the failover server while the primary was busy" && \
1331 run_test 35b "Continue reconnection retries, if the active server is busy"
1335 local FSNAME2=test1234
1336 local fs3ost_HOST=$ost_HOST
1338 [ -n "$ost1_HOST" ] && fs2ost_HOST=$ost1_HOST && fs3ost_HOST=$ost1_HOST
1341 if [ -z "$fs2ost_DEV" -o -z "$fs2mds_DEV" -o -z "$fs3ost_DEV" ]; then
1342 do_facet $SINGLEMDS [ -b "$MDSDEV" ] && \
1343 skip_env "mixed loopback and real device not working" && return
1345 [ $OSTCOUNT -lt 2 ] && skip_env "skipping test for single OST" && return
1347 [ "$ost_HOST" = "`hostname`" -o "$ost1_HOST" = "`hostname`" ] || \
1348 { skip "remote OST" && return 0; }
1350 local fs2mdsdev=${fs2mds_DEV:-${MDSDEV}_2}
1351 local fs2ostdev=${fs2ost_DEV:-$(ostdevname 1)_2}
1352 local fs3ostdev=${fs3ost_DEV:-$(ostdevname 2)_2}
1353 add fs2mds $MDS_MKFS_OPTS --fsname=${FSNAME2} --reformat $fs2mdsdev || exit 10
1354 # XXX after we support non 4K disk blocksize, change following --mkfsoptions with
1356 add fs2ost $OST_MKFS_OPTS --mkfsoptions='-b4096' --fsname=${FSNAME2} --mgsnode=$MGSNID --reformat $fs2ostdev || exit 10
1357 add fs3ost $OST_MKFS_OPTS --mkfsoptions='-b4096' --fsname=${FSNAME2} --mgsnode=$MGSNID --reformat $fs3ostdev || exit 10
1359 start fs2mds $fs2mdsdev $MDS_MOUNT_OPTS
1360 start fs2ost $fs2ostdev $OST_MOUNT_OPTS
1361 start fs3ost $fs3ostdev $OST_MOUNT_OPTS
1363 mount -t lustre $MGSNID:/${FSNAME2} $MOUNT2 || return 1
1365 sleep 5 # until 11778 fixed
1367 dd if=/dev/zero of=$MOUNT2/$tfile bs=1M count=7 || return 2
1369 BKTOTAL=`lctl get_param -n obdfilter.*.kbytestotal | awk 'BEGIN{total=0}; {total+=$1}; END{print total}'`
1370 BKFREE=`lctl get_param -n obdfilter.*.kbytesfree | awk 'BEGIN{free=0}; {free+=$1}; END{print free}'`
1371 BKAVAIL=`lctl get_param -n obdfilter.*.kbytesavail | awk 'BEGIN{avail=0}; {avail+=$1}; END{print avail}'`
1372 STRING=`df -P $MOUNT2 | tail -n 1 | awk '{print $2","$3","$4}'`
1373 DFTOTAL=`echo $STRING | cut -d, -f1`
1374 DFUSED=`echo $STRING | cut -d, -f2`
1375 DFAVAIL=`echo $STRING | cut -d, -f3`
1376 DFFREE=$(($DFTOTAL - $DFUSED))
1378 ALLOWANCE=$((64 * $OSTCOUNT))
1380 if [ $DFTOTAL -lt $(($BKTOTAL - $ALLOWANCE)) ] ||
1381 [ $DFTOTAL -gt $(($BKTOTAL + $ALLOWANCE)) ] ; then
1382 echo "**** FAIL: df total($DFTOTAL) mismatch OST total($BKTOTAL)"
1385 if [ $DFFREE -lt $(($BKFREE - $ALLOWANCE)) ] ||
1386 [ $DFFREE -gt $(($BKFREE + $ALLOWANCE)) ] ; then
1387 echo "**** FAIL: df free($DFFREE) mismatch OST free($BKFREE)"
1390 if [ $DFAVAIL -lt $(($BKAVAIL - $ALLOWANCE)) ] ||
1391 [ $DFAVAIL -gt $(($BKAVAIL + $ALLOWANCE)) ] ; then
1392 echo "**** FAIL: df avail($DFAVAIL) mismatch OST avail($BKAVAIL)"
1397 stop fs3ost -f || return 200
1398 stop fs2ost -f || return 201
1399 stop fs2mds -f || return 202
1400 rm -rf $MOUNT2 $fs2mdsdev $fs2ostdev $fs3ostdev
1401 unload_modules || return 203
1404 run_test 36 "df report consistency on OSTs with different block size"
1407 client_only && skip "client only testing" && return 0
1408 LOCAL_MDSDEV="$TMP/mdt.img"
1409 SYM_MDSDEV="$TMP/sym_mdt.img"
1411 echo "MDS : $LOCAL_MDSDEV"
1412 echo "SYMLINK : $SYM_MDSDEV"
1416 mkfs.lustre --reformat --fsname=lustre --mdt --mgs --device-size=9000 $LOCAL_MDSDEV ||
1417 error "mkfs.lustre $LOCAL_MDSDEV failed"
1418 ln -s $LOCAL_MDSDEV $SYM_MDSDEV
1420 echo "mount symlink device - $SYM_MDSDEV"
1422 mount_op=`mount -v -t lustre -o loop $SYM_MDSDEV ${MOUNT%/*}/mds 2>&1 | grep "unable to set tunable"`
1423 umount -d ${MOUNT%/*}/mds
1424 rm -f $LOCAL_MDSDEV $SYM_MDSDEV
1426 if [ -n "$mount_op" ]; then
1427 error "**** FAIL: set tunables failed for symlink device"
1431 run_test 37 "verify set tunables works for symlink device"
1433 test_38() { # bug 14222
1438 FILES=`find $SRC -type f -mtime +1 | head -n $COUNT`
1439 log "copying $(echo $FILES | wc -w) files to $DIR/$tdir"
1441 tar cf - $FILES | tar xf - -C $DIR/$tdir || \
1442 error "copying $SRC to $DIR/$tdir"
1444 umount_client $MOUNT
1446 log "rename lov_objid file on MDS"
1447 rm -f $TMP/lov_objid.orig
1449 local dev=${SINGLEMDS}_dev
1450 local MDSDEV=${!dev}
1451 do_facet $SINGLEMDS "$DEBUGFS -c -R \\\"dump lov_objid $TMP/lov_objid.orig\\\" $MDSDEV"
1452 do_facet $SINGLEMDS "$DEBUGFS -w -R \\\"rm lov_objid\\\" $MDSDEV"
1454 do_facet $SINGLEMDS "od -Ax -td8 $TMP/lov_objid.orig"
1455 # check create in mds_lov_connect
1459 [ $V ] && log "verifying $DIR/$tdir/$f"
1460 diff -q $f $DIR/$tdir/$f || ERROR=y
1462 do_facet $SINGLEMDS "$DEBUGFS -c -R \\\"dump lov_objid $TMP/lov_objid.new\\\" $MDSDEV"
1463 do_facet $SINGLEMDS "od -Ax -td8 $TMP/lov_objid.new"
1464 [ "$ERROR" = "y" ] && error "old and new files are different after connect" || true
1466 # check it's updates in sync
1467 umount_client $MOUNT
1470 do_facet $SINGLEMDS dd if=/dev/zero of=$TMP/lov_objid.clear bs=4096 count=1
1471 do_facet $SINGLEMDS "$DEBUGFS -w -R \\\"rm lov_objid\\\" $MDSDEV"
1472 do_facet $SINGLEMDS "$DEBUGFS -w -R \\\"write $TMP/lov_objid.clear lov_objid\\\" $MDSDEV "
1477 [ $V ] && log "verifying $DIR/$tdir/$f"
1478 diff -q $f $DIR/$tdir/$f || ERROR=y
1480 do_facet $SINGLEMDS "$DEBUGFS -c -R \\\"dump lov_objid $TMP/lov_objid.new1\\\" $MDSDEV"
1481 do_facet $SINGLEMDS "od -Ax -td8 $TMP/lov_objid.new1"
1482 umount_client $MOUNT
1484 [ "$ERROR" = "y" ] && error "old and new files are different after sync" || true
1486 log "files compared the same"
1489 run_test 38 "MDS recreates missing lov_objid file from OST data"
1495 perl $SRCDIR/leak_finder.pl $TMP/debug 2>&1 | egrep '*** Leak:' &&
1496 error "memory leak detected" || true
1498 run_test 39 "leak_finder recognizes both LUSTRE and LNET malloc messages"
1500 test_40() { # bug 15759
1502 #define OBD_FAIL_TGT_TOOMANY_THREADS 0x706
1503 do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0x80000706"
1507 run_test 40 "race during service thread startup"
1509 test_41() { #bug 14134
1511 local dev=${SINGLEMDS}_dev
1512 local MDSDEV=${!dev}
1514 start $SINGLEMDS $MDSDEV $MDS_MOUNT_OPTS -o nosvc -n
1515 start ost1 `ostdevname 1` $OST_MOUNT_OPTS
1516 start $SINGLEMDS $MDSDEV $MDS_MOUNT_OPTS -o nomgs,force
1518 mount_client $MOUNT || return 1
1521 echo "blah blah" > $MOUNT/$tfile
1524 umount_client $MOUNT
1525 stop ost1 -f || return 201
1526 stop_mds -f || return 202
1527 stop_mds -f || return 203
1528 unload_modules || return 204
1531 run_test 41 "mount mds with --nosvc and --nomgs"
1533 test_42() { #bug 14693
1535 check_mount || return 2
1536 do_facet mgs $LCTL conf_param lustre.llite.some_wrong_param=10
1537 umount_client $MOUNT
1538 mount_client $MOUNT || return 1
1542 run_test 42 "invalid config param should not prevent client from mounting"
1545 [ $UID -ne 0 -o $RUNAS_ID -eq 0 ] && skip_env "run as root"
1547 chmod ugo+x $DIR || error "chmod 0 failed"
1549 "lctl get_param -n mdt.$FSNAME-MDT0000.root_squash" \
1550 "$FSNAME.mdt.root_squash" \
1553 "lctl get_param -n mdt.$FSNAME-MDT0000.nosquash_nids" \
1554 "$FSNAME.mdt.nosquash_nids" \
1558 # create set of test files
1560 echo "111" > $DIR/$tfile-userfile || error "write 1 failed"
1561 chmod go-rw $DIR/$tfile-userfile || error "chmod 1 failed"
1562 chown $RUNAS_ID.$RUNAS_ID $DIR/$tfile-userfile || error "chown failed"
1564 echo "222" > $DIR/$tfile-rootfile || error "write 2 failed"
1565 chmod go-rw $DIR/$tfile-rootfile || error "chmod 2 faield"
1567 mkdir $DIR/$tdir-rootdir -p || error "mkdir failed"
1568 chmod go-rwx $DIR/$tdir-rootdir || error "chmod 3 failed"
1569 touch $DIR/$tdir-rootdir/tfile-1 || error "touch failed"
1572 # check root_squash:
1573 # set root squash UID:GID to RUNAS_ID
1574 # root should be able to access only files owned by RUNAS_ID
1577 "lctl get_param -n mdt.$FSNAME-MDT0000.root_squash" \
1578 "$FSNAME.mdt.root_squash" \
1579 "$RUNAS_ID:$RUNAS_ID"
1581 ST=$(stat -c "%n: owner uid %u (%A)" $DIR/$tfile-userfile)
1582 dd if=$DIR/$tfile-userfile 1>/dev/null 2>/dev/null || \
1583 error "$ST: root read permission is denied"
1584 echo "$ST: root read permission is granted - ok"
1587 dd conv=notrunc if=$DIR/$tfile-userfile 1>/dev/null 2>/dev/null || \
1588 error "$ST: root write permission is denied"
1589 echo "$ST: root write permission is granted - ok"
1591 ST=$(stat -c "%n: owner uid %u (%A)" $DIR/$tfile-rootfile)
1592 dd if=$DIR/$tfile-rootfile 1>/dev/null 2>/dev/null && \
1593 error "$ST: root read permission is granted"
1594 echo "$ST: root read permission is denied - ok"
1597 dd conv=notrunc of=$DIR/$tfile-rootfile 1>/dev/null 2>/dev/null && \
1598 error "$ST: root write permission is granted"
1599 echo "$ST: root write permission is denied - ok"
1601 ST=$(stat -c "%n: owner uid %u (%A)" $DIR/$tdir-rootdir)
1602 rm $DIR/$tdir-rootdir/tfile-1 1>/dev/null 2>/dev/null && \
1603 error "$ST: root unlink permission is granted"
1604 echo "$ST: root unlink permission is denied - ok"
1606 touch $DIR/tdir-rootdir/tfile-2 1>/dev/null 2>/dev/null && \
1607 error "$ST: root create permission is granted"
1608 echo "$ST: root create permission is denied - ok"
1611 # check nosquash_nids:
1612 # put client's NID into nosquash_nids list,
1613 # root should be able to access root file after that
1615 local NIDLIST=$(lctl list_nids all | tr '\n' ' ')
1616 NIDLIST="2@elan $NIDLIST 192.168.0.[2,10]@tcp"
1617 NIDLIST=$(echo $NIDLIST | tr -s ' ' ' ')
1619 "lctl get_param -n mdt.$FSNAME-MDT0000.nosquash_nids" \
1620 "$FSNAME-MDTall.mdt.nosquash_nids" \
1623 ST=$(stat -c "%n: owner uid %u (%A)" $DIR/$tfile-rootfile)
1624 dd if=$DIR/$tfile-rootfile 1>/dev/null 2>/dev/null || \
1625 error "$ST: root read permission is denied"
1626 echo "$ST: root read permission is granted - ok"
1629 dd conv=notrunc of=$DIR/$tfile-rootfile 1>/dev/null 2>/dev/null || \
1630 error "$ST: root write permission is denied"
1631 echo "$ST: root write permission is granted - ok"
1633 ST=$(stat -c "%n: owner uid %u (%A)" $DIR/$tdir-rootdir)
1634 rm $DIR/$tdir-rootdir/tfile-1 || \
1635 error "$ST: root unlink permission is denied"
1636 echo "$ST: root unlink permission is granted - ok"
1637 touch $DIR/$tdir-rootdir/tfile-2 || \
1638 error "$ST: root create permission is denied"
1639 echo "$ST: root create permission is granted - ok"
1643 run_test 43 "check root_squash and nosquash_nids"
1645 umount_client $MOUNT
1650 check_mount || return 2
1651 UUID=$($LCTL get_param llite.${FSNAME}*.uuid | cut -d= -f2)
1653 UUIDS=$(do_facet mds "$LCTL get_param mdt.${FSNAME}*.exports.*.uuid")
1654 for VAL in $UUIDS; do
1655 NID=$(echo $VAL | cut -d= -f1)
1656 CLUUID=$(echo $VAL | cut -d= -f2)
1657 [ "$UUID" = "$CLUUID" ] && STATS_FOUND=yes && break
1659 [ "$STATS_FOUND" = "no" ] && error "stats not found for client"
1663 run_test 44 "mounted client proc entry exists"
1667 check_mount || return 2
1672 #define OBD_FAIL_PTLRPC_LONG_UNLINK 0x50f
1673 do_facet client "lctl set_param fail_loc=0x50f"
1676 manual_umount_client --force || return 3
1677 do_facet client "lctl set_param fail_loc=0x0"
1679 mount_client $MOUNT || return 4
1683 run_test 45 "long unlink handling in ptlrpcd"
1690 umount_client $MOUNT2 || rc=$?
1691 umount_client $MOUNT || rc=$?
1692 while [ $count -gt 0 ]; do
1693 stop ost${count} -f || rc=$?
1697 # writeconf is needed after the test, otherwise,
1698 # we might end up with extra OSTs
1700 cleanup_nocli || rc=$?
1705 echo "Testing with $OSTCOUNT OSTs"
1707 start_mds || return 1
1708 #first client should see only one ost
1709 start_ost || return 2
1710 wait_osc_import_state mds ost FULL
1712 mount_client $MOUNT || return 3
1713 trap "cleanup_46a $OSTCOUNT" EXIT ERR
1716 for (( i=2; i<=$OSTCOUNT; i++ )); do
1717 start ost$i `ostdevname $i` $OST_MOUNT_OPTS || return $((i+2))
1720 # wait until osts in sync
1721 for (( i=2; i<=$OSTCOUNT; i++ )); do
1722 wait_osc_import_state mds ost$i FULL
1726 #second client see all ost's
1728 mount_client $MOUNT2 || return 8
1729 $LFS setstripe $MOUNT2 -c -1 || return 9
1730 $LFS getstripe $MOUNT2 || return 10
1732 echo "ok" > $MOUNT2/widestripe
1733 $LFS getstripe $MOUNT2/widestripe || return 11
1734 # fill acl buffer for avoid expand lsm to them
1735 awk -F : '{if (FNR < 25) { print "u:"$1":rwx" }}' /etc/passwd | while read acl; do
1736 setfacl -m $acl $MOUNT2/widestripe
1740 stat $MOUNT/widestripe || return 12
1742 cleanup_46a $OSTCOUNT || { echo "cleanup_46a failed!" && return 13; }
1745 run_test 46a "handle ost additional - wide striped file"
1750 check_mount || return 2
1751 $LCTL set_param ldlm.namespaces.$FSNAME-*-*-*.lru_size=100
1755 for ns in $($LCTL get_param ldlm.namespaces.$FSNAME-*-*-*.lru_size); do
1756 if echo $ns | grep "MDT[[:digit:]]*"; then
1759 lrs=$(echo $ns | sed 's/.*lru_size=//')
1760 lru_size[count]=$lrs
1765 facet_failover $SINGLEMDS
1766 client_up || return 3
1769 for ns in $($LCTL get_param ldlm.namespaces.$FSNAME-*-*-*.lru_size); do
1770 if echo $ns | grep "MDT[[:digit:]]*"; then
1773 lrs=$(echo $ns | sed 's/.*lru_size=//')
1774 if ! test "$lrs" -eq "${lru_size[count]}"; then
1775 n=$(echo $ns | sed -e 's/ldlm.namespaces.//' -e 's/.lru_size=.*//')
1776 error "$n has lost lru_size: $lrs vs. ${lru_size[count]}"
1784 run_test 47 "server restart does not make client loss lru_resize settings"
1789 # reformat after this test is needed - if test will failed
1790 # we will have unkillable file at FS
1794 test_48() { # bug 17636
1797 check_mount || return 2
1799 $LFS setstripe $MOUNT -c -1 || return 9
1800 $LFS getstripe $MOUNT || return 10
1802 echo "ok" > $MOUNT/widestripe
1803 $LFS getstripe $MOUNT/widestripe || return 11
1805 trap cleanup_48 EXIT ERR
1807 # fill acl buffer for avoid expand lsm to them
1808 getent passwd | awk -F : '{ print "u:"$1":rwx" }' | while read acl; do
1809 setfacl -m $acl $MOUNT/widestripe
1812 stat $MOUNT/widestripe || return 12
1817 run_test 48 "too many acls on file"
1819 # check PARAM_SYS_LDLM_TIMEOUT option of MKFS.LUSTRE
1820 test_49() { # bug 17710
1821 local OLD_MDS_MKFS_OPTS=$MDS_MKFS_OPTS
1822 local OLD_OST_MKFS_OPTS=$OST_MKFS_OPTS
1823 local LOCAL_TIMEOUT=20
1826 OST_MKFS_OPTS="--ost --fsname=$FSNAME --device-size=$OSTSIZE --mgsnode=$MGSNID --param sys.timeout=$LOCAL_TIMEOUT --param sys.ldlm_timeout=$LOCAL_TIMEOUT $MKFSOPT $OSTOPT"
1830 check_mount || return 1
1832 echo "check ldlm_timout..."
1833 LDLM_MDS="`do_facet mds lctl get_param -n ldlm_timeout`"
1834 LDLM_OST1="`do_facet ost1 lctl get_param -n ldlm_timeout`"
1835 LDLM_CLIENT="`do_facet client lctl get_param -n ldlm_timeout`"
1837 if [ $LDLM_MDS -ne $LDLM_OST1 ] || [ $LDLM_MDS -ne $LDLM_CLIENT ]; then
1838 error "Different LDLM_TIMEOUT:$LDLM_MDS $LDLM_OST1 $LDLM_CLIENT"
1841 if [ $LDLM_MDS -ne $((LOCAL_TIMEOUT / 3)) ]; then
1842 error "LDLM_TIMEOUT($LDLM_MDS) is not correct"
1845 umount_client $MOUNT
1846 stop_ost || return 2
1847 stop_mds || return 3
1849 OST_MKFS_OPTS="--ost --fsname=$FSNAME --device-size=$OSTSIZE --mgsnode=$MGSNID --param sys.timeout=$LOCAL_TIMEOUT --param sys.ldlm_timeout=$((LOCAL_TIMEOUT - 1)) $MKFSOPT $OSTOPT"
1853 check_mount || return 7
1855 LDLM_MDS="`do_facet mds lctl get_param -n ldlm_timeout`"
1856 LDLM_OST1="`do_facet ost1 lctl get_param -n ldlm_timeout`"
1857 LDLM_CLIENT="`do_facet client lctl get_param -n ldlm_timeout`"
1859 if [ $LDLM_MDS -ne $LDLM_OST1 ] || [ $LDLM_MDS -ne $LDLM_CLIENT ]; then
1860 error "Different LDLM_TIMEOUT:$LDLM_MDS $LDLM_OST1 $LDLM_CLIENT"
1863 if [ $LDLM_MDS -ne $((LOCAL_TIMEOUT - 1)) ]; then
1864 error "LDLM_TIMEOUT($LDLM_MDS) is not correct"
1867 cleanup || return $?
1869 MDS_MKFS_OPTS=$OLD_MDS_MKFS_OPTS
1870 OST_MKFS_OPTS=$OLD_OST_MKFS_OPTS
1872 run_test 49 "check PARAM_SYS_LDLM_TIMEOUT option of MKFS.LUSTRE"
1875 # Test both statfs and lfs df and fail if either one fails
1876 multiop_bg_pause $1 f_
1879 killall -USR1 multiop
1880 [ $RC1 -ne 0 ] && log "lazystatfs multiop failed"
1881 wait $PID || { RC1=$?; log "multiop return error "; }
1888 if [ $RC2 -eq 0 ]; then
1890 log "lazystatfs df failed"
1894 [[ $RC1 -ne 0 || $RC2 -eq 0 ]] && RC=1
1900 lctl set_param llite.$FSNAME-*.lazystatfs=1
1903 lazystatfs $MOUNT || error "lazystatfs failed but no down servers"
1905 cleanup || return $?
1907 run_test 50a "lazystatfs all servers available =========================="
1911 lctl set_param llite.$FSNAME-*.lazystatfs=1
1914 # Wait for client to detect down OST
1915 stop_ost || error "Unable to stop OST1"
1916 wait_osc_import_state mds ost DISCONN
1918 lazystatfs $MOUNT || error "lazystatfs should don't have returned EIO"
1920 umount_client $MOUNT || error "Unable to unmount client"
1921 stop_mds || error "Unable to stop MDS"
1923 run_test 50b "lazystatfs all servers down =========================="
1926 start_mds || error "Unable to start MDS"
1927 start_ost || error "Unable to start OST1"
1928 start_ost2 || error "Unable to start OST2"
1929 mount_client $MOUNT || error "Unable to mount client"
1930 lctl set_param llite.$FSNAME-*.lazystatfs=1
1933 # Wait for client to detect down OST
1934 stop_ost || error "Unable to stop OST1"
1935 wait_osc_import_state mds ost DISCONN
1936 lazystatfs $MOUNT || error "lazystatfs failed with one down server"
1938 umount_client $MOUNT || error "Unable to unmount client"
1939 stop_ost2 || error "Unable to stop OST2"
1940 stop_mds || error "Unable to stop MDS"
1942 run_test 50c "lazystatfs one server down =========================="
1945 start_mds || error "Unable to start MDS"
1946 start_ost || error "Unable to start OST1"
1947 start_ost2 || error "Unable to start OST2"
1948 mount_client $MOUNT || error "Unable to mount client"
1949 lctl set_param llite.$FSNAME-*.lazystatfs=1
1952 # Issue the statfs during the window where the client still
1953 # belives the OST to be available but it is in fact down.
1954 # No failure just a statfs which hangs for a timeout interval.
1955 stop_ost || error "Unable to stop OST1"
1956 lazystatfs $MOUNT || error "lazystatfs failed with one down server"
1958 umount_client $MOUNT || error "Unable to unmount client"
1959 stop_ost2 || error "Unable to stop OST2"
1960 stop_mds || error "Unable to stop MDS"
1962 run_test 50d "lazystatfs client/server conn race =========================="
1969 start_mds || return 1
1970 #first client should see only one ost
1971 start_ost || return 2
1972 wait_osc_import_state mds ost FULL
1974 # Wait for client to detect down OST
1975 stop_ost || error "Unable to stop OST1"
1976 wait_osc_import_state mds ost DISCONN
1978 mount_client $MOUNT || error "Unable to mount client"
1979 lctl set_param llite.$FSNAME-*.lazystatfs=0
1981 multiop_bg_pause $MOUNT _f
1985 if [ $RC1 -ne 0 ]; then
1986 log "multiop failed $RC1"
1989 sleep $(( $TIMEOUT+1 ))
1991 [ $? -ne 0 ] && error "process isn't sleep"
1992 start_ost || error "Unable to start OST1"
1993 wait $pid || error "statfs failed"
1996 umount_client $MOUNT || error "Unable to unmount client"
1997 stop_ost || error "Unable to stop OST1"
1998 stop_mds || error "Unable to stop MDS"
2000 run_test 50e "normal statfs all servers down =========================="
2005 CONN_PROC="osc.$FSNAME-OST0001-osc-[M]*.ost_server_uuid"
2007 start_mds || error "Unable to start mds"
2008 #first client should see only one ost
2009 start_ost || error "Unable to start OST1"
2010 wait_osc_import_state mds ost FULL
2012 start_ost2 || error "Unable to start OST2"
2013 wait_osc_import_state mds ost2 FULL
2015 # Wait for client to detect down OST
2016 stop_ost2 || error "Unable to stop OST2"
2018 wait_osc_import_state mds ost2 DISCONN
2019 mount_client $MOUNT || error "Unable to mount client"
2020 lctl set_param llite.$FSNAME-*.lazystatfs=0
2022 multiop_bg_pause $MOUNT _f
2026 if [ $RC1 -ne 0 ]; then
2027 log "lazystatfs multiop failed $RC1"
2030 sleep $(( $TIMEOUT+1 ))
2032 [ $? -ne 0 ] && error "process isn't sleep"
2033 start_ost2 || error "Unable to start OST2"
2034 wait $pid || error "statfs failed"
2035 stop_ost2 || error "Unable to stop OST2"
2038 umount_client $MOUNT || error "Unable to unmount client"
2039 stop_ost || error "Unable to stop OST1"
2040 stop_mds || error "Unable to stop MDS"
2043 run_test 50f "normal statfs one server in down =========================="
2046 [ "$OSTCOUNT" -lt "2" ] && skip_env "$OSTCOUNT < 2, skipping" && return
2048 start_ost2 || error "Unable to start OST2"
2050 local PARAM="${FSNAME}-OST0001.osc.active"
2052 $LFS setstripe -c -1 $DIR/$tfile || error "Unable to lfs setstripe"
2053 do_facet mgs $LCTL conf_param $PARAM=0 || error "Unable to deactivate OST"
2055 umount_client $MOUNT || error "Unable to unmount client"
2056 mount_client $MOUNT || error "Unable to mount client"
2057 # This df should not cause a panic
2060 do_facet mgs $LCTL conf_param $PARAM=1 || error "Unable to activate OST"
2062 umount_client $MOUNT || error "Unable to unmount client"
2063 stop_ost2 || error "Unable to stop OST2"
2064 stop_ost || error "Unable to stop OST1"
2065 stop_mds || error "Unable to stop MDS"
2068 run_test 50g "deactivated OST should not cause panic====================="
2071 local LOCAL_TIMEOUT=20
2075 check_mount || return 1
2078 $LFS setstripe -c -1 $MOUNT/d1
2079 #define OBD_FAIL_MDS_REINT_DELAY 0x142
2080 do_facet $SINGLEMDS "lctl set_param fail_loc=0x142"
2081 touch $MOUNT/d1/f1 &
2084 start_ost2 || return 2
2086 stop_ost2 || return 3
2089 run_test 51 "Verify that mdt_reint handles RMF_MDT_MD correctly when an OST is added"
2098 do_node $node mkdir -p $dest
2099 [ $? -eq 0 ] || { error "Unable to create directory"; return 1; }
2101 do_node $node 'tar cf - '$@' | tar xf - -C '$dest';
2102 [ \"\${PIPESTATUS[*]}\" = \"0 0\" ] || exit 1'
2103 [ $? -eq 0 ] || { error "Unable to tar files"; return 2; }
2105 do_node $node 'getfattr -d -m "[a-z]*\\." '$@' > '$xattrs
2106 [ $? -eq 0 ] || { error "Unable to read xattrs"; return 3; }
2116 local backup2=${TMP}/backup2
2118 do_node $node mkdir -p $backup2
2119 [ $? -eq 0 ] || { error "Unable to create directory"; return 1; }
2121 do_node $node 'tar cf - '$@' | tar xf - -C '$backup2';
2122 [ \"\${PIPESTATUS[*]}\" = \"0 0\" ] || exit 1'
2123 [ $? -eq 0 ] || { error "Unable to tar files to diff"; return 2; }
2125 do_node $node "diff -rq $backup $backup2"
2126 [ $? -eq 0 ] || { error "contents differ"; return 3; }
2128 local xattrs2=${TMP}/xattrs2
2129 do_node $node 'getfattr -d -m "[a-z]*\\." '$@' > '$xattrs2
2130 [ $? -eq 0 ] || { error "Unable to read xattrs to diff"; return 4; }
2132 do_node $node "diff $xattrs $xattrs2"
2133 [ $? -eq 0 ] || { error "xattrs differ"; return 5; }
2135 do_node $node "rm -rf $backup2 $xattrs2"
2136 [ $? -eq 0 ] || { error "Unable to delete temporary files"; return 6; }
2141 [ $? -eq 0 ] || { error "Unable to start MDS"; return 1; }
2143 [ $? -eq 0 ] || { error "Unable to start OST1"; return 2; }
2145 [ $? -eq 0 ] || { error "Unable to mount client"; return 3; }
2148 local ost1mnt=${MOUNT%/*}/ost1
2149 local ost1node=$(facet_active_host ost1)
2150 local ost1tmp=$TMP/conf52
2153 [ $? -eq 0 ] || { error "Unable to create tdir"; return 4; }
2154 touch $TMP/modified_first
2155 [ $? -eq 0 ] || { error "Unable to create temporary file"; return 5; }
2156 do_node $ost1node "mkdir -p $ost1tmp && touch $ost1tmp/modified_first"
2157 [ $? -eq 0 ] || { error "Unable to create temporary file"; return 6; }
2160 $LFS setstripe $DIR/$tdir -c -1 -s 1M
2161 [ $? -eq 0 ] || { error "lfs setstripe failed"; return 7; }
2163 for (( i=0; i < nrfiles; i++ )); do
2164 multiop $DIR/$tdir/$tfile-$i Ow1048576w1048576w524288c
2165 [ $? -eq 0 ] || { error "multiop failed"; return 8; }
2171 echo backup files to $TMP/files
2172 local files=$(find $DIR/$tdir -type f -newer $TMP/modified_first)
2173 copy_files_xattrs `hostname` $TMP/files $TMP/file_xattrs $files
2174 [ $? -eq 0 ] || { error "Unable to copy files"; return 9; }
2176 umount_client $MOUNT
2177 [ $? -eq 0 ] || { error "Unable to umount client"; return 10; }
2179 [ $? -eq 0 ] || { error "Unable to stop ost1"; return 11; }
2181 echo mount ost1 as ldiskfs
2182 do_node $ost1node mount -t $FSTYPE $ost1_dev $ost1mnt $OST_MOUNT_OPTS
2183 [ $? -eq 0 ] || { error "Unable to mount ost1 as ldiskfs"; return 12; }
2186 echo backup objects to $ost1tmp/objects
2187 local objects=$(do_node $ost1node 'find '$ost1mnt'/O/0 -type f -size +0'\
2188 '-newer '$ost1tmp'/modified_first -regex ".*\/[0-9]+"')
2189 copy_files_xattrs $ost1node $ost1tmp/objects $ost1tmp/object_xattrs $objects
2190 [ $? -eq 0 ] || { error "Unable to copy objects"; return 13; }
2192 # move objects to lost+found
2193 do_node $ost1node 'mv '$objects' '${ost1mnt}'/lost+found'
2194 [ $? -eq 0 ] || { error "Unable to move objects"; return 14; }
2197 do_node $ost1node "ll_recover_lost_found_objs -d $ost1mnt/lost+found"
2198 [ $? -eq 0 ] || { error "ll_recover_lost_found_objs failed"; return 15; }
2200 # compare restored objects against saved ones
2201 diff_files_xattrs $ost1node $ost1tmp/objects $ost1tmp/object_xattrs $objects
2202 [ $? -eq 0 ] || { error "Unable to diff objects"; return 16; }
2204 do_node $ost1node "umount $ost1_dev"
2205 [ $? -eq 0 ] || { error "Unable to umount ost1 as ldiskfs"; return 17; }
2208 [ $? -eq 0 ] || { error "Unable to start ost1"; return 18; }
2210 [ $? -eq 0 ] || { error "Unable to mount client"; return 19; }
2213 diff_files_xattrs `hostname` $TMP/files $TMP/file_xattrs $files
2214 [ $? -eq 0 ] || { error "Unable to diff files"; return 20; }
2216 rm -rf $TMP/files $TMP/file_xattrs
2217 [ $? -eq 0 ] || { error "Unable to delete temporary files"; return 21; }
2218 do_node $ost1node "rm -rf $ost1tmp"
2219 [ $? -eq 0 ] || { error "Unable to delete temporary files"; return 22; }
2222 run_test 52 "check recovering objects from lost+found"
2224 # Checks threads_min/max/started for some service
2226 # Arguments: service name (OST or MDT), facet (e.g., ost1, $SINGLEMDS), and a
2227 # parameter pattern prefix like 'ost.*.ost'.
2239 local msg="Insane $modname thread counts"
2243 check_mount || return 41
2245 # We need to expand $parampat, but it may match multiple parameters, so
2246 # we'll pick the first one
2247 if ! paramp=$(do_facet $facet "lctl get_param -N ${parampat}.threads_min"|head -1); then
2248 error "Couldn't expand ${parampat}.threads_min parameter name"
2252 # Remove the .threads_min part
2253 paramp=${paramp%.threads_min}
2255 # Check for sanity in defaults
2256 tmin=$(do_facet $facet "lctl get_param -n ${paramp}.threads_min" || echo 0)
2257 tmax=$(do_facet $facet "lctl get_param -n ${paramp}.threads_max" || echo 0)
2258 tstarted=$(do_facet $facet "lctl get_param -n ${paramp}.threads_started" || echo 0)
2259 lassert 23 "$msg (PDSH problems?)" '(($tstarted && $tmin && $tmax))' || return $?
2260 lassert 24 "$msg" '(($tstarted >= $tmin && $tstarted <= tmax ))' || return $?
2262 # Check that we can lower min/max
2263 do_facet $facet "lctl set_param ${paramp}.threads_min=$((tmin - 1))"
2264 do_facet $facet "lctl set_param ${paramp}.threads_max=$((tmax - 10))"
2265 tmin2=$(do_facet $facet "lctl get_param -n ${paramp}.threads_min" || echo 0)
2266 tmax2=$(do_facet $facet "lctl get_param -n ${paramp}.threads_max" || echo 0)
2267 lassert 25 "$msg" '(($tmin2 == ($tmin - 1) && $tmax2 == ($tmax -10)))' || return $?
2269 # Check that we can set min/max to the same value
2270 do_facet $facet "lctl set_param ${paramp}.threads_min=$tmin"
2271 do_facet $facet "lctl set_param ${paramp}.threads_max=$tmin"
2272 tmin2=$(do_facet $facet "lctl get_param -n ${paramp}.threads_min" || echo 0)
2273 tmax2=$(do_facet $facet "lctl get_param -n ${paramp}.threads_max" || echo 0)
2274 lassert 26 "$msg" '(($tmin2 == $tmin && $tmax2 == $tmin))' || return $?
2276 # Check that we can't set max < min
2277 do_facet $facet "lctl set_param ${paramp}.threads_max=$((tmin - 1))"
2278 tmin2=$(do_facet $facet "lctl get_param -n ${paramp}.threads_min" || echo 0)
2279 tmax2=$(do_facet $facet "lctl get_param -n ${paramp}.threads_max" || echo 0)
2280 lassert 27 "$msg" '(($tmin <= $tmax2))' || return $?
2282 # We need to ensure that we get the module options desired; to do this
2283 # we set LOAD_MODULES_REMOTE=true and we call setmodopts below.
2284 LOAD_MODULES_REMOTE=true
2287 setmodopts -a $modname "$opts" oldvalue
2291 check_mount || return 41
2293 # Restore previous setting of MODOPTS_*
2294 setmodopts $modname "$oldvalue"
2296 # Check that $opts took
2297 tmin=$(do_facet $facet "lctl get_param -n ${paramp}.threads_min")
2298 tmax=$(do_facet $facet "lctl get_param -n ${paramp}.threads_max")
2299 tstarted=$(do_facet $facet "lctl get_param -n ${paramp}.threads_started")
2300 lassert 28 "$msg" '(($tstarted == $tmin && $tstarted == $tmax ))' || return $?
2303 # Workaround a YALA bug where YALA expects that modules will remain
2304 # loaded on the servers
2305 LOAD_MODULES_REMOTE=false
2311 thread_sanity OST ost1 'ost.*.ost' 'oss_num_threads=64'
2313 run_test 53a "check OSS thread count params"
2316 thread_sanity MDT $SINGLEMDS 'mdt.*.*.' 'mdt_num_threads=64'
2318 run_test 53b "check MDT thread count params"
2320 if ! combined_mgs_mds ; then
2327 local partial_arg=""
2328 local size=$(df -B G $dir | tail -1 | awk '{print $2}' | sed 's/G//') # Gb
2330 # Run in partial (fast) mode if the size
2331 # of a partition > 10 GB
2332 [ $size -gt 10 ] && partial_arg="-p"
2334 llverfs $partial_arg $dir
2338 do_rpc_nodes $(facet_host ost1) run_llverdev $(ostdevname 1)
2339 [ $? -eq 0 ] || error "llverdev failed!"
2342 run_test 54a "llverdev"
2347 [ $? -eq 0 ] || error "llverfs failed!"
2350 run_test 54b "llverfs"
2353 equals_msg `basename $0`: test complete
2354 [ -f "$TESTSUITELOG" ] && cat $TESTSUITELOG && grep -q FAIL $TESTSUITELOG && exit 1 || true