init_test_env $@
-. ${CONFIG:=$LUSTRE/tests/cfg/local.sh}
+. ${CONFIG:=$LUSTRE/tests/cfg/lmv.sh}
# Skip these tests
ALWAYS_EXCEPT=""
gen_config() {
rm -f $XMLCONFIG
- add_mds mds --dev $MDSDEV --size $MDSSIZE
- if [ ! -z "$mdsfailover_HOST" ]; then
- add_mdsfailover mds --dev $MDSDEV --size $MDSSIZE
+
+ if [ "$MDSCOUNT" -gt 1 ]; then
+ add_lmv lmv1
+ for num in `seq $MDSCOUNT`; do
+ MDSDEV=$TMP/mds${num}-`hostname`
+ add_mds mds$num --dev $MDSDEV --size $MDSSIZE --master --lmv lmv1
+ done
+ add_lov_to_lmv lov1 lmv1 --stripe_sz $STRIPE_BYTES \
+ --stripe_cnt $STRIPES_PER_OBJ --stripe_pattern 0
+ add_ost ost --lov lov1 --dev $OSTDEV --size $OSTSIZE
+ add_ost ost2 --lov lov1 --dev ${OSTDEV}-2 --size $OSTSIZE
+ add_client client --lmv lmv1 --lov lov1 --path $MOUNT
+ else
+ add_mds mds1 --dev $MDSDEV --size $MDSSIZE
+ if [ ! -z "$mdsfailover_HOST" ]; then
+ add_mdsfailover mds --dev $MDSDEV --size $MDSSIZE
+ fi
+
+ add_lov lov1 mds1 --stripe_sz $STRIPE_BYTES \
+ --stripe_cnt $STRIPES_PER_OBJ --stripe_pattern 0
+ add_ost ost --lov lov1 --dev $OSTDEV --size $OSTSIZE
+ add_ost ost2 --lov lov1 --dev ${OSTDEV}-2 --size $OSTSIZE
+ add_client client --mds mds1_svc --lov lov1 --path $MOUNT
fi
- add_lov lov1 mds --stripe_sz $STRIPE_BYTES\
+ add_lov lov1 mds --stripe_sz $STRIPE_BYTES \
--stripe_cnt $STRIPES_PER_OBJ --stripe_pattern 0
add_ost ost --lov lov1 --dev $OSTDEV --size $OSTSIZE
add_ost ost2 --lov lov1 --dev ${OSTDEV}-2 --size $OSTSIZE
if [ $activemds != "mds" ]; then
fail mds
fi
- zconf_umount $MOUNT
- stop mds ${FORCE} $MDSLCONFARGS
+ zconf_umount `hostname` $MOUNT
+ if [ "$MDSCOUNT" -gt 1 ]; then
+ for num in `seq $MDSCOUNT`; do
+ stop mds$num ${FORCE} $MDSLCONFARGS
+ done
+ else
+ stop mds ${FORCE} $MDSLCONFARGS
+ fi
stop ost2 ${FORCE} --dump cleanup.log
stop ost ${FORCE} --dump cleanup.log
}
start ost2 --reformat $OSTLCONFARGS
[ "$DAEMONFILE" ] && $LCTL debug_daemon start $DAEMONFILE $DAEMONSIZE
start mds $MDSLCONFARGS --reformat
- zconf_mount $MOUNT
+ grep " $MOUNT " /proc/mounts || zconf_mount `hostname` $MOUNT
}
$SETUP
mkdir -p $DIR
test_0() {
- replay_barrier mds
- fail mds
+ replay_barrier mds1
+ fail mds1
}
run_test 0 "empty replay"
+test_0b() {
+ # this test attempts to trigger a race in the precreation code,
+ # and must run before any other objects are created on the filesystem
+ fail ost
+ createmany -o $DIR/$tfile 20 || return 1
+ unlinkmany $DIR/$tfile 20 || return 2
+}
+run_test 0b "ensure object created after recover exists. (3284)"
+
test_1() {
- replay_barrier mds
+ replay_barrier mds2
mcreate $DIR/$tfile
- fail mds
+ fail mds2
$CHECKSTAT -t file $DIR/$tfile || return 1
rm $DIR/$tfile
}
}
run_test 2b "touch"
-test_3() {
+test_3a() {
replay_barrier mds
mcreate $DIR/$tfile
o_directory $DIR/$tfile
$CHECKSTAT -t file $DIR/$tfile || return 2
rm $DIR/$tfile
}
-run_test 3 "replay failed open"
+run_test 3a "replay failed open(O_DIRECTORY)"
+
+test_3b() {
+ replay_barrier mds
+#define OBD_FAIL_MDS_OPEN_PACK | OBD_FAIL_ONCE
+ do_facet mds "sysctl -w lustre.fail_loc=0x80000114"
+ touch $DIR/$tfile
+ do_facet mds "sysctl -w lustre.fail_loc=0"
+ fail mds
+ $CHECKSTAT -t file $DIR/$tfile && return 2
+ return 0
+}
+run_test 3b "replay failed open -ENOMEM"
+
+test_3c() {
+ replay_barrier mds
+#define OBD_FAIL_MDS_ALLOC_OBDO | OBD_FAIL_ONCE
+ do_facet mds "sysctl -w lustre.fail_loc=0x80000128"
+ touch $DIR/$tfile
+ do_facet mds "sysctl -w lustre.fail_loc=0"
+ fail mds
+
+ $CHECKSTAT -t file $DIR/$tfile && return 2
+ return 0
+}
+run_test 3c "replay failed open -ENOMEM"
test_4() {
replay_barrier mds
done
fail mds
for i in `seq 10`; do
- grep -q "tag-$i" $DIR/$tfile-$i || error "f1c-$i"
+ grep -q "tag-$i" $DIR/$tfile-$i || error "$tfile-$i"
done
}
run_test 4 "|x| 10 open(O_CREAT)s"
sleep 1
rm -f $DIR/$tfile
touch $DIR/$tfile-2 || return 1
+ echo "pid: $pid will close"
kill -USR1 $pid
wait $pid || return 2
test_35() {
touch $DIR/$tfile
- echo 0x80000119 > /proc/sys/lustre/fail_loc
+#define OBD_FAIL_MDS_REINT_NET_REP 0x119
+ do_facet mds "sysctl -w lustre.fail_loc=0x80000119"
rm -f $DIR/$tfile &
sleep 1
+ sync
+ sleep 1
# give a chance to remove from MDS
fail_abort mds
$CHECKSTAT -t file $DIR/$tfile && return 1 || true
run_test 37 "abort recovery before client does replay (test mds_cleanup_orphans for directories)"
test_38() {
- for i in `seq 1 800`; do
- touch $DIR/$tfile-$i
- done
- for i in `seq 1 400`; do
- rm $DIR/$tfile-$i
- done
-
+ createmany -o $DIR/$tfile-%d 800
+ unlinkmany $DIR/$tfile-%d 0 400
replay_barrier mds
fail mds
- for i in `seq 401 800`; do
- rm $DIR/$tfile-$i
- done
+ unlinkmany $DIR/$tfile-%d 400 400
sleep 2
$CHECKSTAT -t file $DIR/$tfile-* && return 1 || true
}
run_test 38 "test recovery from unlink llog (test llog_gen_rec) "
test_39() {
- for i in `seq 1 800`; do
- touch $DIR/$tfile-$i
- done
-
+ createmany -o $DIR/$tfile-%d 800
replay_barrier mds
- for i in `seq 1 400`; do
- rm $DIR/$tfile-$i
- done
+ unlinkmany $DIR/$tfile-%d 0 400
fail mds
- for i in `seq 401 800`; do
- rm $DIR/$tfile-$i
- done
+ unlinkmany $DIR/$tfile-%d 400 400
sleep 2
$CHECKSTAT -t file $DIR/$tfile-* && return 1 || true
}
run_test 39 "test recovery from unlink llog (test llog_gen_rec) "
+count_ost_writes() {
+ cat /proc/fs/lustre/osc/*/stats |
+ awk -vwrites=0 '/ost_write/ { writes += $2 } END { print writes; }'
+}
+
+#b=2477,2532
+test_40(){
+ $LCTL mark multiop $MOUNT/$tfile OS_c
+ multiop $MOUNT/$tfile OS_c &
+ PID=$!
+ writeme -s $MOUNT/${tfile}-2 &
+ WRITE_PID=$!
+ sleep 1
+ facet_failover mds
+#define OBD_FAIL_MDS_CONNECT_NET 0x117
+ do_facet mds "sysctl -w lustre.fail_loc=0x80000117"
+ kill -USR1 $PID
+ stat1=`count_ost_writes`
+ sleep $TIMEOUT
+ stat2=`count_ost_writes`
+ echo "$stat1, $stat2"
+ if [ $stat1 -lt $stat2 ]; then
+ echo "writes continuing during recovery"
+ RC=0
+ else
+ echo "writes not continuing during recovery, bug 2477"
+ RC=4
+ fi
+ echo "waiting for writeme $WRITE_PID"
+ kill $WRITE_PID
+ wait $WRITE_PID
+
+ echo "waiting for multiop $PID"
+ wait $PID || return 2
+ do_facet client munlink $MOUNT/$tfile || return 3
+ do_facet client munlink $MOUNT/${tfile}-2 || return 3
+ return $RC
+}
+run_test 40 "cause recovery in ptlrpc, ensure IO continues"
+
+
+#b=2814
+# make sure that a read to one osc doesn't try to double-unlock its page just
+# because another osc is invalid. trigger_group_io used to mistakenly return
+# an error if any oscs were invalid even after having successfully put rpcs
+# on valid oscs. This was fatal if the caller was ll_readpage who unlocked
+# the page, guarnateeing that the unlock from the RPC completion would
+# assert on trying to unlock the unlocked page.
+test_41() {
+ local f=$MOUNT/$tfile
+ # make sure the start of the file is ost1
+ lfs setstripe $f $((128 * 1024)) 0 0
+ do_facet client dd if=/dev/zero of=$f bs=4k count=1 || return 3
+ cancel_lru_locks OSC
+ # fail ost2 and read from ost1
+ local osc2_dev=`$LCTL device_list | \
+ awk '(/ost2.*client_facet/){print $4}' `
+ $LCTL --device %$osc2_dev deactivate
+ do_facet client dd if=$f of=/dev/null bs=4k count=1 || return 3
+ $LCTL --device %$osc2_dev activate
+ return 0
+}
+run_test 41 "read from a valid osc while other oscs are invalid"
+
+# test MDS recovery after ost failure
+test_42() {
+ blocks=`df $MOUNT | tail -n 1 | awk '{ print $1 }'`
+ createmany -o $DIR/$tfile-%d 800
+ replay_barrier ost
+ unlinkmany $DIR/$tfile-%d 0 400
+ facet_failover ost
+
+ # osc is evicted, fs is smaller
+ blocks_after=`df $MOUNT | tail -n 1 | awk '{ print $1 }'`
+ [ $blocks_after -lt $blocks ] || return 1
+ echo wait for MDS to timeout and recover
+ sleep $((TIMEOUT * 2))
+ unlinkmany $DIR/$tfile-%d 400 400
+ $CHECKSTAT -t file $DIR/$tfile-* && return 2 || true
+}
+run_test 42 "recovery after ost failure"
+
+# b=2530
+# timeout in MDS/OST recovery RPC will LBUG MDS
+test_43() {
+ replay_barrier mds
+
+ # OBD_FAIL_OST_CREATE_NET 0x204
+ do_facet ost "sysctl -w lustre.fail_loc=0x80000204"
+ facet_failover mds
+ df $MOUNT || return 1
+ sleep 10
+ do_facet ost "sysctl -w lustre.fail_loc=0"
+
+ return 0
+}
+run_test 43 "mds osc import failure during recovery; don't LBUG"
+
+test_44() {
+ mdcdev=`awk '/mds_svc_MNT/ {print $1}' < /proc/fs/lustre/devices`
+ do_facet mds "sysctl -w lustre.fail_loc=0x80000701"
+ $LCTL --device $mdcdev recover
+ df $MOUNT
+ do_facet mds "sysctl -w lustre.fail_loc=0"
+ return 0
+}
+run_test 44 "race in target handle connect"
+
+# Handle failed close
+test_45() {
+ mdcdev=`awk '/mds_svc_MNT/ {print $1}' < /proc/fs/lustre/devices`
+ $LCTL --device $mdcdev recover
+
+ multiop $DIR/$tfile O_c &
+ pid=$!
+ sleep 1
+
+ # This will cause the CLOSE to fail before even
+ # allocating a reply buffer
+ $LCTL --device $mdcdev deactivate
+
+ # try the close
+ kill -USR1 $pid
+ wait $pid || return 1
+
+ $LCTL --device $mdcdev activate
+
+ $CHECKSTAT -t file $DIR/$tfile || return 2
+ return 0
+}
+run_test 45 "Handle failed close"
+
+test_46() {
+ dmesg -c >/dev/null
+ drop_reply "touch $DIR/$tfile"
+ fail mds
+ # ironically, the previous test, 45, will cause a real forced close,
+ # so just look for one for this test
+ dmesg | grep -i "force closing client file handle for $tfile" && return 1
+ return 0
+}
+run_test 46 "Don't leak file handle after open resend (3325)"
+
+# b=2824
+test_47() {
+
+ # create some files to make sure precreate has been done on all
+ # OSTs. (just in case this test is run independently)
+ createmany -o $DIR/$tfile 20 || return 1
+
+ # OBD_FAIL_OST_CREATE_NET 0x204
+ fail ost
+ do_facet ost "sysctl -w lustre.fail_loc=0x80000204"
+ df $MOUNT || return 2
+
+ # let the MDS discover the OST failure, attempt to recover, fail
+ # and recover again.
+ sleep $((3 * TIMEOUT))
+
+ # Without 2824, this createmany would hang
+ createmany -o $DIR/$tfile 20 || return 3
+ unlinkmany $DIR/$tfile 20 || return 4
+
+ do_facet ost "sysctl -w lustre.fail_loc=0"
+ return 0
+}
+run_test 47 "MDS->OSC failure during precreate cleanup (2824)"
+
equals_msg test complete, cleaning up
$CLEANUP
+
git://git.whamcloud.com / fs / lustre-release.git / blobdiff