. ${CONFIG:=$LUSTRE/tests/cfg/local.sh}
# Skip these tests
-ALWAYS_EXCEPT="35"
+ALWAYS_EXCEPT=""
gen_config() {
if [ $activemds != "mds" ]; then
fail mds
fi
- zconf_umount $MOUNT
+ zconf_umount `hostname` $MOUNT
stop mds ${FORCE} $MDSLCONFARGS
stop ost2 ${FORCE} --dump cleanup.log
stop ost ${FORCE} --dump cleanup.log
exit
fi
-gen_config
+SETUP=${SETUP:-"setup"}
+CLEANUP=${CLEANUP:-"cleanup"}
-start ost --reformat $OSTLCONFARGS
-start ost2 --reformat $OSTLCONFARGS
-[ "$DAEMONFILE" ] && $LCTL debug_daemon start $DAEMONFILE $DAEMONSIZE
-start mds $MDSLCONFARGS --reformat
-zconf_mount $MOUNT
+setup() {
+ gen_config
+
+ start ost --reformat $OSTLCONFARGS
+ start ost2 --reformat $OSTLCONFARGS
+ [ "$DAEMONFILE" ] && $LCTL debug_daemon start $DAEMONFILE $DAEMONSIZE
+ start mds $MDSLCONFARGS --reformat
+ zconf_mount `hostname` $MOUNT
+}
+
+$SETUP
if [ "$ONLY" == "setup" ]; then
exit 0
}
run_test 2b "touch"
-test_3() {
+test_3a() {
replay_barrier mds
mcreate $DIR/$tfile
o_directory $DIR/$tfile
$CHECKSTAT -t file $DIR/$tfile || return 2
rm $DIR/$tfile
}
-run_test 3 "replay failed open"
+run_test 3a "replay failed open(O_DIRECTORY)"
+
+test_3b() {
+ replay_barrier mds
+#define OBD_FAIL_MDS_OPEN_PACK | OBD_FAIL_ONCE
+ do_facet mds "sysctl -w lustre.fail_loc=0x80000114"
+ touch $DIR/$tfile
+ do_facet mds "sysctl -w lustre.fail_loc=0"
+ fail mds
+ $CHECKSTAT -t file $DIR/$tfile && return 2
+ return 0
+}
+run_test 3b "replay failed open -ENOMEM"
+
+test_3c() {
+ replay_barrier mds
+#define OBD_FAIL_MDS_ALLOC_OBDO | OBD_FAIL_ONCE
+ do_facet mds "sysctl -w lustre.fail_loc=0x80000128"
+ touch $DIR/$tfile
+ do_facet mds "sysctl -w lustre.fail_loc=0"
+ fail mds
+
+ $CHECKSTAT -t file $DIR/$tfile && return 2
+ return 0
+}
+run_test 3c "replay failed open -ENOMEM"
test_4() {
replay_barrier mds
done
fail mds
for i in `seq 10`; do
- grep -q "tag-$i" $DIR/$tfile-$i || error "f1c-$i"
+ grep -q "tag-$i" $DIR/$tfile-$i || error "$tfile-$i"
done
}
run_test 4 "|x| 10 open(O_CREAT)s"
fail_abort mds
kill -USR1 $pid
[ -e $DIR/$tfile ] && return 1
- sleep 3
- # wait for commitment of removal
+ sync
return 0
}
run_test 34 "abort recovery before client does replay (test mds_cleanup_orphans)"
test_35() {
touch $DIR/$tfile
- echo 0x80000119 > /proc/sys/lustre/fail_loc
+#define OBD_FAIL_MDS_REINT_NET_REP 0x119
+ do_facet mds "sysctl -w lustre.fail_loc=0x80000119"
rm -f $DIR/$tfile &
sleep 1
+ sync
+ sleep 1
# give a chance to remove from MDS
fail_abort mds
$CHECKSTAT -t file $DIR/$tfile && return 1 || true
}
run_test 35 "test recovery from llog for unlink op"
+# b=2432 resent cancel after replay uses wrong cookie,
+# so don't resend cancels
+test_36() {
+ replay_barrier mds
+ touch $DIR/$tfile
+ checkstat $DIR/$tfile
+ facet_failover mds
+ cancel_lru_locks MDC
+ if dmesg | grep "unknown lock cookie"; then
+ echo "cancel after replay failed"
+ return 1
+ fi
+}
+run_test 36 "don't resend cancel"
+
+# b=2368
+# directory orphans can't be unlinked from PENDING directory
+test_37() {
+ rmdir $DIR/$tfile 2>/dev/null
+ multiop $DIR/$tfile dD_c &
+ pid=$!
+ # give multiop a chance to open
+ sleep 1
+ rmdir $DIR/$tfile
+
+ replay_barrier mds
+ # clear the dmesg buffer so we only see errors from this recovery
+ dmesg -c >/dev/null
+ fail_abort mds
+ kill -USR1 $pid
+ dmesg | grep "mds_unlink_orphan.*error .* unlinking orphan" && return 1
+ sync
+ return 0
+}
+run_test 37 "abort recovery before client does replay (test mds_cleanup_orphans for directories)"
+
+test_38() {
+ createmany -o $DIR/$tfile-%d 800
+ unlinkmany $DIR/$tfile-%d 0 400
+ replay_barrier mds
+ fail mds
+ unlinkmany $DIR/$tfile-%d 400 400
+ sleep 2
+ $CHECKSTAT -t file $DIR/$tfile-* && return 1 || true
+}
+run_test 38 "test recovery from unlink llog (test llog_gen_rec) "
+
+test_39() {
+ createmany -o $DIR/$tfile-%d 800
+ replay_barrier mds
+ unlinkmany $DIR/$tfile-%d 0 400
+ fail mds
+ unlinkmany $DIR/$tfile-%d 400 400
+ sleep 2
+ $CHECKSTAT -t file $DIR/$tfile-* && return 1 || true
+}
+run_test 39 "test recovery from unlink llog (test llog_gen_rec) "
+
+count_ost_writes() {
+ cat /proc/fs/lustre/osc/*/stats |
+ awk -vwrites=0 '/ost_write/ { writes += $2 } END { print writes; }'
+}
+
+#b=2477,2532
+test_40(){
+ $LCTL mark multiop $MOUNT/$tfile OS_c
+ multiop $MOUNT/$tfile OS_c &
+ PID=$!
+ writeme -s $MOUNT/${tfile}-2 &
+ WRITE_PID=$!
+ sleep 1
+ facet_failover mds
+#define OBD_FAIL_MDS_CONNECT_NET 0x117
+ do_facet mds "sysctl -w lustre.fail_loc=0x80000117"
+ kill -USR1 $PID
+ stat1=`count_ost_writes`
+ sleep $TIMEOUT
+ stat2=`count_ost_writes`
+ echo "$stat1, $stat2"
+ if [ $stat1 -lt $stat2 ]; then
+ echo "writes continuing during recovery"
+ RC=0
+ else
+ echo "writes not continuing during recovery, bug 2477"
+ RC=4
+ fi
+ echo "waiting for writeme $WRITE_PID"
+ kill $WRITE_PID
+ wait $WRITE_PID
+
+ echo "waiting for multiop $PID"
+ wait $PID || return 2
+ do_facet client munlink $MOUNT/$tfile || return 3
+ do_facet client munlink $MOUNT/${tfile}-2 || return 3
+ return $RC
+}
+run_test 40 "cause recovery in ptlrpc, ensure IO continues"
+
+
+#b=2814
+# make sure that a read to one osc doesn't try to double-unlock its page just
+# because another osc is invalid. trigger_group_io used to mistakenly return
+# an error if any oscs were invalid even after having successfully put rpcs
+# on valid oscs. This was fatal if the caller was ll_readpage who unlocked
+# the page, guarnateeing that the unlock from the RPC completion would
+# assert on trying to unlock the unlocked page.
+test_41() {
+ local f=$MOUNT/$tfile
+ # make sure the start of the file is ost1
+ lfs setstripe $f $((128 * 1024)) 0 0
+ do_facet client dd if=/dev/zero of=$f bs=4k count=1 || return 3
+ cancel_lru_locks OSC
+ # fail ost2 and read from ost1
+ local osc2_dev=`$LCTL device_list | \
+ awk '(/ost2.*client_facet/){print $4}' `
+ $LCTL --device %$osc2_dev deactivate
+ do_facet client dd if=$f of=/dev/null bs=4k count=1 || return 3
+ $LCTL --device %$osc2_dev activate
+ return 0
+}
+run_test 41 "read from a valid osc while other oscs are invalid"
+
+# test MDS recovery after ost failure
+test_42() {
+ blocks=`df $MOUNT | tail -1 | awk '{ print $1 }'`
+ createmany -o $DIR/$tfile-%d 800
+ replay_barrier ost
+ unlinkmany $DIR/$tfile-%d 0 400
+ facet_failover ost
+
+ # osc is evicted, fs is smaller
+ blocks_after=`df $MOUNT | tail -1 | awk '{ print $1 }'`
+ [ $blocks_after -lt $blocks ] || return 1
+ echo wait for MDS to timeout and recover
+ sleep $((TIMEOUT * 2))
+ unlinkmany $DIR/$tfile-%d 400 400
+ $CHECKSTAT -t file $DIR/$tfile-* && return 1 || true
+}
+run_test 42 "recoery after ost failure"
+
+# b=2530
+# directory orphans can't be unlinked from PENDING directory
+test_43() {
+ replay_barrier mds
+
+ # OBD_FAIL_OST_CREATE_NET 0x204
+ do_facet ost "sysctl -w lustre.fail_loc=0x80000204"
+ facet_failover mds
+ df $MOUNT || return 1
+ sleep 10
+ do_facet ost "sysctl -w lustre.fail_loc=0"
+
+ return 0
+}
+run_test 43 "mds osc import failure during recovery; don't LBUG"
+
+test_44() {
+ mdcdev=`awk '/mds_svc_MNT/ {print $1}' < /proc/fs/lustre/devices`
+ do_facet mds "sysctl -w lustre.fail_loc=0x80000701"
+ $LCTL --device $mdcdev recover
+ df $MOUNT
+ do_facet mds "sysctl -w lustre.fail_loc=0"
+ return 0
+}
+run_test 44 "race in target handle connect"
+
equals_msg test complete, cleaning up
-cleanup
+$CLEANUP
+
git://git.whamcloud.com / fs / lustre-release.git / blobdiff