-
#!/bin/bash
set -e
-# bug 11190 5494 7288 5493
-ALWAYS_EXCEPT="19b 24 27 52 $RECOVERY_SMALL_EXCEPT"
+# bug 5494 7288 5493
+ALWAYS_EXCEPT="24 27 52 $RECOVERY_SMALL_EXCEPT"
+
+# also long tests: 19, 21a, 21e, 21f, 23, 27
+# 1 2.5 4 4 (min)"
+[ "$SLOW" = "no" ] && EXCEPT_SLOW="17 26b 50 51 57"
PTLDEBUG=${PTLDEBUG:--1}
LUSTRE=${LUSTRE:-`dirname $0`/..}
# Allow us to override the setup if we already have a mounted system by
# setting SETUP=" " and CLEANUP=" "
-SETUP=${SETUP:-"setup"}
-CLEANUP=${CLEANUP:-"cleanup"}
-
-setup() {
- formatall
- setupall
-}
-
-cleanup() {
- cleanupall || { echo "FAILed to clean up"; exit 20; }
-}
+SETUP=${SETUP:-""}
+CLEANUP=${CLEANUP:-""}
-if [ ! -z "$EVAL" ]; then
- eval "$EVAL"
- exit $?
-fi
-
-if [ "$ONLY" == "cleanup" ]; then
- sysctl -w lnet.debug=0 || true
- cleanup
- exit
-fi
-
-$SETUP
-
-[ "$ONLY" == "setup" ] && exit
+cleanup_and_setup_lustre
+rm -rf $DIR/${TESTSUITE}/[df][0-9]* # bug 13798 new t-f tdir staff
+rm -rf $DIR/[df][0-9]*
test_1() {
drop_request "mcreate $MOUNT/1" || return 1
#b=2494
test_12(){
$LCTL mark multiop $MOUNT/$tfile OS_c
- do_facet mds "sysctl -w lustre.fail_loc=0x115"
- clear_failloc mds $((TIMEOUT * 2)) &
+ do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0x115"
+ clear_failloc $SINGLEMDS $((TIMEOUT * 2)) &
multiop $MOUNT/$tfile OS_c &
PID=$!
#define OBD_FAIL_MDS_CLOSE_NET 0x115
mkdir $MOUNT/readdir || return 1
touch $MOUNT/readdir/newentry || return
# OBD_FAIL_MDS_READPAGE_NET|OBD_FAIL_ONCE
- do_facet mds "sysctl -w lustre.fail_loc=0x80000104"
+ do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0x80000104"
ls $MOUNT/readdir || return 3
- do_facet mds "sysctl -w lustre.fail_loc=0"
+ do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0"
rm -rf $MOUNT/readdir || return 4
}
run_test 13 "mdc_readpage restart test (bug 1138)"
mkdir $MOUNT/readdir
touch $MOUNT/readdir/newentry
# OBD_FAIL_MDS_SENDPAGE|OBD_FAIL_ONCE
- do_facet mds "sysctl -w lustre.fail_loc=0x80000106"
+ do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0x80000106"
ls $MOUNT/readdir || return 1
- do_facet mds "sysctl -w lustre.fail_loc=0"
+ do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0"
}
run_test 14 "mdc_readpage resend test (bug 1138)"
test_15() {
- do_facet mds "sysctl -w lustre.fail_loc=0x80000128"
+ do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0x80000128"
touch $DIR/$tfile && return 1
return 0
}
stop_read_ahead
#define OBD_FAIL_PTLRPC_BULK_PUT_NET 0x504 | OBD_FAIL_ONCE
- do_facet ost1 sysctl -w lustre.fail_loc=0x80000504
+ do_facet ost1 "sysctl -w lustre.fail_loc=0x80000504"
cancel_lru_locks osc
# OST bulk will time out here, client resends
do_facet client "cmp /etc/termcap $MOUNT/termcap" || return 1
- sysctl -w lustre.fail_loc=0
+ do_facet ost1 sysctl -w lustre.fail_loc=0
# give recovery a chance to finish (shouldn't take long)
sleep $TIMEOUT
do_facet client "cmp /etc/termcap $MOUNT/termcap" || return 2
test_17() {
# OBD_FAIL_PTLRPC_BULK_GET_NET 0x0503 | OBD_FAIL_ONCE
# OST bulk will time out here, client retries
- sysctl -w lustre.fail_loc=0x80000503
+ do_facet ost1 sysctl -w lustre.fail_loc=0x80000503
# need to ensure we send an RPC
do_facet client cp /etc/termcap $DIR/$tfile
sync
sleep $TIMEOUT
- sysctl -w lustre.fail_loc=0
+ do_facet ost1 sysctl -w lustre.fail_loc=0
do_facet client "df $DIR"
# expect cmp to succeed, client resent bulk
do_facet client "cmp /etc/termcap $DIR/$tfile" || return 3
run_test 17 "timeout bulk get, don't evict client (2732)"
test_18a() {
- [ -z ${ost2_svc} ] && echo Skipping, needs 2 osts && return 0
+ [ -z ${ost2_svc} ] && skip "needs 2 osts" && return 0
do_facet client mkdir -p $MOUNT/$tdir
f=$MOUNT/$tdir/$tfile
pgcache_empty || return 1
# 1 stripe on ost2
- lfs setstripe $f $((128 * 1024)) 1 1
+ lfs setstripe $f -s $((128 * 1024)) -i 1 -c 1
do_facet client cp /etc/termcap $f
sync
- local osc2dev=`grep ${ost2_svc}-osc- $LPROC/devices | awk '{print $1}'`
+ local osc2dev=`grep ${ost2_svc}-osc- $LPROC/devices | egrep -v 'MDT' | awk '{print $1}'`
$LCTL --device $osc2dev deactivate || return 3
# my understanding is that there should be nothing in the page
# cache after the client reconnects?
pgcache_empty || return 1
# shouldn't have to set stripe size of count==1
- lfs setstripe $f $((128 * 1024)) 0 1
- lfs setstripe $f2 $((128 * 1024)) 0 1
+ lfs setstripe $f -s $((128 * 1024)) -i 0 -c 1
+ lfs setstripe $f2 -s $((128 * 1024)) -i 0 -c 1
do_facet client cp /etc/termcap $f
sync
multiop $DIR/$tdir-1/f O_c &
close_pid=$!
- do_facet mds "sysctl -w lustre.fail_loc=0x80000129"
+ do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0x80000129"
multiop $DIR/$tdir-2/f Oc &
open_pid=$!
sleep 1
- do_facet mds "sysctl -w lustre.fail_loc=0"
+ do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0"
- do_facet mds "sysctl -w lustre.fail_loc=0x80000115"
+ do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0x80000115"
kill -USR1 $close_pid
cancel_lru_locks mdc
wait $close_pid || return 1
wait $open_pid || return 2
- do_facet mds "sysctl -w lustre.fail_loc=0"
+ do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0"
$CHECKSTAT -t file $DIR/$tdir-1/f || return 3
$CHECKSTAT -t file $DIR/$tdir-2/f || return 4
multiop $DIR/$tdir-1/f O_c &
close_pid=$!
- do_facet mds "sysctl -w lustre.fail_loc=0x80000107"
+ do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0x80000107"
mcreate $DIR/$tdir-2/f &
open_pid=$!
sleep 1
- do_facet mds "sysctl -w lustre.fail_loc=0"
+ do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0"
kill -USR1 $close_pid
cancel_lru_locks mdc
multiop $DIR/$tdir-1/f O_c &
close_pid=$!
- do_facet mds "sysctl -w lustre.fail_loc=0x80000107"
+ do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0x80000107"
mcreate $DIR/$tdir-2/f &
open_pid=$!
sleep 3
- do_facet mds "sysctl -w lustre.fail_loc=0"
+ do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0"
- do_facet mds "sysctl -w lustre.fail_loc=0x80000115"
+ do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0x80000115"
kill -USR1 $close_pid
cancel_lru_locks mdc
wait $close_pid || return 1
wait $open_pid || return 2
- do_facet mds "sysctl -w lustre.fail_loc=0"
+ do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0"
$CHECKSTAT -t file $DIR/$tdir-1/f || return 2
$CHECKSTAT -t file $DIR/$tdir-2/f || return 3
multiop $DIR/$tdir-1/f O_c &
pid=$!
- do_facet mds "sysctl -w lustre.fail_loc=0x80000129"
+ do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0x80000129"
multiop $DIR/$tdir-2/f Oc &
sleep 1
- do_facet mds "sysctl -w lustre.fail_loc=0"
+ do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0"
- do_facet mds "sysctl -w lustre.fail_loc=0x80000122"
+ do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0x80000122"
kill -USR1 $pid
cancel_lru_locks mdc
wait $pid || return 1
- do_facet mds "sysctl -w lustre.fail_loc=0"
+ do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0"
$CHECKSTAT -t file $DIR/$tdir-1/f || return 2
$CHECKSTAT -t file $DIR/$tdir-2/f || return 3
multiop $DIR/$tdir-1/f O_c &
pid=$!
- do_facet mds "sysctl -w lustre.fail_loc=0x80000119"
+ do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0x80000119"
touch $DIR/$tdir-2/f &
sleep 1
- do_facet mds "sysctl -w lustre.fail_loc=0"
+ do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0"
kill -USR1 $pid
cancel_lru_locks mdc
multiop $DIR/$tdir-1/f O_c &
pid=$!
- do_facet mds "sysctl -w lustre.fail_loc=0x80000119"
+ do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0x80000119"
touch $DIR/$tdir-2/f &
sleep 1
- do_facet mds "sysctl -w lustre.fail_loc=0"
+ do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0"
- do_facet mds "sysctl -w lustre.fail_loc=0x80000122"
+ do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0x80000122"
kill -USR1 $pid
cancel_lru_locks mdc
wait $pid || return 1
- do_facet mds "sysctl -w lustre.fail_loc=0"
+ do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0"
$CHECKSTAT -t file $DIR/$tdir-1/f || return 2
$CHECKSTAT -t file $DIR/$tdir-2/f || return 3
multiop $DIR/$tdir-1/f O_c &
pid=$!
- do_facet mds "sysctl -w lustre.fail_loc=0x80000119"
+ do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0x80000119"
touch $DIR/$tdir-2/f &
sleep 1
- do_facet mds "sysctl -w lustre.fail_loc=0"
+ do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0"
- do_facet mds "sysctl -w lustre.fail_loc=0x80000115"
+ do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0x80000115"
kill -USR1 $pid
cancel_lru_locks mdc
wait $pid || return 1
- do_facet mds "sysctl -w lustre.fail_loc=0"
+ do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0"
$CHECKSTAT -t file $DIR/$tdir-1/f || return 2
$CHECKSTAT -t file $DIR/$tdir-2/f || return 3
multiop $DIR/$tdir-1/f O_c &
pid=$!
- do_facet mds "sysctl -w lustre.fail_loc=0x80000107"
+ do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0x80000107"
touch $DIR/$tdir-2/f &
touch_pid=$!
sleep 1
- do_facet mds "sysctl -w lustre.fail_loc=0"
+ do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0"
- do_facet mds "sysctl -w lustre.fail_loc=0x80000122"
+ do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0x80000122"
cancel_lru_locks mdc
kill -USR1 $pid
wait $pid || return 1
- do_facet mds "sysctl -w lustre.fail_loc=0"
+ do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0"
wait $touch_pid || return 2
f1=$DIR/${tfile}-1
f2=$DIR/${tfile}-2
- do_facet mds "sysctl -w lustre.fail_loc=0x80000115"
+ do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0x80000115"
multiop $f2 Oc &
close_pid=$!
multiop $f1 msu || return 1
cancel_lru_locks mdc
- do_facet mds "sysctl -w lustre.fail_loc=0"
+ do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0"
wait $close_pid || return 2
rm -rf $f2 || return 4
# try the close
drop_request "kill -USR1 $pid"
- fail mds
+ fail $SINGLEMDS
wait $pid || return 1
return 0
}
rc=$?
sysctl -w lustre.fail_loc=0x0
client_reconnect
- [ $rc -eq 0 ] && error "multiop didn't fail fsync: rc $rc" || true
+ [ $rc -eq 0 ] && error_ignore 5494 "multiop didn't fail fsync: rc $rc" || true
}
run_test 24 "fsync error (should return error)"
test_26() { # bug 5921 - evict dead exports by pinger
# this test can only run from a client on a separate node.
- [ "`lsmod | grep obdfilter`" ] && \
- echo "skipping test 26 (local OST)" && return
- [ "`lsmod | grep mds`" ] && \
- echo "skipping test 26 (local MDS)" && return
+ remote_ost || skip "local OST" && return
+ remote_mds || skip "local MDS" && return
OST_FILE=$LPROC/obdfilter/${ost1_svc}/num_exports
OST_EXP="`do_facet ost1 cat $OST_FILE`"
OST_NEXP1=`echo $OST_EXP | cut -d' ' -f2`
run_test 26 "evict dead exports"
test_26b() { # bug 10140 - evict dead exports by pinger
- zconf_mount `hostname` $MOUNT2
- MDS_FILE=$LPROC/mds/${mds_svc}/num_exports
- MDS_NEXP1="`do_facet mds cat $MDS_FILE | cut -d' ' -f2`"
+ client_df
+ zconf_mount `hostname` $MOUNT2 || error "Failed to mount $MOUNT2"
+ sleep 1 # wait connections being established
+ MDS_FILE=$LPROC/mdt/${mds1_svc}/num_exports
+ MDS_NEXP1="`do_facet $SINGLEMDS cat $MDS_FILE | cut -d' ' -f2`"
OST_FILE=$LPROC/obdfilter/${ost1_svc}/num_exports
OST_NEXP1="`do_facet ost1 cat $OST_FILE | cut -d' ' -f2`"
echo starting with $OST_NEXP1 OST and $MDS_NEXP1 MDS exports
echo Waiting for $(($TIMEOUT * 4)) secs
sleep $(($TIMEOUT * 4))
OST_NEXP2="`do_facet ost1 cat $OST_FILE | cut -d' ' -f2`"
- MDS_NEXP2="`do_facet mds cat $MDS_FILE | cut -d' ' -f2`"
+ MDS_NEXP2="`do_facet $SINGLEMDS cat $MDS_FILE | cut -d' ' -f2`"
echo ending with $OST_NEXP2 OST and $MDS_NEXP2 MDS exports
[ $OST_NEXP1 -le $OST_NEXP2 ] && error "client not evicted from OST"
[ $MDS_NEXP1 -le $MDS_NEXP2 ] && error "client not evicted from MDS"
run_test 26b "evict dead exports"
test_27() {
- [ "`lsmod | grep mds`" ] || \
- { echo "skipping test 27 (non-local MDS)" && return 0; }
+ remote_mds && { skip "remote MDS" && return 0; }
mkdir -p $DIR/$tdir
writemany -q -a $DIR/$tdir/$tfile 0 5 &
CLIENT_PID=$!
sleep 1
FAILURE_MODE="SOFT"
- facet_failover mds
+ facet_failover $SINGLEMDS
#define OBD_FAIL_OSC_SHUTDOWN 0x407
sysctl -w lustre.fail_loc=0x80000407
# need to wait for reconnect
sleep 1
echo -n .
done
- facet_failover mds
+ facet_failover $SINGLEMDS
#no crashes allowed!
kill -USR1 $CLIENT_PID
wait $CLIENT_PID
test_28() { # bug 6086 - error adding new clients
do_facet client mcreate $MOUNT/$tfile || return 1
drop_bl_callback "chmod 0777 $MOUNT/$tfile" ||echo "evicted as expected"
- #define OBD_FAIL_MDS_ADD_CLIENT 0x12f
- do_facet mds sysctl -w lustre.fail_loc=0x8000012f
+ #define OBD_FAIL_MDS_CLIENT_ADD 0x12f
+ do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0x8000012f"
# fail once (evicted), reconnect fail (fail_loc), ok
- df || (sleep 1; df) || (sleep 1; df) || error "reconnect failed"
+ df || (sleep 10; df) || (sleep 10; df) || error "reconnect failed"
rm -f $MOUNT/$tfile
- fail mds # verify MDS last_rcvd can be loaded
+ fail $SINGLEMDS # verify MDS last_rcvd can be loaded
}
run_test 28 "handle error adding new clients (bug 6086)"
echo writemany pid $CLIENT_PID
sleep 10
FAILURE_MODE="SOFT"
- fail mds
+ fail $SINGLEMDS
# wait for client to reconnect to MDS
sleep 60
- fail mds
+ fail $SINGLEMDS
sleep 60
- fail mds
+ fail $SINGLEMDS
# client process should see no problems even though MDS went down
sleep $TIMEOUT
kill -USR1 $CLIENT_PID
rc=$?
echo writemany returned $rc
#these may fail because of eviction due to slow AST response.
- return $rc
+ [ $rc -eq 0 ] || error_ignore 13652 "writemany returned rc $rc" || true
}
run_test 50 "failover MDS under load"
CLIENT_PID=$!
sleep 1
FAILURE_MODE="SOFT"
- facet_failover mds
+ facet_failover $SINGLEMDS
# failover at various points during recovery
SEQ="1 5 10 $(seq $TIMEOUT 5 $(($TIMEOUT+10)))"
echo will failover at $SEQ
do
echo failover in $i sec
sleep $i
- facet_failover mds
+ facet_failover $SINGLEMDS
done
# client process should see no problems even though MDS went down
# and recovery was interrupted
wait $CLIENT_PID
rc=$?
echo writemany returned $rc
- return $rc
+ [ $rc -eq 0 ] || error_ignore 13652 "writemany returned rc $rc" || true
}
run_test 51 "failover MDS during recovery"
# test of open reconstruct
test_53() {
touch $DIR/$tfile
- drop_ldlm_reply "./openfile -f O_RDWR:O_CREAT -m 0755 $DIR/$tfile" ||\
+ drop_ldlm_reply "openfile -f O_RDWR:O_CREAT -m 0755 $DIR/$tfile" ||\
return 2
}
run_test 53 "touch: drop rep"
touch $DIR2/$tfile.1
sleep 10
cat $DIR2/$tfile.missing # save transno = 0, rc != 0 into last_rcvd
- fail mds
+ fail $SINGLEMDS
umount $MOUNT2
ERROR=`dmesg | egrep "(test 54|went back in time)" | tail -n1 | grep "went back in time"`
[ x"$ERROR" == x ] || error "back in time occured"
# bug 11330 - liblustre application death during I/O locks up OST
test_55() {
- [ "`lsmod | grep obdfilter`" ] || \
- { echo "skipping test 55 (non-local OST)" && return 0; }
+ remote_ost && { skip "remote OST" && return 0; }
mkdir -p $DIR/$tdir
test_56() { # b=11277
#define OBD_FAIL_MDS_RESEND 0x136
- touch $MOUNT/$tfile
- do_facet mds sysctl -w lustre.fail_loc=0x80000136
- stat $MOUNT/$tfile
- do_facet mds sysctl -w lustre.fail_loc=0
- rm -f $MOUNT/$tfile
+ touch $DIR/$tfile
+ do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0x80000136"
+ stat $DIR/$tfile
+ do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0"
+ rm -f $DIR/$tfile
}
-run_test 56 "replace lock race"
+run_test 56 "do not allow reconnect to busy exports"
test_57_helper() {
# no oscs means no client or mdt
sysctl -w lustre.fail_loc=0x80000B00
zconf_umount `hostname` $DIR
sysctl -w lustre.fail_loc=0x80000B00
- fail_abort mds
+ fail_abort $SINGLEMDS
kill -9 $pid
sysctl -w lustre.fail_loc=0
mount_client $DIR
}
run_test 57 "read procfs entries causes kernel crash"
-$CLEANUP
-echo "$0: completed"
+test_58() { # bug 11546
+#define OBD_FAIL_MDC_ENQUEUE_PAUSE 0x801
+ touch $MOUNT/$tfile
+ ls -la $MOUNT/$tfile
+ sysctl -w lustre.fail_loc=0x80000801
+ cp $MOUNT/$tfile /dev/null &
+ pid=$!
+ sleep 1
+ sysctl -w lustre.fail_loc=0
+ drop_bl_callback rm -f $MOUNT/$tfile
+ wait $pid
+ do_facet client "df $DIR"
+}
+run_test 58 "Eviction in the middle of open RPC reply processing"
+
+test_59() { # bug 10589
+ zconf_mount `hostname` $MOUNT2 || error "Failed to mount $MOUNT2"
+#define OBD_FAIL_LDLM_CANCEL_EVICT_RACE 0x311
+ sysctl -w lustre.fail_loc=0x311
+ writes=`dd if=/dev/zero of=$DIR2/$tfile count=1 2>&1 | awk 'BEGIN { FS="+" } /out/ {print $1}'`
+ sysctl -w lustre.fail_loc=0
+ sync
+ zconf_umount `hostname` $DIR2 -f
+ reads=`dd if=$DIR/$tfile of=/dev/null 2>&1 | awk 'BEGIN { FS="+" } /in/ {print $1}'`
+ [ $reads -eq $writes ] || error "read" $reads "blocks, must be" $writes
+}
+run_test 59 "Read cancel race on client eviction"
+
+equals_msg `basename $0`: test complete, cleaning up
+check_and_cleanup_lustre
+[ -f "$TESTSUITELOG" ] && cat $TESTSUITELOG || true