. $LUSTRE/tests/test-framework.sh
init_test_env $@
. ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
+init_logging
+
+if [ "$FAILURE_MODE" = "HARD" ] && mixed_ost_devs; then
+ CONFIG_EXCEPTIONS="52"
+ echo -n "Several ost services on one ost node are used with FAILURE_MODE=$FAILURE_MODE. "
+ echo "Except the tests: $CONFIG_EXCEPTIONS"
+ ALWAYS_EXCEPT="$ALWAYS_EXCEPT $CONFIG_EXCEPTIONS"
+fi
+
+require_dsh_mds || exit 0
# also long tests: 19, 21a, 21e, 21f, 23, 27
# 1 2.5 2.5 4 4 (min)"
SETUP=${SETUP:-""}
CLEANUP=${CLEANUP:-""}
-cleanup_and_setup_lustre
+check_and_setup_lustre
+
assert_DIR
rm -rf $DIR/[df][0-9]*
#bug 1420
test_9() {
+ remote_ost_nodsh && skip "remote OST with nodsh" && return 0
+
pause_bulk "cp /etc/profile $DIR/$tfile" || return 1
do_facet client "cp $SAMPLE_FILE $DIR/${tfile}.2" || return 2
do_facet client "sync"
}
test_16() {
+ remote_ost_nodsh && skip "remote OST with nodsh" && return 0
+
do_facet client cp $SAMPLE_FILE $DIR
sync
stop_read_ahead
test_17() {
local at_max_saved=0
+ remote_ost_nodsh && skip "remote OST with nodsh" && return 0
+
# With adaptive timeouts, bulk_get won't expire until adaptive_timeout_max
- if at_is_valid && at_is_enabled; then
+ if at_is_enabled; then
at_max_saved=$(at_max_get ost1)
at_max_set $TIMEOUT ost1
fi
# expect cmp to succeed, client resent bulk
do_facet client "cmp /etc/termcap $DIR/$tfile" || return 3
do_facet client "rm $DIR/$tfile" || return 4
- [ $at_max_saved -ne 0 ] && $(at_max_set $at_max_saved ost1)
+ [ $at_max_saved -ne 0 ] && at_max_set $at_max_saved ost1
return 0
}
run_test 17 "timeout bulk get, don't evict client (2732)"
test_18a() {
- [ -z ${ost2_svc} ] && skip "needs 2 osts" && return 0
+ [ -z ${ost2_svc} ] && skip_env "needs 2 osts" && return 0
do_facet client mkdir -p $DIR/$tdir
f=$DIR/$tdir/$tfile
# 1 stripe on ost2
lfs setstripe $f -s $((128 * 1024)) -i 1 -c 1
+ get_stripe_info client $f
+ if [ $stripe_index -ne 1 ]; then
+ lfs getstripe $f
+ error "$f: different stripe offset ($stripe_index)" && return
+ fi
do_facet client cp $SAMPLE_FILE $f
sync
run_test 18a "manual ost invalidate clears page cache immediately"
test_18b() {
+ remote_ost_nodsh && skip "remote OST with nodsh" && return 0
+
do_facet client mkdir -p $DIR/$tdir
f=$DIR/$tdir/$tfile
- f2=$DIR/$tdir/${tfile}-2
cancel_lru_locks osc
pgcache_empty || return 1
# shouldn't have to set stripe size of count==1
lfs setstripe $f -s $((128 * 1024)) -i 0 -c 1
- lfs setstripe $f2 -s $((128 * 1024)) -i 0 -c 1
+ get_stripe_info client $f
+ if [ $stripe_index -ne 0 ]; then
+ lfs getstripe $f
+ error "$f: different stripe offset ($stripe_index)" && return
+ fi
do_facet client cp $SAMPLE_FILE $f
sync
# cache after the client reconnects?
rc=0
pgcache_empty || rc=2
- rm -f $f $f2
+ rm -f $f
return $rc
}
run_test 18b "eviction and reconnect clears page cache (2766)"
test_18c() {
+ remote_ost_nodsh && skip "remote OST with nodsh" && return 0
+
do_facet client mkdir -p $DIR/$tdir
f=$DIR/$tdir/$tfile
- f2=$DIR/$tdir/${tfile}-2
cancel_lru_locks osc
pgcache_empty || return 1
# shouldn't have to set stripe size of count==1
lfs setstripe $f -s $((128 * 1024)) -i 0 -c 1
- lfs setstripe $f2 -s $((128 * 1024)) -i 0 -c 1
+ get_stripe_info client $f
+ if [ $stripe_index -ne 0 ]; then
+ lfs getstripe $f
+ error "$f: different stripe offset ($stripe_index)" && return
+ fi
do_facet client cp $SAMPLE_FILE $f
sync
# cache after the client reconnects?
rc=0
pgcache_empty || rc=2
- rm -f $f $f2
+ rm -f $f
return $rc
}
run_test 18c "Dropped connect reply after eviction handing (14755)"
run_test 19b "test expired_lock_main on ost (2867)"
test_20a() { # bug 2983 - ldlm_handle_enqueue cleanup
+ remote_ost_nodsh && skip "remote OST with nodsh" && return 0
+
mkdir -p $DIR/$tdir
lfs setstripe $DIR/$tdir/${tfile} -i 0 -c 1
multiop_bg_pause $DIR/$tdir/${tfile} O_wc || return 1
run_test 20a "ldlm_handle_enqueue error (should return error)"
test_20b() { # bug 2986 - ldlm_handle_enqueue error during open
+ remote_ost_nodsh && skip "remote OST with nodsh" && return 0
+
mkdir -p $DIR/$tdir
lfs setstripe $DIR/$tdir/${tfile} -i 0 -c 1
cancel_lru_locks osc
run_test 23 "client hang when close a file after mds crash"
test_24() { # bug 11710 details correct fsync() behavior
+ remote_ost_nodsh && skip "remote OST with nodsh" && return 0
+
mkdir -p $DIR/$tdir
lfs setstripe $DIR/$tdir -s 0 -i 0 -c 1
cancel_lru_locks osc
}
run_test 24 "fsync error (should return error)"
+wait_client_evicted () {
+ local facet=$1
+ local exports=$2
+ local varsvc=${facet}_svc
+
+ wait_update $(facet_host $facet) "lctl get_param -n *.${!varsvc}.num_exports | cut -d' ' -f2" $((exports - 1)) $3
+}
+
test_26a() { # was test_26 bug 5921 - evict dead exports by pinger
# this test can only run from a client on a separate node.
remote_ost || { skip "local OST" && return 0; }
+ remote_ost_nodsh && skip "remote OST with nodsh" && return 0
remote_mds || { skip "local MDS" && return 0; }
- OST_FILE=obdfilter.${ost1_svc}.num_exports
- OST_EXP="`do_facet ost1 lctl get_param -n $OST_FILE`"
- OST_NEXP1=`echo $OST_EXP | cut -d' ' -f2`
- echo starting with $OST_NEXP1 OST exports
+
+ if [ $(facet_host mgs) = $(facet_host ost1) ]; then
+ skip "msg and ost1 are at the same node"
+ return 0
+ fi
+
+ check_timeout || return 1
+
+ local OST_NEXP=$(do_facet ost1 lctl get_param -n obdfilter.${ost1_svc}.num_exports | cut -d' ' -f2)
+
+ echo starting with $OST_NEXP OST exports
# OBD_FAIL_PTLRPC_DROP_RPC 0x505
do_facet client lctl set_param fail_loc=0x505
# evictor takes PING_EVICT_TIMEOUT + 3 * PING_INTERVAL to evict.
# But if there's a race to start the evictor from various obds,
# the loser might have to wait for the next ping.
- echo Waiting for $(($TIMEOUT * 8)) secs
- sleep $(($TIMEOUT * 8))
- OST_EXP="`do_facet ost1 lctl get_param -n $OST_FILE`"
- OST_NEXP2=`echo $OST_EXP | cut -d' ' -f2`
- echo ending with $OST_NEXP2 OST exports
+
+ local rc=0
+ wait_client_evicted ost1 $OST_NEXP $((TIMEOUT * 2 + TIMEOUT * 3 / 4))
+ rc=$?
do_facet client lctl set_param fail_loc=0x0
- [ $OST_NEXP1 -le $OST_NEXP2 ] && error "client not evicted"
- return 0
+ [ $rc -eq 0 ] || error "client not evicted from OST"
}
run_test 26a "evict dead exports"
test_26b() { # bug 10140 - evict dead exports by pinger
- client_df
- zconf_mount `hostname` $MOUNT2 || error "Failed to mount $MOUNT2"
- sleep 1 # wait connections being established
- MDS_FILE=mdt.${mds1_svc}.num_exports
- MDS_NEXP1="`do_facet $SINGLEMDS lctl get_param -n $MDS_FILE | cut -d' ' -f2`"
- OST_FILE=obdfilter.${ost1_svc}.num_exports
- OST_NEXP1="`do_facet ost1 lctl get_param -n $OST_FILE | cut -d' ' -f2`"
- echo starting with $OST_NEXP1 OST and $MDS_NEXP1 MDS exports
- zconf_umount `hostname` $MOUNT2 -f
- # evictor takes PING_EVICT_TIMEOUT + 3 * PING_INTERVAL to evict.
- # But if there's a race to start the evictor from various obds,
- # the loser might have to wait for the next ping.
- echo Waiting for $(($TIMEOUT * 3)) secs
- sleep $(($TIMEOUT * 3))
- OST_NEXP2="`do_facet ost1 lctl get_param -n $OST_FILE | cut -d' ' -f2`"
- MDS_NEXP2="`do_facet $SINGLEMDS lctl get_param -n $MDS_FILE | cut -d' ' -f2`"
- echo ending with $OST_NEXP2 OST and $MDS_NEXP2 MDS exports
- [ $OST_NEXP1 -le $OST_NEXP2 ] && error "client not evicted from OST"
- [ $MDS_NEXP1 -le $MDS_NEXP2 ] && error "client not evicted from MDS"
- return 0
+ remote_ost_nodsh && skip "remote OST with nodsh" && return 0
+
+ if [ $(facet_host mgs) = $(facet_host ost1) ]; then
+ skip "msg and ost1 are at the same node"
+ return 0
+ fi
+
+ check_timeout || return 1
+ clients_up
+ zconf_mount `hostname` $MOUNT2 ||
+ { error "Failed to mount $MOUNT2"; return 2; }
+ sleep 1 # wait connections being established
+
+ local MDS_NEXP=$(do_facet $SINGLEMDS lctl get_param -n mdt.${mds1_svc}.num_exports | cut -d' ' -f2)
+ local OST_NEXP=$(do_facet ost1 lctl get_param -n obdfilter.${ost1_svc}.num_exports | cut -d' ' -f2)
+
+ echo starting with $OST_NEXP OST and $MDS_NEXP MDS exports
+
+ zconf_umount `hostname` $MOUNT2 -f
+
+ # PING_INTERVAL max(obd_timeout / 4, 1U)
+ # PING_EVICT_TIMEOUT (PING_INTERVAL * 6)
+
+ # evictor takes PING_EVICT_TIMEOUT + 3 * PING_INTERVAL to evict.
+ # But if there's a race to start the evictor from various obds,
+ # the loser might have to wait for the next ping.
+ # = 9 * PING_INTERVAL + PING_INTERVAL
+ # = 10 PING_INTERVAL = 10 obd_timeout / 4 = 2.5 obd_timeout
+ # let's wait $((TIMEOUT * 3)) # bug 19887
+ local rc=0
+ wait_client_evicted ost1 $OST_NEXP $((TIMEOUT * 3)) || \
+ error "Client was not evicted by ost" rc=1
+ wait_client_evicted $SINGLEMDS $MDS_NEXP $((TIMEOUT * 3)) || \
+ error "Client was not evicted by mds"
}
run_test 26b "evict dead exports"
test_27() {
- remote_mds && { skip "remote MDS" && return 0; }
mkdir -p $DIR/$tdir
writemany -q -a $DIR/$tdir/$tfile 0 5 &
CLIENT_PID=$!
sleep 1
+ local save_FAILURE_MODE=$FAILURE_MODE
FAILURE_MODE="SOFT"
facet_failover $SINGLEMDS
#define OBD_FAIL_OSC_SHUTDOWN 0x407
- lctl set_param fail_loc=0x80000407
+ do_facet $SINGLEMDS lctl set_param fail_loc=0x80000407
# need to wait for reconnect
echo -n waiting for fail_loc
- while [ `lctl get_param -n fail_loc` -eq -2147482617 ]; do
+ while [ $(do_facet $SINGLEMDS lctl get_param -n fail_loc) -eq -2147482617 ]; do
sleep 1
echo -n .
done
+ do_facet $SINGLEMDS lctl get_param -n fail_loc
facet_failover $SINGLEMDS
#no crashes allowed!
kill -USR1 $CLIENT_PID
wait $CLIENT_PID
true
+ FAILURE_MODE=$save_FAILURE_MODE
}
run_test 27 "fail LOV while using OSC's"
#define OBD_FAIL_MDS_CLIENT_ADD 0x12f
do_facet $SINGLEMDS "lctl set_param fail_loc=0x8000012f"
# fail once (evicted), reconnect fail (fail_loc), ok
- df || (sleep 10; df) || (sleep 10; df) || error "reconnect failed"
+ client_up || (sleep 10; client_up) || (sleep 10; client_up) || error "reconnect failed"
rm -f $DIR/$tfile
fail $SINGLEMDS # verify MDS last_rcvd can be loaded
}
run_test 28 "handle error adding new clients (bug 6086)"
+test_29a() { # bug 22273 - error adding new clients
+ #define OBD_FAIL_TGT_CLIENT_ADD 0x711
+ do_facet $SINGLEMDS "lctl set_param fail_loc=0x80000711"
+ # fail abort so client will be new again
+ fail_abort $SINGLEMDS
+ client_up || error "reconnect failed"
+ return 0
+}
+run_test 29a "error adding new clients doesn't cause LBUG (bug 22273)"
+
+test_29b() { # bug 22273 - error adding new clients
+ #define OBD_FAIL_TGT_CLIENT_ADD 0x711
+ do_facet ost1 "lctl set_param fail_loc=0x80000711"
+ # fail abort so client will be new again
+ fail_abort ost1
+ client_up || error "reconnect failed"
+ return 0
+}
+run_test 29b "error adding new clients doesn't cause LBUG (bug 22273)"
+
test_50() {
mkdir -p $DIR/$tdir
# put a load of file creates/writes/deletes
}
test_52() {
+ remote_ost_nodsh && skip "remote OST with nodsh" && return 0
+
mkdir -p $DIR/$tdir
test_52_guts
rc=$?
# bug 11330 - liblustre application death during I/O locks up OST
test_55() {
- remote_ost && { skip "remote OST" && return 0; }
+ remote_ost_nodsh && skip "remote OST with nodsh" && return 0
mkdir -p $DIR/$tdir
# first dd should be finished quickly
+ lfs setstripe $DIR/$tdir/$tfile-1 -c 1 -i 0
dd if=/dev/zero of=$DIR/$tdir/$tfile-1 bs=32M count=4 &
DDPID=$!
count=0
done
echo "(dd_pid=$DDPID, time=$count)successful"
- #define OBD_FAIL_OST_DROP_REQ 0x21d
- do_facet ost lctl set_param fail_loc=0x0000021d
+ lfs setstripe $DIR/$tdir/$tfile-2 -c 1 -i 0
+ #define OBD_FAIL_OST_DROP_REQ 0x21d
+ do_facet ost1 lctl set_param fail_loc=0x0000021d
# second dd will be never finished
dd if=/dev/zero of=$DIR/$tdir/$tfile-2 bs=32M count=4 &
DDPID=$!
echo "(dd_pid=$DDPID, time=$count)successful"
#Recover fail_loc and dd will finish soon
- do_facet ost lctl set_param fail_loc=0
+ do_facet ost1 lctl set_param fail_loc=0
count=0
echo "step3: testing ......"
while [ true ]; do
}
run_test 59 "Read cancel race on client eviction"
+err17935 () {
+ # we assume that all md changes are in the MDT0 changelog
+ if [ $MDSCOUNT -gt 1 ]; then
+ error_ignore 17935 $*
+ else
+ error $*
+ fi
+}
+
+test_60() {
+ MDT0=$($LCTL get_param -n mdc.*.mds_server_uuid | \
+ awk '{gsub(/_UUID/,""); print $1}' | head -1)
+
+ NUM_FILES=15000
+ mkdir -p $DIR/$tdir
+
+ # Register (and start) changelog
+ USER=$(do_facet $SINGLEMDS lctl --device $MDT0 changelog_register -n)
+ echo "Registered as $MDT0 changelog user $USER"
+
+ # Generate a large number of changelog entries
+ createmany -o $DIR/$tdir/$tfile $NUM_FILES
+ sync
+ sleep 5
+
+ # Unlink files in the background
+ unlinkmany $DIR/$tdir/$tfile $NUM_FILES &
+ CLIENT_PID=$!
+ sleep 1
+
+ # Failover the MDS while unlinks are happening
+ facet_failover $SINGLEMDS
+
+ # Wait for unlinkmany to finish
+ wait $CLIENT_PID
+
+ # Check if all the create/unlink events were recorded
+ # in the changelog
+ $LFS changelog $MDT0 >> $DIR/$tdir/changelog
+ local cl_count=$(grep UNLNK $DIR/$tdir/changelog | wc -l)
+ echo "$cl_count unlinks in $MDT0 changelog"
+
+ do_facet $SINGLEMDS lctl --device $MDT0 changelog_deregister $USER
+ USERS=$(( $(do_facet $SINGLEMDS lctl get_param -n \
+ mdd.$MDT0.changelog_users | wc -l) - 2 ))
+ if [ $USERS -eq 0 ]; then
+ [ $cl_count -eq $NUM_FILES ] || \
+ err17935 "Recorded ${cl_count} unlinks out of $NUM_FILES"
+ # Also make sure we can clear large changelogs
+ cl_count=$($LFS changelog $FSNAME | wc -l)
+ [ $cl_count -le 2 ] || \
+ error "Changelog not empty: $cl_count entries"
+ else
+ # If there are other users, there may be other unlinks in the log
+ [ $cl_count -ge $NUM_FILES ] || \
+ err17935 "Recorded ${cl_count} unlinks out of $NUM_FILES"
+ echo "$USERS other changelog users; can't verify clear"
+ fi
+}
+run_test 60 "Add Changelog entries during MDS failover"
+
+test_61()
+{
+ local cflags='osc.*-OST0000-osc-MDT*.connect_flags'
+ do_facet $SINGLEMDS "lctl get_param -n $cflags" |grep -q skip_orphan
+ [ $? -ne 0 ] && skip "don't have skip orphan feature" && return
+
+ mkdir -p $DIR/$tdir || error "mkdir dir $DIR/$tdir failed"
+ # Set the default stripe of $DIR/$tdir to put the files to ost1
+ $LFS setstripe -c 1 --index 0 $DIR/$tdir
+
+ replay_barrier $SINGLEMDS
+ createmany -o $DIR/$tdir/$tfile-%d 10
+ local oid=`do_facet ost1 "lctl get_param -n obdfilter.*OST0000.last_id"`
+
+ fail_abort $SINGLEMDS
+
+ touch $DIR/$tdir/$tfile
+ local id=`$LFS getstripe $DIR/$tdir/$tfile |awk '$2 ~ /^[1-9]+/ {print $2}'`
+ [ $id -le $oid ] && error "the orphan objid was reused, failed"
+
+ # Cleanup
+ rm -rf $DIR/$tdir
+}
+run_test 61 "Verify to not reuse orphan objects - bug 17025"
+
equals_msg `basename $0`: test complete, cleaning up
check_and_cleanup_lustre
-[ -f "$TESTSUITELOG" ] && cat $TESTSUITELOG || true
+[ -f "$TESTSUITELOG" ] && cat $TESTSUITELOG && grep -q FAIL $TESTSUITELOG && exit 1 || true
git://git.whamcloud.com / fs / lustre-release.git / blobdiff