X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=blobdiff_plain;f=lustre%2Ftests%2Freplay-single.sh;h=1124d693201ee9e88082acb2d0c16fc3eebde2b6;hp=06e2ab64051c6e1ffd8ea6cd8ffdc2e8af0861e0;hb=d968245894506f9620dfc652e9f8fdec0476cff6;hpb=85c25e844fd8253b124255aaecce4c2f58ab8498 diff --git a/lustre/tests/replay-single.sh b/lustre/tests/replay-single.sh index 06e2ab6..1124d69 100755 --- a/lustre/tests/replay-single.sh +++ b/lustre/tests/replay-single.sh @@ -1,135 +1,118 @@ -#!/bin/sh +#!/bin/bash set -e +#set -v # # This test needs to be run on the client # - +SAVE_PWD=$PWD LUSTRE=${LUSTRE:-`dirname $0`/..} +SETUP=${SETUP:-} +CLEANUP=${CLEANUP:-} . $LUSTRE/tests/test-framework.sh - init_test_env $@ +. ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh} +CHECK_GRANT=${CHECK_GRANT:-"yes"} +GRANT_CHECK_LIST=${GRANT_CHECK_LIST:-""} -. ${CONFIG:=$LUSTRE/tests/cfg/lmv.sh} - -build_test_filter - -assert_env MDSCOUNT # Skip these tests -# 46 - The MDS will always have to force close the cached opens -ALWAYS_EXCEPT="46" - -if [ `using_krb5_sec $SECURITY` == 'n' ] ; then - ALWAYS_EXCEPT="0c $ALWAYS_EXCEPT" -fi - - -gen_config() { - rm -f $XMLCONFIG - - if [ "$MDSCOUNT" -gt 1 ]; then - add_lmv lmv1_svc - for mds in `mds_list`; do - MDSDEV=$TMP/${mds}-`hostname` - add_mds $mds --dev $MDSDEV --size $MDSSIZE --lmv lmv1_svc - done - add_lov_to_lmv lov1 lmv1_svc --stripe_sz $STRIPE_BYTES \ - --stripe_cnt $STRIPES_PER_OBJ --stripe_pattern 0 - MDS=lmv1 - else - add_mds $SINGLEMDS --dev $MDSDEV --size $MDSSIZE - add_lov lov1 $SINGLEMDS --stripe_sz $STRIPE_BYTES \ - --stripe_cnt $STRIPES_PER_OBJ --stripe_pattern 0 - MDS=$SINGLEMDS - fi - - add_ost ost --lov lov1 --dev $OSTDEV --size $OSTSIZE - add_ost ost2 --lov lov1 --dev ${OSTDEV}-2 --size $OSTSIZE - add_client client $MDS --lov lov1 --path $MOUNT -} +# bug number: 2766 4176 +ALWAYS_EXCEPT="0b 39 $REPLAY_SINGLE_EXCEPT" build_test_filter -cleanup() { - # make sure we are using the primary MDS, so the config log will - # be able to clean up properly. - activemds=`facet_active $SINGLEMDS` - if [ $activemds != "$SINGLEMDS" ]; then - fail $SINGLEMDS - fi - zconf_umount `hostname` $MOUNT - for mds in `mds_list`; do - stop $mds ${FORCE} $MDSLCONFARGS - done - stop ost2 ${FORCE} --dump cleanup.log - stop ost ${FORCE} --dump cleanup.log - stop_lgssd - stop_lsvcgssd -} - -if [ "$ONLY" == "cleanup" ]; then - sysctl -w portals.debug=0 || true - cleanup - exit -fi - -SETUP=${SETUP:-"setup"} -CLEANUP=${CLEANUP:-"cleanup"} - -setup() { - gen_config - - start_krb5_kdc || exit 1 - start_lsvcgssd || exit 2 - start_lgssd || exit 3 - start ost --reformat $OSTLCONFARGS - start ost2 --reformat $OSTLCONFARGS - [ "$DAEMONFILE" ] && $LCTL debug_daemon start $DAEMONFILE $DAEMONSIZE - for mds in `mds_list`; do - start $mds --reformat $MDSLCONFARGS - done - grep " $MOUNT " /proc/mounts || zconf_mount `hostname` $MOUNT -} - -$SETUP - -if [ "$ONLY" == "setup" ]; then - exit 0 -fi +cleanup_and_setup_lustre mkdir -p $DIR test_0() { + sleep 10 + mkdir $DIR/$tfile replay_barrier $SINGLEMDS fail $SINGLEMDS + rmdir $DIR/$tfile } run_test 0 "empty replay" test_0b() { # this test attempts to trigger a race in the precreation code, # and must run before any other objects are created on the filesystem - fail ost + fail ost1 createmany -o $DIR/$tfile 20 || return 1 unlinkmany $DIR/$tfile 20 || return 2 } run_test 0b "ensure object created after recover exists. (3284)" +seq_set_width() +{ + local mds=$1 + local width=$2 + local file=`ls /proc/fs/lustre/seq/cli-srv-$mds-mdc-*/width` + echo $width > $file +} + +seq_get_width() +{ + local mds=$1 + local file=`ls /proc/fs/lustre/seq/cli-srv-$mds-mdc-*/width` + cat $file +} + +# This test should pass for single-mds and multi-mds configs. +# But for different configurations it tests different things. +# +# single-mds +# ---------- +# (1) fld_create replay should happen; +# +# (2) fld_create replay should not return -EEXISTS, if it does +# this means sequence manager recovery code is buggy and allocated +# same sequence two times after recovery. +# +# multi-mds +# --------- +# (1) fld_create replay may not happen, because its home MDS is +# MDS2 which is not involved to revovery; +# +# (2) as fld_create does not happen on MDS1, it does not make any +# problem. test_0c() { - if [ `using_krb5_sec $SECURITY` == 'n' ] ; then - echo "Skip 0c in non-gss mode" - return 0 - fi - # drop gss error notification - replay_barrier $SINGLEMDS - fail_drop $SINGLEMDS 0x760 + local label=`mdsdevlabel 1` + [ -z "$label" ] && echo "No label for mds1" && return 1 - # drop gss init request replay_barrier $SINGLEMDS - fail_drop $SINGLEMDS 0x780 + local sw=`seq_get_width $label` + + # make seq manager switch to next sequence each + # time as new fid is needed. + seq_set_width $label 1 + + # make sure that fld has created at least one new + # entry on server + touch $DIR/$tfile || return 2 + seq_set_width $label $sw + + # fail $SINGLEMDS and start recovery, replay RPCs, etc. + fail $SINGLEMDS + + # wait for recovery finish + sleep 10 + df $MOUNT + + # flush fld cache and dentry cache to make it lookup + # created entry instead of revalidating existent one + umount $MOUNT + zconf_mount `hostname` $MOUNT + + # issue lookup which should call fld lookup which + # should fail if client did not replay fld create + # correctly and server has no fld entry + touch $DIR/$tfile || return 3 + rm $DIR/$tfile || return 4 } -run_test 0c "empty replay with gss init failures" +run_test 0c "fld create" test_1() { replay_barrier $SINGLEMDS @@ -150,7 +133,7 @@ test_2a() { run_test 2a "touch" test_2b() { - ./mcreate $DIR/$tfile + mcreate $DIR/$tfile replay_barrier $SINGLEMDS touch $DIR/$tfile fail $SINGLEMDS @@ -172,9 +155,9 @@ run_test 3a "replay failed open(O_DIRECTORY)" test_3b() { replay_barrier $SINGLEMDS #define OBD_FAIL_MDS_OPEN_PACK | OBD_FAIL_ONCE - do_facet mds "sysctl -w lustre.fail_loc=0x80000114" + do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0x80000114" touch $DIR/$tfile - do_facet mds "sysctl -w lustre.fail_loc=0" + do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0" fail $SINGLEMDS $CHECKSTAT -t file $DIR/$tfile && return 2 return 0 @@ -184,9 +167,9 @@ run_test 3b "replay failed open -ENOMEM" test_3c() { replay_barrier $SINGLEMDS #define OBD_FAIL_MDS_ALLOC_OBDO | OBD_FAIL_ONCE - do_facet mds "sysctl -w lustre.fail_loc=0x80000128" + do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0x80000128" touch $DIR/$tfile - do_facet mds "sysctl -w lustre.fail_loc=0" + do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0" fail $SINGLEMDS $CHECKSTAT -t file $DIR/$tfile && return 2 @@ -264,6 +247,8 @@ test_7() { run_test 7 "mkdir |X| contained create" test_8() { + # make sure no side-effect from previous test. + rm -f $DIR/$tfile replay_barrier $SINGLEMDS multiop $DIR/$tfile mo_c & MULTIPID=$! @@ -302,9 +287,8 @@ test_10() { mv $DIR/$tfile $DIR/$tfile-2 rm -f $DIR/$tfile fail $SINGLEMDS - $CHECKSTAT $DIR/$tfile && return 1 - $CHECKSTAT $DIR/$tfile-2 || return 2 + $CHECKSTAT $DIR/$tfile-2 ||return 2 rm $DIR/$tfile-2 return 0 } @@ -474,6 +458,51 @@ test_20() { } run_test 20 "|X| open(O_CREAT), unlink, replay, close (test mds_cleanup_orphans)" +test_20b() { # bug 10480 + BEFOREUSED=`df -P $DIR | tail -1 | awk '{ print $3 }'` + + dd if=/dev/zero of=$DIR/$tfile bs=4k count=10000 & + pid=$! + while [ ! -e $DIR/$tfile ] ; do + usleep 60 # give dd a chance to start + done + + lfs getstripe $DIR/$tfile || return 1 + rm -f $DIR/$tfile || return 2 # make it an orphan + mds_evict_client + df -P $DIR || df -P $DIR || true # reconnect + + fail $SINGLEMDS # start orphan recovery + df -P $DIR || df -P $DIR || true # reconnect + wait_mds_recovery_done || error "MDS recovery not done" + + AFTERUSED=`df -P $DIR | tail -1 | awk '{ print $3 }'` + log "before $BEFOREUSED, after $AFTERUSED" + [ $AFTERUSED -gt $((BEFOREUSED + 20)) ] && \ + error "after $AFTERUSED > before $BEFOREUSED" && return 5 + return 0 +} +run_test 20b "write, unlink, eviction, replay, (test mds_cleanup_orphans)" + +test_20c() { # bug 10480 + multiop $DIR/$tfile Ow_c & + pid=$! + # give multiop a chance to open + sleep 1 + + ls -la $DIR/$tfile + + mds_evict_client + + df -P $DIR || df -P $DIR || true # reconnect + + kill -USR1 $pid + test -s $DIR/$tfile || error "File was truncated" + + return 0 +} +run_test 20c "check that client eviction does not affect file content" + test_21() { replay_barrier $SINGLEMDS multiop $DIR/$tfile O_tSc & @@ -699,15 +728,13 @@ test_32() { pid1=$! multiop $DIR/$tfile O_c & pid2=$! - # give multiop a chance to open. - # 1 second is not enough, I increased it to 5, however in ideal word - # I should have to wait for open finish in more smart manner. --umka - sleep 5 + # give multiop a chance to open + sleep 1 mds_evict_client df $MOUNT || sleep 1 && df $MOUNT || return 1 kill -USR1 $pid1 kill -USR1 $pid2 - sleep 5 + sleep 1 return 0 } run_test 32 "close() notices client eviction; close() after client eviction" @@ -715,14 +742,28 @@ run_test 32 "close() notices client eviction; close() after client eviction" # Abort recovery before client complete test_33() { replay_barrier $SINGLEMDS - touch $DIR/$tfile + createmany -o $DIR/$tfile-%d 100 fail_abort $SINGLEMDS # this file should be gone, because the replay was aborted - $CHECKSTAT -t file $DIR/$tfile && return 1 + $CHECKSTAT -t file $DIR/$tfile-* && return 3 + unlinkmany $DIR/$tfile-%d 0 100 return 0 } run_test 33 "abort recovery before client does replay" +# Stale FID sequence +test_33a() { + replay_barrier $SINGLEMDS + createmany -o $DIR/$tfile-%d 10 + fail_abort $SINGLEMDS + unlinkmany $DIR/$tfile-%d 0 10 + # recreate shouldn't fail + createmany -o $DIR/$tfile-%d 10 || return 3 + unlinkmany $DIR/$tfile-%d 0 10 + return 0 +} +run_test 33a "fid shouldn't be reused after abort recovery" + test_34() { multiop $DIR/$tfile O_c & pid=$! @@ -744,7 +785,7 @@ test_35() { touch $DIR/$tfile #define OBD_FAIL_MDS_REINT_NET_REP 0x119 - do_facet mds "sysctl -w lustre.fail_loc=0x80000119" + do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0x80000119" rm -f $DIR/$tfile & sleep 1 sync @@ -762,7 +803,7 @@ test_36() { touch $DIR/$tfile checkstat $DIR/$tfile facet_failover $SINGLEMDS - cancel_lru_locks MDC + cancel_lru_locks mdc if dmesg | grep "unknown lock cookie"; then echo "cancel after replay failed" return 1 @@ -802,7 +843,7 @@ test_38() { } run_test 38 "test recovery from unlink llog (test llog_gen_rec) " -test_39() { +test_39() { # bug 4176 createmany -o $DIR/$tfile-%d 800 replay_barrier $SINGLEMDS unlinkmany $DIR/$tfile-%d 0 400 @@ -814,8 +855,7 @@ test_39() { run_test 39 "test recovery from unlink llog (test llog_gen_rec) " count_ost_writes() { - cat /proc/fs/lustre/osc/*/stats | - awk -vwrites=0 '/ost_write/ { writes += $2 } END { print writes; }' + awk -vwrites=0 '/ost_write/ { writes += $2 } END { print writes; }' $LPROC/osc/*/stats } #b=2477,2532 @@ -828,7 +868,7 @@ test_40(){ sleep 1 facet_failover $SINGLEMDS #define OBD_FAIL_MDS_CONNECT_NET 0x117 - do_facet mds "sysctl -w lustre.fail_loc=0x80000117" + do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0x80000117" kill -USR1 $PID stat1=`count_ost_writes` sleep $TIMEOUT @@ -862,68 +902,92 @@ run_test 40 "cause recovery in ptlrpc, ensure IO continues" # the page, guarnateeing that the unlock from the RPC completion would # assert on trying to unlock the unlocked page. test_41() { + [ $OSTCOUNT -lt 2 ] && \ + skip "skipping test 41: we don't have a second OST to test with" && \ + return + local f=$MOUNT/$tfile # make sure the start of the file is ost1 lfs setstripe $f $((128 * 1024)) 0 0 do_facet client dd if=/dev/zero of=$f bs=4k count=1 || return 3 - cancel_lru_locks OSC + cancel_lru_locks osc # fail ost2 and read from ost1 - local osc2_dev=`$LCTL device_list | \ - awk '(/ost2.*client_facet/){print $4}' ` - $LCTL --device %$osc2_dev deactivate + local osc2dev=`do_facet mds "grep ${ost2_svc}-osc-MDT0000 $LPROC/devices" | awk '{print $1}'` + [ -z "$osc2dev" ] && echo "OST: $ost2_svc" && cat $LPROC/devices && return 4 + do_facet mds $LCTL --device $osc2dev deactivate || return 1 do_facet client dd if=$f of=/dev/null bs=4k count=1 || return 3 - $LCTL --device %$osc2_dev activate + do_facet mds $LCTL --device $osc2dev activate || return 2 return 0 } run_test 41 "read from a valid osc while other oscs are invalid" # test MDS recovery after ost failure test_42() { - blocks=`df $MOUNT | tail -n 1 | awk '{ print $1 }'` + blocks=`df -P $MOUNT | tail -n 1 | awk '{ print $2 }'` createmany -o $DIR/$tfile-%d 800 - replay_barrier ost + replay_barrier ost1 unlinkmany $DIR/$tfile-%d 0 400 - facet_failover ost + debugsave + sysctl -w lnet.debug=-1 + facet_failover ost1 - # osc is evicted, fs is smaller - blocks_after=`df $MOUNT | tail -n 1 | awk '{ print $1 }'` - [ $blocks_after -lt $blocks ] || return 1 + # osc is evicted, fs is smaller (but only with failout OSTs (bug 7287) + #blocks_after=`df -P $MOUNT | tail -n 1 | awk '{ print $2 }'` + #[ $blocks_after -lt $blocks ] || return 1 echo wait for MDS to timeout and recover sleep $((TIMEOUT * 2)) + debugrestore unlinkmany $DIR/$tfile-%d 400 400 $CHECKSTAT -t file $DIR/$tfile-* && return 2 || true } run_test 42 "recovery after ost failure" -# b=2530 # timeout in MDS/OST recovery RPC will LBUG MDS -test_43() { +test_43() { # bug 2530 replay_barrier $SINGLEMDS # OBD_FAIL_OST_CREATE_NET 0x204 - do_facet ost "sysctl -w lustre.fail_loc=0x80000204" - facet_failover $SINGLEMDS - df $MOUNT || return 1 + do_facet ost1 "sysctl -w lustre.fail_loc=0x80000204" + fail $SINGLEMDS sleep 10 - do_facet ost "sysctl -w lustre.fail_loc=0" + do_facet ost1 "sysctl -w lustre.fail_loc=0" return 0 } run_test 43 "mds osc import failure during recovery; don't LBUG" test_44() { - mdcdev=`awk '/mds_svc_MNT/ {print $1}' < /proc/fs/lustre/devices` - do_facet mds "sysctl -w lustre.fail_loc=0x80000701" - $LCTL --device $mdcdev recover - df $MOUNT - do_facet mds "sysctl -w lustre.fail_loc=0" + mdcdev=`awk '/MDT0000-mdc-/ {print $1}' $LPROC/devices` + [ "$mdcdev" ] || exit 2 + for i in `seq 1 10`; do + #define OBD_FAIL_TGT_CONN_RACE 0x701 + do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0x80000701" + $LCTL --device $mdcdev recover + df $MOUNT + done + do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0" return 0 } run_test 44 "race in target handle connect" +test_44b() { + mdcdev=`awk '/MDT0000-mdc-/ {print $1}' $LPROC/devices` + [ "$mdcdev" ] || exit 2 + for i in `seq 1 10`; do + #define OBD_FAIL_TGT_DELAY_RECONNECT 0x704 + do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0x80000704" + $LCTL --device $mdcdev recover + df $MOUNT + done + do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0" + return 0 +} +run_test 44b "race in target handle connect" + # Handle failed close test_45() { - mdcdev=`awk '/mds_svc_MNT/ {print $1}' < /proc/fs/lustre/devices` + mdcdev=`awk '/MDT0000-mdc-/ {print $1}' $LPROC/devices` + [ "$mdcdev" ] || exit 2 $LCTL --device $mdcdev recover multiop $DIR/$tfile O_c & @@ -932,13 +996,13 @@ test_45() { # This will cause the CLOSE to fail before even # allocating a reply buffer - $LCTL --device $mdcdev deactivate + $LCTL --device $mdcdev deactivate || return 4 # try the close kill -USR1 $pid wait $pid || return 1 - $LCTL --device $mdcdev activate + $LCTL --device $mdcdev activate || return 5 sleep 1 $CHECKSTAT -t file $DIR/$tfile || return 2 @@ -957,16 +1021,14 @@ test_46() { } run_test 46 "Don't leak file handle after open resend (3325)" -# b=2824 -test_47() { - +test_47() { # bug 2824 # create some files to make sure precreate has been done on all # OSTs. (just in case this test is run independently) createmany -o $DIR/$tfile 20 || return 1 # OBD_FAIL_OST_CREATE_NET 0x204 - fail ost - do_facet ost "sysctl -w lustre.fail_loc=0x80000204" + fail ost1 + do_facet ost1 "sysctl -w lustre.fail_loc=0x80000204" df $MOUNT || return 2 # let the MDS discover the OST failure, attempt to recover, fail @@ -977,355 +1039,139 @@ test_47() { createmany -o $DIR/$tfile 20 || return 3 unlinkmany $DIR/$tfile 20 || return 4 - do_facet ost "sysctl -w lustre.fail_loc=0" + do_facet ost1 "sysctl -w lustre.fail_loc=0" return 0 } run_test 47 "MDS->OSC failure during precreate cleanup (2824)" - test_48() { - createmany -o $DIR/${tfile}- 100 - $CHECKSTAT $DIR/${tfile}-99 || return 1 - mds_evict_client - df $MOUNT || echo "first df failed" - sleep 1 - df $MOUNT || return 2 - sleep 1 - $CHECKSTAT $DIR/${tfile}-99 || return 3 - - dmesg -c >/dev/null replay_barrier $SINGLEMDS + createmany -o $DIR/$tfile 20 || return 1 + # OBD_FAIL_OST_EROFS 0x216 fail $SINGLEMDS - unlinkmany $DIR/${tfile}- 100 || return 4 - if dmesg | grep "back in time"; then - echo "server went back in time!" - return 5 - fi - return 0 -} -run_test 48 "Don't lose transno when client is evicted (2525)" + do_facet ost1 "sysctl -w lustre.fail_loc=0x80000216" + df $MOUNT || return 2 -# b=3550 - replay of unlink -test_49() { - replay_barrier $SINGLEMDS - createmany -o $DIR/$tfile-%d 400 || return 1 - unlinkmany $DIR/$tfile-%d 0 400 || return 2 - fail $SINGLEMDS - $CHECKSTAT -t file $DIR/$tfile-* && return 3 || true + createmany -o $DIR/$tfile 20 20 || return 2 + unlinkmany $DIR/$tfile 40 || return 3 + + do_facet ost1 "sysctl -w lustre.fail_loc=0" + return 0 } -run_test 49 "re-write records to llog as written during fail" +run_test 48 "MDS->OSC failure during precreate cleanup (2824)" test_50() { - local osc_dev=`$LCTL device_list | \ - awk '(/ost_svc_$SINGLEMDS_svc/){print $4}' ` - $LCTL --device %$osc_dev recover && $LCTL --device %$osc_dev recover + local oscdev=`grep ${ost1_svc}-osc-MDT0000 $LPROC/devices | awk '{print $1}'` + [ "$oscdev" ] || return 1 + $LCTL --device $oscdev recover && $LCTL --device $oscdev recover # give the mds_lov_sync threads a chance to run sleep 5 } run_test 50 "Double OSC recovery, don't LASSERT (3812)" -# bug 3462 - simultaneous MDC requests -test_51a() { - replay_barrier_nodf $SINGLEMDS - mkdir -p $DIR/${tdir}-1 - mkdir -p $DIR/${tdir}-2 - touch $DIR/${tdir}-2/f - multiop $DIR/${tdir}-1/f O_c & - pid=$! - # give multiop a chance to open - sleep 1 - - do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0x80000115" - kill -USR1 $pid - do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0" - $CHECKSTAT -t file $DIR/${tdir}-2/f || return 1 - - fail $SINGLEMDS - - wait $pid || return 2 - $CHECKSTAT -t file $DIR/${tdir}-1/f || return 3 - rm -rf $DIR/${tdir}-* -} -run_test 51a "|X| close request while two MDC requests in flight" - -test_51b() { - replay_barrier_nodf $SINGLEMDS - mkdir -p $DIR/$tdir-1 - mkdir -p $DIR/$tdir-2 - multiop $DIR/$tdir-1/f O_c & - pid=$! - - # give multiop a chance to open - # 1 second seems to be not enough, we met already such a cases - # --umka - sleep 5 - - do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0x80000107" - touch $DIR/${tdir}-2/f & - usleep 500 - do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0" - - kill -USR1 $pid - wait $pid || return 1 - - fail $SINGLEMDS - - $CHECKSTAT -t file $DIR/${tdir}-1/f || return 2 - $CHECKSTAT -t file $DIR/${tdir}-2/f || return 3 - rm -rf $DIR/${tdir}-* -} -run_test 51b "|X| open request while two MDC requests in flight" - -test_51c() { - replay_barrier_nodf $SINGLEMDS - mkdir -p $DIR/${tdir}-1 - mkdir -p $DIR/${tdir}-2 - multiop $DIR/${tdir}-1/f O_c & - pid=$! - # give multiop a chance to open - sleep 1 - - do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0x80000107" - touch $DIR/${tdir}-2/f & - do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0" - - do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0x80000115" - kill -USR1 $pid - do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0" - - fail $SINGLEMDS - - wait $pid || return 1 - $CHECKSTAT -t file $DIR/${tdir}-1/f || return 2 - $CHECKSTAT -t file $DIR/${tdir}-2/f || return 3 - rm -rf $DIR/${tdir}-* -} -run_test 51c "|X| open request and close request while two MDC requests in flight" - -test_51d() { - replay_barrier_nodf $SINGLEMDS - mkdir -p $DIR/${tdir}-1 - mkdir -p $DIR/${tdir}-2 - touch $DIR/${tdir}-2/f - multiop $DIR/${tdir}-1/f O_c & - pid=$! - # give multiop a chance to open - sleep 1 - - do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0x80000122" - kill -USR1 $pid - do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0" - #$CHECKSTAT -t file $DIR/${tdir}-2/f || return 1 - - fail $SINGLEMDS - - wait $pid || return 2 - $CHECKSTAT -t file $DIR/${tdir}-1/f || return 3 - rm -rf $DIR/${tdir}-* -} -run_test 51d "|X| close reply while two MDC requests in flight" - -test_51e() { - replay_barrier_nodf $SINGLEMDS - mkdir -p $DIR/$tdir-1 - mkdir -p $DIR/$tdir-2 - multiop $DIR/$tdir-1/f O_c & - pid=$! - # give multiop a chance to open - sleep 1 - - do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0x80000119" - touch $DIR/${tdir}-2/f & - usleep 500 - do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0" - - kill -USR1 $pid - wait $pid || return 1 - - fail $SINGLEMDS - - $CHECKSTAT -t file $DIR/${tdir}-1/f || return 2 - $CHECKSTAT -t file $DIR/${tdir}-2/f || return 3 - rm -rf $DIR/${tdir}-* -} -run_test 51e "|X| open reply while two MDC requests in flight" - -test_51f() { - replay_barrier_nodf $SINGLEMDS - mkdir -p $DIR/${tdir}-1 - mkdir -p $DIR/${tdir}-2 - multiop $DIR/${tdir}-1/f O_c & - pid=$! - # give multiop a chance to open - sleep 1 - - do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0x80000119" - touch $DIR/${tdir}-2/f & - do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0" - - do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0x80000122" - kill -USR1 $pid - do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0" - - fail $SINGLEMDS - - wait $pid || return 1 - $CHECKSTAT -t file $DIR/${tdir}-1/f || return 2 - $CHECKSTAT -t file $DIR/${tdir}-2/f || return 3 - rm -rf $DIR/${tdir}-* -} -run_test 51f "|X| open reply and close reply while two MDC requests in flight" - -test_51g() { - replay_barrier_nodf $SINGLEMDS - mkdir -p $DIR/${tdir}-1 - mkdir -p $DIR/${tdir}-2 - multiop $DIR/${tdir}-1/f O_c & - pid=$! - # give multiop a chance to open - sleep 1 - - do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0x80000119" - touch $DIR/${tdir}-2/f & - do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0" - - do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0x80000115" - kill -USR1 $pid - do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0" - - fail $SINGLEMDS - - wait $pid || return 1 - $CHECKSTAT -t file $DIR/${tdir}-1/f || return 2 - $CHECKSTAT -t file $DIR/${tdir}-2/f || return 3 - rm -rf $DIR/${tdir}-* -} -run_test 51g "|X| open reply and close request while two MDC requests in flight" - -test_51h() { - replay_barrier_nodf $SINGLEMDS - mkdir -p $DIR/${tdir}-1 - mkdir -p $DIR/${tdir}-2 - multiop $DIR/${tdir}-1/f O_c & - pid=$! - # give multio:wp a chance to open - sleep 1 - - do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0x80000107" - touch $DIR/${tdir}-2/f & - do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0" - - do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0x80000122" - kill -USR1 $pid - do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0" - - fail $SINGLEMDS - - wait $pid || return 1 - $CHECKSTAT -t file $DIR/${tdir}-1/f || return 2 - $CHECKSTAT -t file $DIR/${tdir}-2/f || return 3 - rm -rf $DIR/${tdir}-* -} -run_test 51h "|X| open request and close reply while two MDC requests in flight" - # b3764 timed out lock replay test_52() { touch $DIR/$tfile - cancel_lru_locks MDC + cancel_lru_locks mdc - multiop $DIR/$tfile s + multiop $DIR/$tfile s || return 1 replay_barrier $SINGLEMDS +#define OBD_FAIL_LDLM_REPLY 0x30c do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0x8000030c" - fail $SINGLEMDS + fail $SINGLEMDS || return 2 do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0x0" $CHECKSTAT -t file $DIR/$tfile-* && return 3 || true } run_test 52 "time out lock replay (3764)" -test_53() { - replay_barrier_nodf $SINGLEMDS - f1=$DIR/${tfile}-1 - cat < $f1 -#!/bin/sh -true -EOF - chmod +x $f1 - do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0x80000107" - $f1 || return 1 - do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0" - - fail $SINGLEMDS - rm -f $f1 -} -run_test 53 "|X| open request and close reply while two MDC requests in flight" +#b_cray 53 "|X| open request and close reply while two MDC requests in flight" +#b_cray 54 "|X| open request and close reply while two MDC requests in flight" -test_54() { - replay_barrier $SINGLEMDS - createmany -o $DIR/$tfile 20 - unlinkmany $DIR/$tfile 20 - fail $SINGLEMDS +#b3761 ASSERTION(hash != 0) failed +test_55() { +# OBD_FAIL_MDS_OPEN_CREATE | OBD_FAIL_ONCE + do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0x8000012b" + touch $DIR/$tfile & + # give touch a chance to run + sleep 5 + do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0x0" + rm $DIR/$tfile + return 0 } -run_test 54 "|X| open request and close reply while two MDC requests in flight" +run_test 55 "let MDS_CHECK_RESENT return the original return code instead of 0" #b3440 ASSERTION(rec->ur_fid2->id) failed -test_55() { - sysctl -w portals.debug=-1 portals.debug_mb=25 +test_56() { ln -s foo $DIR/$tfile replay_barrier $SINGLEMDS #drop_reply "cat $DIR/$tfile" fail $SINGLEMDS sleep 10 } -run_test 55 "don't replay a symlink open request (3440)" +run_test 56 "don't replay a symlink open request (3440)" -#b3761 ASSERTION(hash != 0) failed -test_56() { -# OBD_FAIL_MDS_OPEN_CREATE | OBD_FAIL_ONCE - do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0x8000012b" +#recovery one mds-ost setattr from llog +test_57() { +#define OBD_FAIL_MDS_OST_SETATTR 0x12c + do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0x8000012c" touch $DIR/$tfile - pid=$! - # give a chance for touch to run - sleep 5 + replay_barrier $SINGLEMDS + fail $SINGLEMDS + sleep 1 + $CHECKSTAT -t file $DIR/$tfile || return 1 do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0x0" - wait $pid || return 1 rm $DIR/$tfile - return 0 } -run_test 56 "let MDS_CHECK_RESENT return the original return code instead of 0" +run_test 57 "test recovery from llog for setattr op" -#b7312 LASSERT(!IS_ERR(parent)) in reconstruct_open() -test_57() { - mkdir $DIR/$tdir || return 1 - touch $DIR/$tdir/$tfile || return 2 - multiop $DIR/$tdir/$tfile o_ & - MULTIPID=$! - sleep 1 - rm -f $DIR/$tdir/$tfile || return 3 - rm -rf $DIR/$tdir || return 4 - # drop first reint reply - sysctl -w lustre.fail_loc=0x0000030c - facet_failover $SINGLEMDS - df $MOUNT || return 1 - kill -USR1 $MULTIPID || return 5 - wait $MULTIPID || return 6 - sysctl -w lustre.fail_loc=0 +#recovery many mds-ost setattr from llog +test_58() { +#define OBD_FAIL_MDS_OST_SETATTR 0x12c + do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0x8000012c" + mkdir $DIR/$tdir + createmany -o $DIR/$tdir/$tfile-%d 2500 + replay_barrier $SINGLEMDS + fail $SINGLEMDS + sleep 2 + $CHECKSTAT -t file $DIR/$tdir/$tfile-* || return 1 + do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0x0" + unlinkmany $DIR/$tdir/$tfile-%d 2500 + rmdir $DIR/$tdir } -run_test 57 "open orphan in reconstruct_open()" +run_test 58 "test recovery from llog for setattr op (test llog_gen_rec)" -test_58() { - capa=`cat /proc/fs/lustre/mds/mds*/capa` - if [ "$capa" == "0" ]; then - echo "skip testing - no capa enabled" - return 0 - fi - sysctl -w lustre.fail_loc=0x0000030c - rc=multiop $DIR/$tfile Ow - sysctl -w lustre.fail_loc=0 - return rc +# log_commit_thread vs filter_destroy race used to lead to import use after free +# bug 11658 +test_59() { + mkdir $DIR/$tdir + createmany -o $DIR/$tdir/$tfile-%d 200 + sync + unlinkmany $DIR/$tdir/$tfile-%d 200 +#define OBD_FAIL_PTLRPC_DELAY_RECOV 0x507 + do_facet ost1 "sysctl -w lustre.fail_loc=0x507" + fail ost1 + fail $SINGLEMDS + do_facet ost1 "sysctl -w lustre.fail_loc=0x0" + sleep 20 + rmdir $DIR/$tdir } -run_test 58 "open+create resent should return capa" +run_test 59 "test log_commit_thread vs filter_destroy race" -equals_msg test complete, cleaning up -$CLEANUP +# race between add unlink llog vs cat log init in post_recovery (only for b1_6) +# bug 12086: should no oops and No ctxt error for this test +test_60() { + mkdir $DIR/$tdir + createmany -o $DIR/$tdir/$tfile-%d 200 + replay_barrier $SINGLEMDS + unlinkmany $DIR/$tdir/$tfile-%d 0 100 + fail $SINGLEMDS + unlinkmany $DIR/$tdir/$tfile-%d 100 100 + local no_ctxt=`dmesg | grep "No ctxt"` + [ -z "$no_ctxt" ] || error "ctxt is not initialized in recovery" +} +run_test 60 "test llog post recovery init vs llog unlink" +equals_msg `basename $0`: test complete, cleaning up +check_and_cleanup_lustre +[ -f "$TESTSUITELOG" ] && cat $TESTSUITELOG || true