# Skip these tests
# bug number for skipped tests:
-# b=17466/LU-472 : 61d
-# LU-5319 : 53a 53d
-# LU-6780 : 80d 80h 81d 81h 110e 110f 110g 111c 111d 111e 111f 111g 112
-ALWAYS_EXCEPT="61d 53a 53d 80d 80h 81d 81h 110e 110f 110g 111c 111d 111e 111f 111g 112 $REPLAY_SINGLE_EXCEPT"
+# b=17466/LU-472
+ALWAYS_EXCEPT="$REPLAY_SINGLE_EXCEPT 61d"
# UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT!
case "$(lsb_release -sr)" in # only disable tests for el7
# bug 3462 - simultaneous MDC requests
test_53a() {
+ [[ $(lctl get_param mdc.*.import |
+ grep "connect_flags:.*multi_mod_rpc") ]] ||
+ { skip "Need MDC with 'multi_mod_rpcs' feature"; return 0; }
+
cancel_lru_locks mdc # cleanup locks from former test cases
mkdir $DIR/${tdir}-1 || error "mkdir $DIR/${tdir}-1 failed"
mkdir $DIR/${tdir}-2 || error "mkdir $DIR/${tdir}-2 failed"
run_test 53c "|X| open request and close request while two MDC requests in flight"
test_53d() {
+ [[ $(lctl get_param mdc.*.import |
+ grep "connect_flags:.*multi_mod_rpc") ]] ||
+ { skip "Need MDC with 'multi_mod_rpcs' feature"; return 0; }
+
cancel_lru_locks mdc # cleanup locks from former test cases
mkdir $DIR/${tdir}-1 || error "mkdir $DIR/${tdir}-1 failed"
killall_process $clients "$prog" -0
}
-killall_process () {
- local clients=${1:-$(hostname)}
- local name=$2
- local signal=$3
- local rc=0
-
- do_nodes $clients "killall $signal $name"
-}
-
test_70b () {
local clients=${CLIENTS:-$HOSTNAME}
run_test 70b "dbench ${MDSCOUNT}mdts recovery; $CLIENTCOUNT clients"
# end multi-client tests
+random_fail_mdt() {
+ local max_index=$1
+ local duration=$2
+ local monitor_pid=$3
+ local elapsed
+ local start_ts=$(date +%s)
+ local num_failovers=0
+ local fail_index
+
+ elapsed=$(($(date +%s) - start_ts))
+ while [ $elapsed -lt $duration ]; do
+ fail_index=$((RANDOM%max_index+1))
+ kill -0 $monitor_pid ||
+ error "$monitor_pid stopped"
+ sleep 120
+ replay_barrier mds$fail_index
+ sleep 10
+ # Increment the number of failovers
+ num_failovers=$((num_failovers+1))
+ log "$TESTNAME fail mds$fail_index $num_failovers times"
+ fail mds$fail_index
+ elapsed=$(($(date +%s) - start_ts))
+ done
+}
+
+cleanup_70c() {
+ trap 0
+ kill -9 $tar_70c_pid
+}
+test_70c () {
+ local clients=${CLIENTS:-$HOSTNAME}
+ local rc=0
+
+ zconf_mount_clients $clients $MOUNT
+
+ local duration=300
+ [ "$SLOW" = "no" ] && duration=180
+ # set duration to 900 because it takes some time to boot node
+ [ "$FAILURE_MODE" = HARD ] && duration=600
+
+ local elapsed
+ local start_ts=$(date +%s)
+
+ trap cleanup_70c EXIT
+ (
+ while true; do
+ test_mkdir -p -c$MDSCOUNT $DIR/$tdir || break
+ if [ $MDSCOUNT -ge 2 ]; then
+ $LFS setdirstripe -D -c$MDSCOUNT $DIR/$tdir ||
+ error "set default dirstripe failed"
+ fi
+ cd $DIR/$tdir || break
+ tar cf - /etc | tar xf - || error "tar failed"
+ cd $DIR || break
+ rm -rf $DIR/$tdir || break
+ done
+ )&
+ tar_70c_pid=$!
+ echo "Started tar $tar_70c_pid"
+
+ random_fail_mdt $MDSCOUNT $duration $tar_70c_pid
+ kill -0 $tar_70c_pid || error "tar $tar_70c_pid stopped"
+
+ cleanup_70c
+ true
+}
+run_test 70c "tar ${MDSCOUNT}mdts recovery"
+
+cleanup_70d() {
+ trap 0
+ kill -9 $mkdir_70d_pid
+}
+
+test_70d () {
+ [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
+ local clients=${CLIENTS:-$HOSTNAME}
+ local rc=0
+
+ zconf_mount_clients $clients $MOUNT
+
+ local duration=300
+ [ "$SLOW" = "no" ] && duration=180
+ # set duration to 900 because it takes some time to boot node
+ [ "$FAILURE_MODE" = HARD ] && duration=900
+
+ mkdir -p $DIR/$tdir
+
+ local elapsed
+ local start_ts=$(date +%s)
+
+ trap cleanup_70d EXIT
+ (
+ while true; do
+ $LFS mkdir -i0 -c2 $DIR/$tdir/test || {
+ echo "mkdir fails"
+ break
+ }
+ $LFS mkdir -i1 -c2 $DIR/$tdir/test1 || {
+ echo "mkdir fails"
+ break
+ }
+
+ touch $DIR/$tdir/test/a || {
+ echo "touch fails"
+ break;
+ }
+ mkdir $DIR/$tdir/test/b || {
+ echo "mkdir fails"
+ break;
+ }
+ rm -rf $DIR/$tdir/test || {
+ echo "rmdir fails"
+ break
+ }
+
+ touch $DIR/$tdir/test1/a || {
+ echo "touch fails"
+ break;
+ }
+ mkdir $DIR/$tdir/test1/b || {
+ echo "mkdir fails"
+ break;
+ }
+
+ rm -rf $DIR/$tdir/test1 || {
+ echo "rmdir fails"
+ break
+ }
+ done
+ )&
+ mkdir_70d_pid=$!
+ echo "Started $mkdir_70d_pid"
+
+ random_fail_mdt $MDSCOUNT $duration $mkdir_70d_pid
+ kill -0 $mkdir_70d_pid || error "mkdir/rmdir $mkdir_70d_pid stopped"
+
+ cleanup_70d
+ true
+}
+run_test 70d "mkdir/rmdir striped dir ${MDSCOUNT}mdts recovery"
+
+cleanup_70e() {
+ trap 0
+ kill -9 $rename_70e_pid
+}
+
+test_70e () {
+ [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
+ local clients=${CLIENTS:-$HOSTNAME}
+ local rc=0
+
+ echo ha > /proc/sys/lnet/debug
+ zconf_mount_clients $clients $MOUNT
+
+ local duration=300
+ [ "$SLOW" = "no" ] && duration=180
+ # set duration to 900 because it takes some time to boot node
+ [ "$FAILURE_MODE" = HARD ] && duration=900
+
+ mkdir -p $DIR/$tdir
+ $LFS mkdir -i0 $DIR/$tdir/test_0
+ $LFS mkdir -i0 $DIR/$tdir/test_1
+ touch $DIR/$tdir/test_0/a
+ touch $DIR/$tdir/test_1/b
+ trap cleanup_70e EXIT
+ (
+ while true; do
+ mrename $DIR/$tdir/test_0/a $DIR/$tdir/test_1/b > \
+ /dev/null || {
+ echo "a->b fails"
+ break;
+ }
+
+ checkstat $DIR/$tdir/test_0/a && {
+ echo "a still exists"
+ break
+ }
+
+ checkstat $DIR/$tdir/test_1/b || {
+ echo "b still exists"
+ break
+ }
+
+ touch $DIR/$tdir/test_0/a || {
+ echo "touch a fails"
+ break
+ }
+
+ mrename $DIR/$tdir/test_1/b $DIR/$tdir/test_0/a > \
+ /dev/null || {
+ echo "a->a fails"
+ break;
+ }
+ done
+ )&
+ rename_70e_pid=$!
+ echo "Started $rename_70e_pid"
+
+ random_fail_mdt 2 $duration $rename_70e_pid
+ kill -0 $rename_70e_pid || error "rename $rename_70e_pid stopped"
+
+ cleanup_70e
+ true
+}
+run_test 70e "rename cross-MDT with random fails"
+
+cleanup_71a() {
+ trap 0
+ kill -9 $mkdir_71a_pid
+}
+
+random_double_fail_mdt() {
+ local max_index=$1
+ local duration=$2
+ local monitor_pid=$3
+ local elapsed
+ local start_ts=$(date +%s)
+ local num_failovers=0
+ local fail_index
+ local second_index
+
+ elapsed=$(($(date +%s) - start_ts))
+ while [ $elapsed -lt $duration ]; do
+ fail_index=$((RANDOM%max_index + 1))
+ if [ $fail_index -eq $max_index ]; then
+ second_index=1
+ else
+ second_index=$((fail_index + 1))
+ fi
+ kill -0 $monitor_pid ||
+ error "$monitor_pid stopped"
+ sleep 120
+ replay_barrier mds$fail_index
+ replay_barrier mds$second_index
+ sleep 10
+ # Increment the number of failovers
+ num_failovers=$((num_failovers+1))
+ log "fail mds$fail_index mds$second_index $num_failovers times"
+ fail mds${fail_index},mds${second_index}
+ elapsed=$(($(date +%s) - start_ts))
+ done
+}
+
+test_71a () {
+ [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
+ local clients=${CLIENTS:-$HOSTNAME}
+ local rc=0
+
+ zconf_mount_clients $clients $MOUNT
+
+ local duration=300
+ [ "$SLOW" = "no" ] && duration=180
+ # set duration to 900 because it takes some time to boot node
+ [ "$FAILURE_MODE" = HARD ] && duration=900
+
+ mkdir -p $DIR/$tdir
+
+ local elapsed
+ local start_ts=$(date +%s)
+
+ trap cleanup_71a EXIT
+ (
+ while true; do
+ $LFS mkdir -i0 -c2 $DIR/$tdir/test
+ rmdir $DIR/$tdir/test
+ done
+ )&
+ mkdir_71a_pid=$!
+ echo "Started $mkdir_71a_pid"
+
+ random_double_fail_mdt 2 $duration $mkdir_71a_pid
+ kill -0 $mkdir_71a_pid || error "mkdir/rmdir $mkdir_71a_pid stopped"
+
+ cleanup_71a
+ true
+}
+run_test 71a "mkdir/rmdir striped dir with 2 mdts recovery"
+
test_73a() {
multiop_bg_pause $DIR/$tfile O_tSc ||
error "multiop_bg_pause $DIR/$tfile failed"
}
run_test 101 "Shouldn't reassign precreated objs to other files after recovery"
+test_102a() {
+ local idx
+ local facet
+ local num
+ local i
+ local pids pid
+
+ [[ $(lctl get_param mdc.*.import |
+ grep "connect_flags:.*multi_mod_rpc") ]] ||
+ { skip "Need MDC with 'multi_mod_rpcs' feature"; return 0; }
+
+ $LFS mkdir -c1 $DIR/$tdir || error "mkdir $DIR/$tdir failed"
+ idx=$(printf "%04x" $($LFS getdirstripe -i $DIR/$tdir))
+ facet="mds$((0x$idx + 1))"
+
+ # get current value of max_mod_rcps_in_flight
+ num=$($LCTL get_param -n \
+ mdc.$FSNAME-MDT$idx-mdc-*.max_mod_rpcs_in_flight)
+ # set default value if client does not support multi mod RPCs
+ [ -z "$num" ] && num=1
+
+ echo "creating $num files ..."
+ umask 0022
+ for i in $(seq $num); do
+ touch $DIR/$tdir/file-$i
+ done
+
+ # drop request on MDT to force resend
+ #define OBD_FAIL_MDS_REINT_MULTI_NET 0x159
+ do_facet $facet "$LCTL set_param fail_loc=0x159"
+ echo "launch $num chmod in parallel ($(date +%H:%M:%S)) ..."
+ for i in $(seq $num); do
+ chmod 0600 $DIR/$tdir/file-$i &
+ pids="$pids $!"
+ done
+ sleep 1
+ do_facet $facet "$LCTL set_param fail_loc=0"
+ for pid in $pids; do
+ wait $pid || error "chmod failed"
+ done
+ echo "done ($(date +%H:%M:%S))"
+
+ # check chmod succeed
+ for i in $(seq $num); do
+ checkstat -vp 0600 $DIR/$tdir/file-$i
+ done
+
+ rm -rf $DIR/$tdir
+}
+run_test 102a "check resend (request lost) with multiple modify RPCs in flight"
+
+test_102b() {
+ local idx
+ local facet
+ local num
+ local i
+ local pids pid
+
+ [[ $(lctl get_param mdc.*.import |
+ grep "connect_flags:.*multi_mod_rpc") ]] ||
+ { skip "Need MDC with 'multi_mod_rpcs' feature"; return 0; }
+
+ $LFS mkdir -c1 $DIR/$tdir || error "mkdir $DIR/$tdir failed"
+ idx=$(printf "%04x" $($LFS getdirstripe -i $DIR/$tdir))
+ facet="mds$((0x$idx + 1))"
+
+ # get current value of max_mod_rcps_in_flight
+ num=$($LCTL get_param -n \
+ mdc.$FSNAME-MDT$idx-mdc-*.max_mod_rpcs_in_flight)
+ # set default value if client does not support multi mod RPCs
+ [ -z "$num" ] && num=1
+
+ echo "creating $num files ..."
+ umask 0022
+ for i in $(seq $num); do
+ touch $DIR/$tdir/file-$i
+ done
+
+ # drop reply on MDT to force reconstruction
+ #define OBD_FAIL_MDS_REINT_MULTI_NET_REP 0x15a
+ do_facet $facet "$LCTL set_param fail_loc=0x15a"
+ echo "launch $num chmod in parallel ($(date +%H:%M:%S)) ..."
+ for i in $(seq $num); do
+ chmod 0600 $DIR/$tdir/file-$i &
+ pids="$pids $!"
+ done
+ sleep 1
+ do_facet $facet "$LCTL set_param fail_loc=0"
+ for pid in $pids; do
+ wait $pid || error "chmod failed"
+ done
+ echo "done ($(date +%H:%M:%S))"
+
+ # check chmod succeed
+ for i in $(seq $num); do
+ checkstat -vp 0600 $DIR/$tdir/file-$i
+ done
+
+ rm -rf $DIR/$tdir
+}
+run_test 102b "check resend (reply lost) with multiple modify RPCs in flight"
+
+test_102c() {
+ local idx
+ local facet
+ local num
+ local i
+ local pids pid
+
+ [[ $(lctl get_param mdc.*.import |
+ grep "connect_flags:.*multi_mod_rpc") ]] ||
+ { skip "Need MDC with 'multi_mod_rpcs' feature"; return 0; }
+
+ $LFS mkdir -c1 $DIR/$tdir || error "mkdir $DIR/$tdir failed"
+ idx=$(printf "%04x" $($LFS getdirstripe -i $DIR/$tdir))
+ facet="mds$((0x$idx + 1))"
+
+ # get current value of max_mod_rcps_in_flight
+ num=$($LCTL get_param -n \
+ mdc.$FSNAME-MDT$idx-mdc-*.max_mod_rpcs_in_flight)
+ # set default value if client does not support multi mod RPCs
+ [ -z "$num" ] && num=1
+
+ echo "creating $num files ..."
+ umask 0022
+ for i in $(seq $num); do
+ touch $DIR/$tdir/file-$i
+ done
+
+ replay_barrier $facet
+
+ # drop reply on MDT
+ #define OBD_FAIL_MDS_REINT_MULTI_NET_REP 0x15a
+ do_facet $facet "$LCTL set_param fail_loc=0x15a"
+ echo "launch $num chmod in parallel ($(date +%H:%M:%S)) ..."
+ for i in $(seq $num); do
+ chmod 0600 $DIR/$tdir/file-$i &
+ pids="$pids $!"
+ done
+ sleep 1
+ do_facet $facet "$LCTL set_param fail_loc=0"
+
+ # fail MDT
+ fail $facet
+
+ for pid in $pids; do
+ wait $pid || error "chmod failed"
+ done
+ echo "done ($(date +%H:%M:%S))"
+
+ # check chmod succeed
+ for i in $(seq $num); do
+ checkstat -vp 0600 $DIR/$tdir/file-$i
+ done
+
+ rm -rf $DIR/$tdir
+}
+run_test 102c "check replay w/o reconstruction with multiple mod RPCs in flight"
+
+test_102d() {
+ local idx
+ local facet
+ local num
+ local i
+ local pids pid
+
+ [[ $(lctl get_param mdc.*.import |
+ grep "connect_flags:.*multi_mod_rpc") ]] ||
+ { skip "Need MDC with 'multi_mod_rpcs' feature"; return 0; }
+
+ $LFS mkdir -c1 $DIR/$tdir || error "mkdir $DIR/$tdir failed"
+ idx=$(printf "%04x" $($LFS getdirstripe -i $DIR/$tdir))
+ facet="mds$((0x$idx + 1))"
+
+ # get current value of max_mod_rcps_in_flight
+ num=$($LCTL get_param -n \
+ mdc.$FSNAME-MDT$idx-mdc-*.max_mod_rpcs_in_flight)
+ # set default value if client does not support multi mod RPCs
+ [ -z "$num" ] && num=1
+
+ echo "creating $num files ..."
+ umask 0022
+ for i in $(seq $num); do
+ touch $DIR/$tdir/file-$i
+ done
+
+ # drop reply on MDT
+ #define OBD_FAIL_MDS_REINT_MULTI_NET_REP 0x15a
+ do_facet $facet "$LCTL set_param fail_loc=0x15a"
+ echo "launch $num chmod in parallel ($(date +%H:%M:%S)) ..."
+ for i in $(seq $num); do
+ chmod 0600 $DIR/$tdir/file-$i &
+ pids="$pids $!"
+ done
+ sleep 1
+
+ # write MDT transactions to disk
+ do_facet $facet "sync; sync; sync"
+
+ do_facet $facet "$LCTL set_param fail_loc=0"
+
+ # fail MDT
+ fail $facet
+
+ for pid in $pids; do
+ wait $pid || error "chmod failed"
+ done
+ echo "done ($(date +%H:%M:%S))"
+
+ # check chmod succeed
+ for i in $(seq $num); do
+ checkstat -vp 0600 $DIR/$tdir/file-$i
+ done
+
+ rm -rf $DIR/$tdir
+}
+run_test 102d "check replay & reconstruction with multiple mod RPCs in flight"
+
check_striped_dir_110()
{
$CHECKSTAT -t dir $DIR/$tdir/striped_dir ||
git://git.whamcloud.com / fs / lustre-release.git / blobdiff