# Skip these tests
# bug number for skipped tests:
-# b=17466/LU-472
+# LU-472
ALWAYS_EXCEPT="$REPLAY_SINGLE_EXCEPT 61d"
# UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT!
-case "$(lsb_release -sr)" in # only disable tests for el7
-7*) # bug number: LU-6455-----
- ALWAYS_EXCEPT="$ALWAYS_EXCEPT 28"
- ;;
-esac
-
-# 63 min 7 min AT AT AT AT"
-[ "$SLOW" = "no" ] && EXCEPT_SLOW="1 2 3 4 6 12 16 44a 44b 65 66 67 68"
+# 7.5 (min)"
+[ "$SLOW" = "no" ] && EXCEPT_SLOW="44b"
[ $(facet_fstype $SINGLEMDS) = "zfs" ] &&
# bug number for skipped test: LU-1867 LU-3127
cleanup_70c() {
trap 0
- kill -9 $tar_70c_pid
+ rm -f $DIR/replay-single.70c.lck
+ rm -rf /$DIR/$tdir
}
+
test_70c () {
local clients=${CLIENTS:-$HOSTNAME}
local rc=0
trap cleanup_70c EXIT
(
- while true; do
+ while [ ! -e $DIR/replay-single.70c.lck ]; do
test_mkdir -p -c$MDSCOUNT $DIR/$tdir || break
if [ $MDSCOUNT -ge 2 ]; then
$LFS setdirstripe -D -c$MDSCOUNT $DIR/$tdir ||
error "set default dirstripe failed"
fi
cd $DIR/$tdir || break
- tar cf - /etc | tar xf - || error "tar failed"
- cd $DIR || break
- rm -rf $DIR/$tdir || break
+ tar cf - /etc | tar xf - || error "tar failed in loop"
done
)&
tar_70c_pid=$!
random_fail_mdt $MDSCOUNT $duration $tar_70c_pid
kill -0 $tar_70c_pid || error "tar $tar_70c_pid stopped"
+ touch $DIR/replay-single.70c.lck
+ wait $tar_70c_pid || error "$?: tar failed"
+
cleanup_70c
true
}
while true; do
mrename $DIR/$tdir/test_0/a $DIR/$tdir/test_1/b > \
/dev/null || {
- echo "a->b fails"
+ echo "a->b fails"
break;
}
}
run_test 70e "rename cross-MDT with random fails"
+test_70f_write_and_read(){
+ local srcfile=$1
+ local stopflag=$2
+ local client
+
+ echo "Write/read files in: '$DIR/$tdir', clients: '$CLIENTS' ..."
+ for client in ${CLIENTS//,/ }; do
+ [ -f $stopflag ] || return
+
+ local tgtfile=$DIR/$tdir/$tfile.$client
+ do_node $client dd $DD_OPTS bs=1M count=10 if=$srcfile \
+ of=$tgtfile 2>/dev/null ||
+ error "dd $DD_OPTS bs=1M count=10 if=$srcfile " \
+ "of=$tgtfile failed on $client, rc=$?"
+ done
+
+ local prev_client=$(echo ${CLIENTS//,/ } | awk '{ print $NF }')
+ local index=0
+
+ for client in ${CLIENTS//,/ }; do
+ [ -f $stopflag ] || return
+
+ # flush client cache in case test is running on only one client
+ # do_node $client cancel_lru_locks osc
+ do_node $client $LCTL set_param ldlm.namespaces.*.lru_size=clear
+
+ tgtfile=$DIR/$tdir/$tfile.$client
+ local md5=$(do_node $prev_client "md5sum $tgtfile")
+ [ ${checksum[$index]// */} = ${md5// */} ] ||
+ error "$tgtfile: checksum doesn't match on $prev_client"
+ index=$((index + 1))
+ prev_client=$client
+ done
+}
+
+test_70f_loop(){
+ local srcfile=$1
+ local stopflag=$2
+ DD_OPTS=
+
+ mkdir -p $DIR/$tdir || error "cannot create $DIR/$tdir directory"
+ $SETSTRIPE -c -1 $DIR/$tdir || error "cannot $SETSTRIPE $DIR/$tdir"
+
+ touch $stopflag
+ while [ -f $stopflag ]; do
+ test_70f_write_and_read $srcfile $stopflag
+ # use direct IO and buffer cache in turns if loop
+ [ -n "$DD_OPTS" ] && DD_OPTS="" || DD_OPTS="oflag=direct"
+ done
+}
+
+test_70f_cleanup() {
+ trap 0
+ rm -f $TMP/$tfile.stop
+ do_nodes $CLIENTS rm -f $TMP/$tfile
+ rm -f $DIR/$tdir/$tfile.*
+}
+
+test_70f() {
+# [ x$ost1failover_HOST = x$ost_HOST ] &&
+# { skip "Failover host not defined" && return; }
+# [ -z "$CLIENTS" ] &&
+# { skip "CLIENTS are not specified." && return; }
+# [ $CLIENTCOUNT -lt 2 ] &&
+# { skip "Need 2 or more clients, have $CLIENTCOUNT" && return; }
+
+ echo "mount clients $CLIENTS ..."
+ zconf_mount_clients $CLIENTS $MOUNT
+
+ local srcfile=$TMP/$tfile
+ local client
+ local index=0
+
+ trap test_70f_cleanup EXIT
+ # create a different source file local to each client node so we can
+ # detect if the file wasn't written out properly after failover
+ do_nodes $CLIENTS dd bs=1M count=10 if=/dev/urandom of=$srcfile \
+ 2>/dev/null || error "can't create $srcfile on $CLIENTS"
+ for client in ${CLIENTS//,/ }; do
+ checksum[$index]=$(do_node $client "md5sum $srcfile")
+ index=$((index + 1))
+ done
+
+ local duration=120
+ [ "$SLOW" = "no" ] && duration=60
+ # set duration to 900 because it takes some time to boot node
+ [ "$FAILURE_MODE" = HARD ] && duration=900
+
+ local stopflag=$TMP/$tfile.stop
+ test_70f_loop $srcfile $stopflag &
+ local pid=$!
+
+ local elapsed=0
+ local num_failovers=0
+ local start_ts=$SECONDS
+ while [ $elapsed -lt $duration ]; do
+ sleep 3
+ replay_barrier ost1
+ sleep 1
+ num_failovers=$((num_failovers + 1))
+ log "$TESTNAME failing OST $num_failovers times"
+ fail ost1
+ sleep 2
+ elapsed=$((SECONDS - start_ts))
+ done
+
+ rm -f $stopflag
+ wait $pid
+ test_70f_cleanup
+}
+run_test 70f "OSS O_DIRECT recovery with $CLIENTCOUNT clients"
+
cleanup_71a() {
trap 0
kill -9 $mkdir_71a_pid
done
lov_id=$(lctl dl | grep "clilov")
- addr=$(echo $lov_id | awk '{print $4}' | awk -F '-' '{print $3}')
+ addr=$(echo $lov_id | awk '{print $4}' | awk -F '-' '{print $NF}')
count=$(lctl get_param -n \
ldlm.namespaces.*MDT0000*$addr.lock_unused_count)
echo "before recovery: unused locks count = $count"
done
lov_id=$(lctl dl | grep "clilov")
- addr=$(echo $lov_id | awk '{print $4}' | awk -F '-' '{print $3}')
- count=$(lctl get_param \
- -n ldlm.namespaces.*OST0000*$addr.lock_unused_count)
+ addr=$(echo $lov_id | awk '{print $4}' | awk -F '-' '{print $NF}')
+ count=$(lctl get_param -n \
+ ldlm.namespaces.*OST0000*$addr.lock_unused_count)
echo "before recovery: unused locks count = $count"
- [ $count != 0 ] || error "unused locks ($count) should be zero"
+ [ $count -ne 0 ] || error "unused locks ($count) should be zero"
fail ost1
}
run_test 86 "umount server after clear nid_stats should not hit LBUG"
-test_87() {
+test_87a() {
do_facet ost1 "lctl set_param -n obdfilter.${ost1_svc}.sync_journal 0"
replay_barrier ost1
error "New checksum $cksum2 does not match original $cksum"
fi
}
-run_test 87 "write replay"
+run_test 87a "write replay"
test_87b() {
do_facet ost1 "lctl set_param -n obdfilter.${ost1_svc}.sync_journal 0"
return 0
fi
fi
+ # ensure all OSTs are active to allow allocations
+ wait_osts_up
mkdir $dir || error "mkdir $dir failed"
}
run_test 90 "lfs find identifies the missing striped file segments"
-test_93() {
+test_93a() {
local server_version=$(lustre_version_code $SINGLEMDS)
[[ $server_version -ge $(version_code 2.6.90) ]] ||
[[ $server_version -ge $(version_code 2.5.4) &&
do_facet ost1 "$LCTL set_param fail_loc=0x715"
fail ost1
}
-run_test 93 "replay + reconnect"
+run_test 93a "replay + reconnect"
+
+test_93b() {
+ local server_version=$(lustre_version_code $SINGLEMDS)
+ [[ $server_version -ge $(version_code 2.7.90) ]] ||
+ { skip "Need MDS version 2.7.90+"; return; }
+
+ cancel_lru_locks mdc
+
+ createmany -o $DIR/$tfile 20 ||
+ error "createmany -o $DIR/$tfile failed"
+
+ #define OBD_FAIL_TGT_REPLAY_RECONNECT 0x715
+ # We need to emulate a state that MDT is waiting for other clients
+ # not completing the recovery. Final ping is queued, but reply will be
+ # sent on the recovery completion. It is done by sleep before
+ # processing final pings
+ do_facet mds1 "$LCTL set_param fail_val=80"
+ do_facet mds1 "$LCTL set_param fail_loc=0x715"
+ fail mds1
+}
+run_test 93b "replay + reconnect on mds"
striped_dir_check_100() {
local striped_dir=$DIR/$tdir/striped_dir
}
run_test 102d "check replay & reconstruction with multiple mod RPCs in flight"
+test_103() {
+ remote_mds_nodsh && skip "remote MDS with nodsh" && return
+#define OBD_FAIL_MDS_TRACK_OVERFLOW 0x162
+ do_facet mds1 $LCTL set_param fail_loc=0x80000162
+
+ mkdir -p $DIR/$tdir
+ createmany -o $DIR/$tdir/t- 30 ||
+ error "create files on remote directory failed"
+ sync
+ rm -rf $DIR/$tdir/t-*
+ sync
+#MDS should crash with tr->otr_next_id overflow
+ fail mds1
+}
+run_test 103 "Check otr_next_id overflow"
+
+
check_striped_dir_110()
{
$CHECKSTAT -t dir $DIR/$tdir/striped_dir ||
}
run_test 117 "DNE: cross MDT unlink, fail MDT1 and MDT2"
+test_118() {
+ [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
+ [ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.7.64) ] &&
+ skip "Do not support large update log before 2.7.64" &&
+ return 0
+
+ mkdir -p $DIR/$tdir
+
+ $LFS setdirstripe -c2 $DIR/$tdir/striped_dir ||
+ error "setdirstripe fails"
+ $LFS setdirstripe -c2 $DIR/$tdir/striped_dir1 ||
+ error "setdirstripe fails 1"
+ rm -rf $DIR/$tdir/striped_dir* || error "rmdir fails"
+
+ # OBD_FAIL_INVALIDATE_UPDATE 0x1705
+ do_facet mds1 "lctl set_param fail_loc=0x1705"
+ $LFS setdirstripe -c2 $DIR/$tdir/striped_dir
+ $LFS setdirstripe -c2 $DIR/$tdir/striped_dir1
+ do_facet mds1 "lctl set_param fail_loc=0x0"
+
+ replay_barrier mds1
+ $LFS setdirstripe -c2 $DIR/$tdir/striped_dir
+ $LFS setdirstripe -c2 $DIR/$tdir/striped_dir1
+ fail mds1
+
+ true
+}
+run_test 118 "invalidate osp update will not cause update log corruption"
+
+test_119() {
+ [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
+ [ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.7.64) ] &&
+ skip "Do not support large update log before 2.7.64" &&
+ return 0
+ local stripe_count
+ local hard_timeout=$(do_facet mds1 \
+ "lctl get_param -n mdt.$FSNAME-MDT0000.recovery_time_hard")
+
+ local clients=${CLIENTS:-$HOSTNAME}
+ local time_min=$(recovery_time_min)
+
+ mkdir -p $DIR/$tdir
+ mkdir $DIR/$tdir/tmp
+ rmdir $DIR/$tdir/tmp
+
+ replay_barrier mds1
+ mkdir $DIR/$tdir/dir_1
+ for ((i = 0; i < 20; i++)); do
+ $LFS setdirstripe -c2 $DIR/$tdir/stripe_dir-$i
+ done
+
+ stop mds1
+ change_active mds1
+ wait_for_facet mds1
+
+ #define OBD_FAIL_TGT_REPLAY_DELAY 0x714
+ do_facet mds1 $LCTL set_param fail_loc=0x80000714
+ #sleep (timeout + 5), so mds will evict the client exports,
+ #but DNE update recovery will keep going.
+ do_facet mds1 $LCTL set_param fail_val=$((time_min + 5))
+
+ mount_facet mds1 "-o recovery_time_hard=$time_min"
+
+ wait_clients_import_state "$clients" mds1 FULL
+
+ clients_up || clients_up || error "failover df: $?"
+
+ #revert back the hard timeout
+ do_facet mds1 $LCTL set_param \
+ mdt.$FSNAME-MDT0000.recovery_time_hard=$hard_timeout
+
+ for ((i = 0; i < 20; i++)); do
+ stripe_count=$($LFS getdirstripe -c $DIR/$tdir/stripe_dir-$i)
+ [ $stripe_count == 2 ] || {
+ error "stripe_dir-$i creation replay fails"
+ break
+ }
+ done
+}
+run_test 119 "timeout of normal replay does not cause DNE replay fails "
+
+test_120() {
+ [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
+ [ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.7.64) ] &&
+ skip "Do not support large update log before 2.7.64" &&
+ return 0
+
+ mkdir $DIR/$tdir
+ replay_barrier_nosync mds1
+ for ((i = 0; i < 20; i++)); do
+ mkdir $DIR/$tdir/dir-$i || {
+ error "create dir-$i fails"
+ break
+ }
+ $LFS setdirstripe -c2 $DIR/$tdir/stripe_dir-$i || {
+ error "create stripe_dir-$i fails"
+ break
+ }
+ done
+
+ fail_abort mds1
+
+ for ((i = 0; i < 20; i++)); do
+ [ ! -e "$DIR/$tdir/dir-$i" ] || {
+ error "dir-$i still exists"
+ break
+ }
+ [ ! -e "$DIR/$tdir/stripe_dir-$i" ] || {
+ error "stripe_dir-$i still exists"
+ break
+ }
+ done
+}
+run_test 120 "DNE fail abort should stop both normal and DNE replay"
+
complete $SECONDS
check_and_cleanup_lustre
exit_status
git://git.whamcloud.com / fs / lustre-release.git / blobdiff