+random_fail_mdt() {
+ local max_index=$1
+ local duration=$2
+ local monitor_pid=$3
+ local elapsed
+ local start_ts=$(date +%s)
+ local num_failovers=0
+ local fail_index
+
+ elapsed=$(($(date +%s) - start_ts))
+ while [ $elapsed -lt $duration ]; do
+ fail_index=$((RANDOM%max_index+1))
+ kill -0 $monitor_pid ||
+ error "$monitor_pid stopped"
+ sleep 120
+ replay_barrier mds$fail_index
+ sleep 10
+ # Increment the number of failovers
+ num_failovers=$((num_failovers+1))
+ log "$TESTNAME fail mds$fail_index $num_failovers times"
+ fail mds$fail_index
+ elapsed=$(($(date +%s) - start_ts))
+ done
+}
+
+cleanup_70c() {
+ trap 0
+ rm -f $DIR/replay-single.70c.lck
+ rm -rf /$DIR/$tdir
+}
+
+test_70c () {
+ local clients=${CLIENTS:-$HOSTNAME}
+ local rc=0
+
+ zconf_mount_clients $clients $MOUNT
+
+ local duration=300
+ [ "$SLOW" = "no" ] && duration=180
+ # set duration to 900 because it takes some time to boot node
+ [ "$FAILURE_MODE" = HARD ] && duration=600
+
+ local elapsed
+ local start_ts=$(date +%s)
+
+ trap cleanup_70c EXIT
+ (
+ while [ ! -e $DIR/replay-single.70c.lck ]; do
+ test_mkdir -p -c$MDSCOUNT $DIR/$tdir || break
+ if [ $MDSCOUNT -ge 2 ]; then
+ $LFS setdirstripe -D -c$MDSCOUNT $DIR/$tdir ||
+ error "set default dirstripe failed"
+ fi
+ cd $DIR/$tdir || break
+ tar cf - /etc | tar xf - || error "tar failed in loop"
+ done
+ )&
+ tar_70c_pid=$!
+ echo "Started tar $tar_70c_pid"
+
+ random_fail_mdt $MDSCOUNT $duration $tar_70c_pid
+ kill -0 $tar_70c_pid || error "tar $tar_70c_pid stopped"
+
+ touch $DIR/replay-single.70c.lck
+ wait $tar_70c_pid || error "$?: tar failed"
+
+ cleanup_70c
+ true
+}
+run_test 70c "tar ${MDSCOUNT}mdts recovery"
+
+cleanup_70d() {
+ trap 0
+ kill -9 $mkdir_70d_pid
+}
+
+test_70d () {
+ [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
+ local clients=${CLIENTS:-$HOSTNAME}
+ local rc=0
+
+ zconf_mount_clients $clients $MOUNT
+
+ local duration=300
+ [ "$SLOW" = "no" ] && duration=180
+ # set duration to 900 because it takes some time to boot node
+ [ "$FAILURE_MODE" = HARD ] && duration=900
+
+ mkdir -p $DIR/$tdir
+
+ local elapsed
+ local start_ts=$(date +%s)
+
+ trap cleanup_70d EXIT
+ (
+ while true; do
+ $LFS mkdir -i0 -c2 $DIR/$tdir/test || {
+ echo "mkdir fails"
+ break
+ }
+ $LFS mkdir -i1 -c2 $DIR/$tdir/test1 || {
+ echo "mkdir fails"
+ break
+ }
+
+ touch $DIR/$tdir/test/a || {
+ echo "touch fails"
+ break;
+ }
+ mkdir $DIR/$tdir/test/b || {
+ echo "mkdir fails"
+ break;
+ }
+ rm -rf $DIR/$tdir/test || {
+ echo "rmdir fails"
+ break
+ }
+
+ touch $DIR/$tdir/test1/a || {
+ echo "touch fails"
+ break;
+ }
+ mkdir $DIR/$tdir/test1/b || {
+ echo "mkdir fails"
+ break;
+ }
+
+ rm -rf $DIR/$tdir/test1 || {
+ echo "rmdir fails"
+ break
+ }
+ done
+ )&
+ mkdir_70d_pid=$!
+ echo "Started $mkdir_70d_pid"
+
+ random_fail_mdt $MDSCOUNT $duration $mkdir_70d_pid
+ kill -0 $mkdir_70d_pid || error "mkdir/rmdir $mkdir_70d_pid stopped"
+
+ cleanup_70d
+ true
+}
+run_test 70d "mkdir/rmdir striped dir ${MDSCOUNT}mdts recovery"
+
+cleanup_70e() {
+ trap 0
+ kill -9 $rename_70e_pid
+}
+
+test_70e () {
+ [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
+ local clients=${CLIENTS:-$HOSTNAME}
+ local rc=0
+
+ echo ha > /proc/sys/lnet/debug
+ zconf_mount_clients $clients $MOUNT
+
+ local duration=300
+ [ "$SLOW" = "no" ] && duration=180
+ # set duration to 900 because it takes some time to boot node
+ [ "$FAILURE_MODE" = HARD ] && duration=900
+
+ mkdir -p $DIR/$tdir
+ $LFS mkdir -i0 $DIR/$tdir/test_0
+ $LFS mkdir -i0 $DIR/$tdir/test_1
+ touch $DIR/$tdir/test_0/a
+ touch $DIR/$tdir/test_1/b
+ trap cleanup_70e EXIT
+ (
+ while true; do
+ mrename $DIR/$tdir/test_0/a $DIR/$tdir/test_1/b > \
+ /dev/null || {
+ echo "a->b fails"
+ break;
+ }
+
+ checkstat $DIR/$tdir/test_0/a && {
+ echo "a still exists"
+ break
+ }
+
+ checkstat $DIR/$tdir/test_1/b || {
+ echo "b still exists"
+ break
+ }
+
+ touch $DIR/$tdir/test_0/a || {
+ echo "touch a fails"
+ break
+ }
+
+ mrename $DIR/$tdir/test_1/b $DIR/$tdir/test_0/a > \
+ /dev/null || {
+ echo "a->a fails"
+ break;
+ }
+ done
+ )&
+ rename_70e_pid=$!
+ echo "Started $rename_70e_pid"
+
+ random_fail_mdt 2 $duration $rename_70e_pid
+ kill -0 $rename_70e_pid || error "rename $rename_70e_pid stopped"
+
+ cleanup_70e
+ true
+}
+run_test 70e "rename cross-MDT with random fails"
+
+test_70f_write_and_read(){
+ local srcfile=$1
+ local stopflag=$2
+ local client
+
+ echo "Write/read files in: '$DIR/$tdir', clients: '$CLIENTS' ..."
+ for client in ${CLIENTS//,/ }; do
+ [ -f $stopflag ] || return
+
+ local tgtfile=$DIR/$tdir/$tfile.$client
+ do_node $client dd $DD_OPTS bs=1M count=10 if=$srcfile \
+ of=$tgtfile 2>/dev/null ||
+ error "dd $DD_OPTS bs=1M count=10 if=$srcfile " \
+ "of=$tgtfile failed on $client, rc=$?"
+ done
+
+ local prev_client=$(echo ${CLIENTS//,/ } | awk '{ print $NF }')
+ local index=0
+
+ for client in ${CLIENTS//,/ }; do
+ [ -f $stopflag ] || return
+
+ # flush client cache in case test is running on only one client
+ # do_node $client cancel_lru_locks osc
+ do_node $client $LCTL set_param ldlm.namespaces.*.lru_size=clear
+
+ tgtfile=$DIR/$tdir/$tfile.$client
+ local md5=$(do_node $prev_client "md5sum $tgtfile")
+ [ ${checksum[$index]// */} = ${md5// */} ] ||
+ error "$tgtfile: checksum doesn't match on $prev_client"
+ index=$((index + 1))
+ prev_client=$client
+ done
+}
+
+test_70f_loop(){
+ local srcfile=$1
+ local stopflag=$2
+ DD_OPTS=
+
+ mkdir -p $DIR/$tdir || error "cannot create $DIR/$tdir directory"
+ $SETSTRIPE -c -1 $DIR/$tdir || error "cannot $SETSTRIPE $DIR/$tdir"
+
+ touch $stopflag
+ while [ -f $stopflag ]; do
+ test_70f_write_and_read $srcfile $stopflag
+ # use direct IO and buffer cache in turns if loop
+ [ -n "$DD_OPTS" ] && DD_OPTS="" || DD_OPTS="oflag=direct"
+ done
+}
+
+test_70f_cleanup() {
+ trap 0
+ rm -f $TMP/$tfile.stop
+ do_nodes $CLIENTS rm -f $TMP/$tfile
+ rm -f $DIR/$tdir/$tfile.*
+}
+
+test_70f() {
+# [ x$ost1failover_HOST = x$ost_HOST ] &&
+# { skip "Failover host not defined" && return; }
+# [ -z "$CLIENTS" ] &&
+# { skip "CLIENTS are not specified." && return; }
+# [ $CLIENTCOUNT -lt 2 ] &&
+# { skip "Need 2 or more clients, have $CLIENTCOUNT" && return; }
+
+ echo "mount clients $CLIENTS ..."
+ zconf_mount_clients $CLIENTS $MOUNT
+
+ local srcfile=$TMP/$tfile
+ local client
+ local index=0
+
+ trap test_70f_cleanup EXIT
+ # create a different source file local to each client node so we can
+ # detect if the file wasn't written out properly after failover
+ do_nodes $CLIENTS dd bs=1M count=10 if=/dev/urandom of=$srcfile \
+ 2>/dev/null || error "can't create $srcfile on $CLIENTS"
+ for client in ${CLIENTS//,/ }; do
+ checksum[$index]=$(do_node $client "md5sum $srcfile")
+ index=$((index + 1))
+ done
+
+ local duration=120
+ [ "$SLOW" = "no" ] && duration=60
+ # set duration to 900 because it takes some time to boot node
+ [ "$FAILURE_MODE" = HARD ] && duration=900
+
+ local stopflag=$TMP/$tfile.stop
+ test_70f_loop $srcfile $stopflag &
+ local pid=$!
+
+ local elapsed=0
+ local num_failovers=0
+ local start_ts=$SECONDS
+ while [ $elapsed -lt $duration ]; do
+ sleep 3
+ replay_barrier ost1
+ sleep 1
+ num_failovers=$((num_failovers + 1))
+ log "$TESTNAME failing OST $num_failovers times"
+ fail ost1
+ sleep 2
+ elapsed=$((SECONDS - start_ts))
+ done
+
+ rm -f $stopflag
+ wait $pid
+ test_70f_cleanup
+}
+run_test 70f "OSS O_DIRECT recovery with $CLIENTCOUNT clients"
+
+cleanup_71a() {
+ trap 0
+ kill -9 $mkdir_71a_pid
+}
+
+random_double_fail_mdt() {
+ local max_index=$1
+ local duration=$2
+ local monitor_pid=$3
+ local elapsed
+ local start_ts=$(date +%s)
+ local num_failovers=0
+ local fail_index
+ local second_index
+
+ elapsed=$(($(date +%s) - start_ts))
+ while [ $elapsed -lt $duration ]; do
+ fail_index=$((RANDOM%max_index + 1))
+ if [ $fail_index -eq $max_index ]; then
+ second_index=1
+ else
+ second_index=$((fail_index + 1))
+ fi
+ kill -0 $monitor_pid ||
+ error "$monitor_pid stopped"
+ sleep 120
+ replay_barrier mds$fail_index
+ replay_barrier mds$second_index
+ sleep 10
+ # Increment the number of failovers
+ num_failovers=$((num_failovers+1))
+ log "fail mds$fail_index mds$second_index $num_failovers times"
+ fail mds${fail_index},mds${second_index}
+ elapsed=$(($(date +%s) - start_ts))
+ done
+}
+
+test_71a () {
+ [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
+ local clients=${CLIENTS:-$HOSTNAME}
+ local rc=0
+
+ zconf_mount_clients $clients $MOUNT
+
+ local duration=300
+ [ "$SLOW" = "no" ] && duration=180
+ # set duration to 900 because it takes some time to boot node
+ [ "$FAILURE_MODE" = HARD ] && duration=900
+
+ mkdir -p $DIR/$tdir
+
+ local elapsed
+ local start_ts=$(date +%s)
+
+ trap cleanup_71a EXIT
+ (
+ while true; do
+ $LFS mkdir -i0 -c2 $DIR/$tdir/test
+ rmdir $DIR/$tdir/test
+ done
+ )&
+ mkdir_71a_pid=$!
+ echo "Started $mkdir_71a_pid"
+
+ random_double_fail_mdt 2 $duration $mkdir_71a_pid
+ kill -0 $mkdir_71a_pid || error "mkdir/rmdir $mkdir_71a_pid stopped"
+
+ cleanup_71a
+ true
+}
+run_test 71a "mkdir/rmdir striped dir with 2 mdts recovery"
+
git://git.whamcloud.com / fs / lustre-release.git / blobdiff