set -e
-PTLDEBUG=${PTLDEBUG:--1}
LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)}
SETUP=${SETUP:-""}
CLEANUP=${CLEANUP:-""}
remote_ost_nodsh && skip "remote OST with nodsh" && exit 0
if [ "$FAILURE_MODE" = "HARD" ] && mixed_ost_devs; then
- skip "$0: Several ost services on one ost node are used with FAILURE_MODE=$FAILURE_MODE. "
+ skip_env "$0: Several ost services on one ost node are used with FAILURE_MODE=$FAILURE_MODE. "
exit 0
fi
}
run_test 4 "Fail OST during read, with verification"
+iozone_bg () {
+ local args=$@
+
+ local tmppipe=$TMP/${TESTSUITE}.${TESTNAME}.pipe
+ mkfifo $tmppipe
+
+ echo "+ iozone $args"
+ iozone $args > $tmppipe &
+
+ local pid=$!
+
+ echo "tmppipe=$tmppipe"
+ echo iozone pid=$pid
+
+ # iozone exit code is 0 even if iozone is not completed
+ # need to check iozone output on "complete"
+ local iozonelog=$TMP/${TESTSUITE}.iozone.log
+ rm -f $iozonelog
+ cat $tmppipe | while read line ; do
+ echo "$line"
+ echo "$line" >>$iozonelog
+ done;
+
+ local rc=0
+ wait $pid
+ rc=$?
+ if ! $(tail -1 $iozonelog | grep -q complete); then
+ echo iozone failed!
+ rc=1
+ fi
+ rm -f $tmppipe
+ rm -f $iozonelog
+ return $rc
+}
+
test_5() {
- [ -z "`which iozone 2> /dev/null`" ] && skip "iozone missing" && return 0
- FREE=`df -P $TDIR | tail -n 1 | awk '{ print $4/2 }'`
- GB=1048576 # 1048576KB == 1GB
- if (( FREE > GB )); then
- FREE=$GB
+ [ -z "`which iozone 2> /dev/null`" ] && skip_env "iozone missing" && return 0
+
+ # striping is -c 1, get min of available
+ local minavail=$(lctl get_param -n osc.*[oO][sS][cC][-_]*.kbytesavail | sort -n | head -1)
+ local size=$(( minavail * 3/4 ))
+ local GB=1048576 # 1048576KB == 1GB
+
+ if (( size > GB )); then
+ size=$GB
fi
- IOZONE_OPTS="-i 0 -i 1 -i 2 -+d -r 4 -s $FREE"
- iozone $IOZONE_OPTS -f $TDIR/$tfile &
- PID=$!
-
+ local iozone_opts="-i 0 -i 1 -i 2 -+d -r 4 -s $size -f $TDIR/$tfile"
+
+ iozone_bg $iozone_opts &
+ local pid=$!
+
+ echo iozone bg pid=$pid
+
sleep 8
fail ost1
- wait $PID
- RC=$?
- log "iozone rc=$RC"
+ local rc=0
+ wait $pid
+ rc=$?
+ log "iozone rc=$rc"
rm -f $TDIR/$tfile
- [ $RC -ne 0 ] && return $RC || true
+ [ $rc -eq 0 ] || error "iozone failed"
+ return $rc
}
run_test 5 "Fail OST during iozone"
before=`kbytesfree`
dd if=/dev/urandom bs=4096 count=1280 of=$f || return 28
lfs getstripe $f
+ get_stripe_info client $f
+
sync
- sleep 2 # ensure we have a fresh statfs
+ sleep 2 # ensure we have a fresh statfs
sync
#define OBD_FAIL_MDS_REINT_NET_REP 0x119
do_facet mds "lctl set_param fail_loc=0x80000119"
log "before: $before after_dd: $after_dd"
(( $before > $after_dd )) || return 1
rm -f $f
- fail ost1
+ fail ost$((stripe_index + 1))
+ wait_recovery_complete ost$((stripe_index + 1)) || error "OST recovery not done"
$CHECKSTAT -t file $f && return 2 || true
sync
# let the delete happen
- sleep 5
+ wait_mds_ost_sync || return 4
+ wait_destroy_complete || return 5
after=`kbytesfree`
log "before: $before after: $after"
(( $before <= $after + 40 )) || return 3 # take OST logs into account
before=`kbytesfree`
dd if=/dev/urandom bs=4096 count=1280 of=$f || return 4
sync
- sleep 2 # ensure we have a fresh statfs
+ sleep 2 # ensure we have a fresh statfs
sync
after_dd=`kbytesfree`
log "before: $before after_dd: $after_dd"
replay_barrier ost1
rm -f $f
fail ost1
+ wait_recovery_complete ost1 || error "OST recovery not done"
$CHECKSTAT -t file $f && return 2 || true
sync
# let the delete happen
- sleep 5
+ wait_mds_ost_sync || return 4
+ wait_destroy_complete || return 5
after=`kbytesfree`
log "before: $before after: $after"
(( $before <= $after + 40 )) || return 3 # take OST logs into account