init_test_env $@
. ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
-TESTSUITELOG=${TESTSUITELOG:-$TMP/recovery-double-scale}
-DEBUGLOG=$TESTSUITELOG.debug
+init_logging
+
+DEBUGLOG=$TESTLOG_PREFIX.suite_debug_log.$(hostname -s).log
+
exec 2>$DEBUGLOG
echo "--- env ---" >&2
env >&2
set -x
[ "$SHARED_DIRECTORY" ] || \
- { skip "$0: Empty SHARED_DIRECTORY" && exit 0; }
+ { FAIL_ON_ERROR=true skip_env "$0 Empty SHARED_DIRECTORY" && exit 0; }
+
+check_shared_dir $SHARED_DIRECTORY ||
+ error "$SHARED_DIRECTORY isn't a shared directory"
+
+[ -n "$CLIENTS" ] || \
+ { FAIL_ON_ERROR=true skip_env "$0 Need two or more remote clients" && exit 0; }
-[ -n "$CLIENTS" ] || { skip "$0 Need two or more remote clients" && exit 0; }
[ $CLIENTCOUNT -ge 3 ] || \
- { skip "$0 Need two or more remote clients, have $CLIENTCOUNT" && exit 0; }
+ { FAIL_ON_ERROR=true skip_env "$0 Need two or more remote clients, have $((CLIENTCOUNT - 1))" && exit 0; }
END_RUN_FILE=${END_RUN_FILE:-$SHARED_DIRECTORY/end_run_file}
LOAD_PID_FILE=${LOAD_PID_FILE:-$TMP/client-load.pid}
check_timeout || exit 1
+[[ $FAILURE_MODE = SOFT ]] && \
+ log "WARNING: $0 is not functional with FAILURE_MODE = SOFT, bz22797"
+
build_test_filter
check_and_setup_lustre
# MDS, OST item contains the facet
case $nodetype in
MDS|OST ) facet_failover $item
- [ "$SERIAL" ] && wait_recovery_complete $item $((timeout * 4)) || true
+ [ "$SERIAL" ] && wait_recovery_complete $item || true
;;
clients) for c in ${item//,/ }; do
+ # make sure the client loads die
+ do_nodes $c "set -x; test -f $LOAD_PID_FILE &&
+ { kill -s TERM \\\$(cat $LOAD_PID_FILE);
+ rm -f $LOAD_PID_FILE || true; }"
shutdown_client $c
boot_node $c
echo "Reintegrating $c"
summary_and_cleanup () {
local rc=$?
+ local var
trap 0
# Having not empty END_RUN_FILE means the failed loads only
local END_RUN_NODE=
read END_RUN_NODE < $END_RUN_FILE
- # a client load will end (i.e. fail) if it finds
- # the end run file. that does not mean that that client load
- # actually failed though. the first node in the END_RUN_NODE is
- # the one we are really interested in.
+ # A client load will stop if it found the END_RUN_FILE file.
+ # That does not mean the client load actually failed though.
+ # The first node in END_RUN_FILE is the one we are interested in.
if [ -n "$END_RUN_NODE" ]; then
- var=$(client_var_name $END_RUN_NODE)_load
+ var=$(node_var_name $END_RUN_NODE)_load
echo "Client load failed on node $END_RUN_NODE"
echo
- echo "client $END_RUN_NODE load debug output :"
- local logfile=${TESTSUITELOG}_run_${!var}.sh-${END_RUN_NODE}.debug
- do_node ${END_RUN_NODE} "set -x; [ -e $logfile ] && cat $logfile " || true
+ echo "Client $END_RUN_NODE load stdout and debug files:
+ $TESTLOG_PREFIX.run_${!var}_stdout.$END_RUN_NODE.log
+ $TESTLOG_PREFIX.run_${!var}_debug.$END_RUN_NODE.log"
fi
rc=1
fi
Status: $result: rc=$rc"
# make sure the client loads die
- do_nodes $NODES_TO_USE "set -x; test -f $TMP/client-load.pid && \
- { kill -s TERM \$(cat $TMP/client-load.pid) || true; }"
+ do_nodes $NODES_TO_USE "set -x; test -f $LOAD_PID_FILE &&
+ { kill -s TERM \\\$(cat $LOAD_PID_FILE);
+ rm -f $LOAD_PID_FILE || true; }"
# and free up the pdshes that started them, if any are still around
if [ -n "$CLIENT_LOAD_PIDS" ]; then
sleep 5
kill -9 $CLIENT_LOAD_PIDS || true
fi
+
+ if [ $rc -ne 0 ]; then
+ # we are interested in only on failed clients and servers
+ local failedclients=$(cat $END_RUN_FILE | grep -v $0)
+ # FIXME: need ostfailover-s nodes also for FLAVOR=OST
+ local product=$(gather_logs $(comma_list $(osts_nodes) \
+ $(mdts_nodes) $mdsfailover_HOST $failedclients) 1)
+ echo $product
+ fi
+
[ $rc -eq 0 ] && zconf_mount $(hostname) $MOUNT
exit $rc
}
# Start client loads.
start_client_loads $NODES_TO_USE
+
echo clients load pids:
-if ! do_nodes $NODES_TO_USE "set -x; echo \$(hostname): && cat $TMP/client-load.pid"; then
- if [ -e $DEBUGLOG ]; then
- exec 2<&-
- cat $DEBUGLOG
- exit 3
- fi
+if ! do_nodesv $NODES_TO_USE "cat $LOAD_PID_FILE"; then
+ exit 3
fi
# FIXME: Do we want to have an initial sleep period where the clients