init_test_env $@
. ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
-TESTSUITELOG=${TESTSUITELOG:-$TMP/recovery-double-scale}
+init_logging
+TESTSUITELOG=${TESTSUITELOG:-$TMP/$(basename $0 .sh)}
DEBUGLOG=$TESTSUITELOG.debug
+
+cleanup_logs
+
exec 2>$DEBUGLOG
echo "--- env ---" >&2
env >&2
echo "--- env ---" >&2
set -x
+[ "$SHARED_DIRECTORY" ] || \
+ { skip "$0: Empty SHARED_DIRECTORY" && exit 0; }
+
[ -n "$CLIENTS" ] || { skip "$0 Need two or more remote clients" && exit 0; }
[ $CLIENTCOUNT -ge 3 ] || \
{ skip "$0 Need two or more remote clients, have $CLIENTCOUNT" && exit 0; }
shutdown_client $c
boot_node $c
echo "Reintegrating $c"
- zconf_mount $c $MOUNT || return $?
+ # one client fails; need dk logs from this client only
+ zconf_mount $c $MOUNT || NODES="$c $(mdts_nodes) $(osts_nodes)" error_exit "zconf_mount failed"
done
- start_client_loads $item || return $?
+ start_client_loads $item
;;
- * ) error "reboot_recover_node: nodetype=$nodetype. Must be one of 'MDS', 'OST', or 'clients'."
+ # script failure:
+ # don't use error (), the logs from all nodes not needed
+ * ) echo "reboot_recover_node: nodetype=$nodetype. Must be one of 'MDS', 'OST', or 'clients'."
exit 1;;
esac
}
OST ) list=$OSTS;;
clients) list=$NODES_TO_USE
;;
- * ) error "Invalid type=$type. Must be one of 'MDS', 'OST', or 'clients'."
+ # script failure:
+ # don't use error (), the logs from all nodes not needed
+ * ) echo "Invalid type=$type. Must be one of 'MDS', 'OST', or 'clients'."
exit 1;;
esac
log "Done checking client loads. Failing type1=$type1 item1=$item1 ... "
- reboot_recover_node $item1 $type1 || return $?
+ reboot_recover_node $item1 $type1
# Hendrix test17 description:
# Introduce a failure, wait at
# do not need a sleep between failures for "double failures"
log " Failing type2=$type2 item2=$item2 ... "
- reboot_recover_node $item2 $type2 || return $?
+ reboot_recover_node $item2 $type2
# Client loads are allowed to die while in recovery, so we just
# restart them.
# actually failed though. the first node in the END_RUN_NODE is
# the one we are really interested in.
if [ -n "$END_RUN_NODE" ]; then
- var=${END_RUN_NODE}_load
+ var=$(client_var_name $END_RUN_NODE)_load
echo "Client load failed on node $END_RUN_NODE"
echo
echo "client $END_RUN_NODE load debug output :"
sleep 5
kill -9 $CLIENT_LOAD_PIDS || true
fi
+
+ if [ $rc -ne 0 ]; then
+ # we are interested in only on failed clients and servers
+ local failedclients=$(cat $END_RUN_FILE | grep -v $0)
+ # FIXME: need ostfailover-s nodes also for FLAVOR=OST
+ local product=$(gather_logs $(comma_list $(osts_nodes) \
+ $(mdts_nodes) $mdsfailover_HOST $failedclients))
+ echo logs files $product
+ fi
+
[ $rc -eq 0 ] && zconf_mount $(hostname) $MOUNT
exit $rc
}
# Start client loads.
start_client_loads $NODES_TO_USE
echo clients load pids:
-if ! do_nodes $NODES_TO_USE "set -x; echo \$(hostname): && cat $TMP/client-load.pid"; then
- if [ -e $DEBUGLOG ]; then
- exec 2<&-
- cat $DEBUGLOG
+if ! do_nodesv $NODES_TO_USE "cat $TMP/client-load.pid"; then
exit 3
- fi
fi
# FIXME: Do we want to have an initial sleep period where the clients