+boot_node() {
+ local node=$1
+ if [ "$FAILURE_MODE" = HARD ]; then
+ $POWER_UP $node
+ fi
+}
+
+# recovery-scale functions
+check_progs_installed () {
+ local clients=$1
+ shift
+ local progs=$@
+
+ do_nodes $clients "set -x ; PATH=:$PATH status=true; for prog in $progs; do
+ which \\\$prog || { echo \\\$prog missing on \\\$(hostname) && status=false; }
+ done;
+ eval \\\$status"
+}
+
+start_client_load() {
+ local client=$1
+ local var=${client}_load
+
+ do_node $client "PATH=$PATH MOUNT=$MOUNT ERRORS_OK=$ERRORS_OK \
+ BREAK_ON_ERROR=$BREAK_ON_ERROR \
+ END_RUN_FILE=$END_RUN_FILE \
+ LOAD_PID_FILE=$LOAD_PID_FILE \
+ TESTSUITELOG=$TESTSUITELOG \
+ run_${!var}.sh" &
+ CLIENT_LOAD_PIDS="$CLIENT_LOAD_PIDS $!"
+ log "Started client load: ${!var} on $client"
+
+ return 0
+}
+
+start_client_loads () {
+ local clients=(${1//,/ })
+ local numloads=${#CLIENT_LOADS[@]}
+ local testnum
+
+ for ((nodenum=0; nodenum < ${#clients[@]}; nodenum++ )); do
+ testnum=$((nodenum % numloads))
+ eval export ${clients[nodenum]}_load=${CLIENT_LOADS[testnum]}
+ start_client_load ${clients[nodenum]}
+ done
+}
+
+# only for remote client
+check_client_load () {
+ local client=$1
+ local var=${client}_load
+
+ local TESTLOAD=run_${!var}.sh
+
+ ps auxww | grep -v grep | grep $client | grep -q "$TESTLOAD" || return 1
+
+ check_catastrophe $client || return 2
+
+ # see if the load is still on the client
+ local tries=3
+ local RC=254
+ while [ $RC = 254 -a $tries -gt 0 ]; do
+ let tries=$tries-1
+ # assume success
+ RC=0
+ if ! do_node $client "ps auxwww | grep -v grep | grep -q $TESTLOAD"; then
+ RC=${PIPESTATUS[0]}
+ sleep 30
+ fi
+ done
+ if [ $RC = 254 ]; then
+ echo "got a return status of $RC from do_node while checking (i.e. with 'ps') the client load on the remote system"
+ # see if we can diagnose a bit why this is
+ fi
+
+ return $RC
+}
+check_client_loads () {
+ local clients=${1//,/ }
+ local client=
+ local rc=0
+
+ for client in $clients; do
+ check_client_load $client
+ rc=${PIPESTATUS[0]}
+ if [ "$rc" != 0 ]; then
+ log "Client load failed on node $client, rc=$rc"
+ return $rc
+ fi
+ done
+}
+
+restart_client_loads () {
+ local clients=${1//,/ }
+ local expectedfail=${2:-""}
+ local client=
+ local rc=0
+
+ for client in $clients; do
+ check_client_load $client
+ rc=${PIPESTATUS[0]}
+ if [ "$rc" != 0 -a "$expectedfail" ]; then
+ start_client_load $client
+ echo "Restarted client load: on $client. Checking ..."
+ check_client_load $client
+ rc=${PIPESTATUS[0]}
+ if [ "$rc" != 0 ]; then
+ log "Client load failed to restart on node $client, rc=$rc"
+ # failure one client load means test fail
+ # we do not need to check other
+ return $rc
+ fi
+ else
+ return $rc
+ fi
+ done
+}
+# End recovery-scale functions
+