+
+# Start vmstat and save its process ID in a file.
+start_vmstat() {
+ local nodes=$1
+ local pid_file=$2
+
+ [ -z "$nodes" -o -z "$pid_file" ] && return 0
+
+ do_nodes $nodes \
+ "vmstat 1 > $TESTLOG_PREFIX.$TESTNAME.vmstat.\\\$(hostname -s).log \
+ 2>/dev/null </dev/null & echo \\\$! > $pid_file"
+}
+
+# Display the nodes on which client loads failed.
+print_end_run_file() {
+ local file=$1
+ local node
+
+ [ -s $file ] || return 0
+
+ echo "Found the END_RUN_FILE file: $file"
+ cat $file
+
+ # A client load will stop if it finds the END_RUN_FILE file.
+ # That does not mean the client load actually failed though.
+ # The first node in END_RUN_FILE is the one we are interested in.
+ read node < $file
+
+ if [ -n "$node" ]; then
+ local var=$(node_var_name $node)_load
+
+ local prefix=$TESTLOG_PREFIX
+ [ -n "$TESTNAME" ] && prefix=$prefix.$TESTNAME
+ local stdout_log=$prefix.run_${!var}_stdout.$node.log
+ local debug_log=$(echo $stdout_log | sed 's/\(.*\)stdout/\1debug/')
+
+ echo "Client load ${!var} failed on node $node:"
+ echo "$stdout_log"
+ echo "$debug_log"
+ fi
+}
+
+# Stop the process which had its PID saved in a file.
+stop_process() {
+ local nodes=$1
+ local pid_file=$2
+
+ [ -z "$nodes" -o -z "$pid_file" ] && return 0
+
+ do_nodes $nodes "test -f $pid_file &&
+ { kill -s TERM \\\$(cat $pid_file); rm -f $pid_file; }" || true
+}
+
+# Stop all client loads.
+stop_client_loads() {
+ local nodes=${1:-$CLIENTS}
+ local pid_file=$2
+
+ # stop the client loads
+ stop_process $nodes $pid_file
+
+ # clean up the processes that started them
+ [ -n "$CLIENT_LOAD_PIDS" ] && kill -9 $CLIENT_LOAD_PIDS 2>/dev/null || true
+}